diff --git a/.github/actions/setup-dynamo-operator/action.yml b/.github/actions/setup-dynamo-operator/action.yml index eb5ea0f6c240..58ca80246f09 100644 --- a/.github/actions/setup-dynamo-operator/action.yml +++ b/.github/actions/setup-dynamo-operator/action.yml @@ -552,6 +552,9 @@ runs: Smoke: componentType: worker replicas: 0 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0 EOF } @@ -569,6 +572,11 @@ runs: - name: Smoke type: worker replicas: 0 + podTemplate: + spec: + containers: + - name: main + image: registry.example/runtime:1.1.0 EOF } diff --git a/.github/workflows/post-merge-ci.yml b/.github/workflows/post-merge-ci.yml index daf4e07da330..5f2331b202ff 100644 --- a/.github/workflows/post-merge-ci.yml +++ b/.github/workflows/post-merge-ci.yml @@ -687,6 +687,8 @@ jobs: needs: [vllm-copy-to-acr] if: github.event_name != 'workflow_dispatch' runs-on: prod-default-v2 + outputs: + image_tag: ${{ format('{0}-ci-{1}-vllm-placeholder', needs.vllm-copy-to-acr.outputs.runtime_version, github.sha) }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -705,9 +707,9 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-placeholder + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ format('{0}-ci-{1}-vllm-placeholder', needs.vllm-copy-to-acr.outputs.runtime_version, github.sha) }} extra_build_args: | - BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-runtime + BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.vllm-copy-to-acr.outputs.image_tag }} vllm-dev-copy-to-acr: name: vllm-dev # This name overlaps with other vllm jobs to group them in the UI @@ -735,6 +737,8 @@ jobs: needs: [sglang-copy-to-acr] if: github.event_name != 'workflow_dispatch' runs-on: prod-default-v2 + outputs: + image_tag: ${{ format('{0}-ci-{1}-sglang-placeholder', needs.sglang-copy-to-acr.outputs.runtime_version, github.sha) }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -753,9 +757,9 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-sglang-placeholder + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ format('{0}-ci-{1}-sglang-placeholder', needs.sglang-copy-to-acr.outputs.runtime_version, github.sha) }} extra_build_args: | - BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-sglang-runtime + BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.sglang-copy-to-acr.outputs.image_tag }} sglang-dev-copy-to-acr: name: sglang-dev # This name overlaps with other sglang jobs to group them in the UI @@ -786,6 +790,8 @@ jobs: needs: [trtllm-copy-to-acr] if: github.event_name != 'workflow_dispatch' runs-on: prod-default-v2 + outputs: + image_tag: ${{ format('{0}-ci-{1}-trtllm-placeholder', needs.trtllm-copy-to-acr.outputs.runtime_version, github.sha) }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -804,9 +810,9 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-trtllm-placeholder + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ format('{0}-ci-{1}-trtllm-placeholder', needs.trtllm-copy-to-acr.outputs.runtime_version, github.sha) }} extra_build_args: | - BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-trtllm-runtime + BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.trtllm-copy-to-acr.outputs.image_tag }} trtllm-dev-copy-to-acr: name: trtllm-dev # This name overlaps with other trtllm jobs to group them in the UI @@ -869,7 +875,7 @@ jobs: with: framework: vllm profiles: '["agg", "agg_router", "disagg", "disagg_router"]' - image_suffix: vllm-runtime + image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -881,7 +887,7 @@ jobs: with: framework: sglang profiles: '["agg", "agg_router"]' - image_suffix: sglang-runtime + image_tag: ${{ needs.sglang-copy-to-acr.outputs.image_tag }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -893,7 +899,7 @@ jobs: with: framework: trtllm profiles: '["agg", "agg_router"]' - image_suffix: trtllm-runtime + image_tag: ${{ needs.trtllm-copy-to-acr.outputs.image_tag }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -1174,13 +1180,13 @@ jobs: operator_tag: ${{ needs.deploy-operator-checkpoint-vllm.outputs.operator_tag }} framework: checkpoint profile: dgd_restore - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-placeholder + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.snapshot-placeholder-vllm.outputs.image_tag }} test_name: checkpoint_dgd_restore_vllm test_file: tests/deploy/test_dynamocheckpoint.py extra_pytest_args: >- -m dynamocheckpoint --checkpoint-backend=vllm - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} deploy-test-checkpoint-sglang: name: SGLang DynamoCheckpoint Deploy Test @@ -1238,13 +1244,13 @@ jobs: operator_tag: ${{ needs.deploy-operator-checkpoint-sglang.outputs.operator_tag }} framework: checkpoint profile: dgd_restore - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-sglang-placeholder + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.snapshot-placeholder-sglang.outputs.image_tag }} test_name: checkpoint_dgd_restore_sglang test_file: tests/deploy/test_dynamocheckpoint.py extra_pytest_args: >- -m dynamocheckpoint --checkpoint-backend=sglang - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} deploy-test-checkpoint-trtllm: name: TRTLLM DynamoCheckpoint Deploy Test @@ -1302,13 +1308,13 @@ jobs: operator_tag: ${{ needs.deploy-operator-checkpoint-trtllm.outputs.operator_tag }} framework: checkpoint profile: dgd_restore - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-trtllm-placeholder + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.snapshot-placeholder-trtllm.outputs.image_tag }} test_name: checkpoint_dgd_restore_trtllm test_file: tests/deploy/test_dynamocheckpoint.py extra_pytest_args: >- -m dynamocheckpoint --checkpoint-backend=trtllm - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} deploy-cleanup: if: always() @@ -1422,8 +1428,8 @@ jobs: test_name: gaie extra_pytest_args: >- -m framework_with_gaie - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend - --image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-runtime + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} + --image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.vllm-copy-to-acr.outputs.image_tag }} deploy-status-check: runs-on: ubuntu-latest diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 66f58eb6c1ab..2426f4a0ce9b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -992,6 +992,8 @@ jobs: (needs.changed-files.outputs.snapshot == 'true' || needs.changed-files.outputs.snapshot_vllm == 'true') runs-on: prod-default-v2 + outputs: + image_tag: ${{ format('{0}-ci-{1}-vllm-placeholder', needs.vllm-copy-to-acr.outputs.runtime_version, github.sha) }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -1010,9 +1012,9 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-placeholder + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ format('{0}-ci-{1}-vllm-placeholder', needs.vllm-copy-to-acr.outputs.runtime_version, github.sha) }} extra_build_args: | - BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-runtime + BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.vllm-copy-to-acr.outputs.image_tag }} sglang-copy-to-acr: name: sglang-runtime # This name overlaps with other sglang jobs to group them in the UI @@ -1042,6 +1044,8 @@ jobs: (needs.changed-files.outputs.snapshot == 'true' || needs.changed-files.outputs.snapshot_sglang == 'true') runs-on: prod-default-v2 + outputs: + image_tag: ${{ format('{0}-ci-{1}-sglang-placeholder', needs.sglang-copy-to-acr.outputs.runtime_version, github.sha) }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -1060,9 +1064,9 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-sglang-placeholder + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ format('{0}-ci-{1}-sglang-placeholder', needs.sglang-copy-to-acr.outputs.runtime_version, github.sha) }} extra_build_args: | - BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-sglang-runtime + BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.sglang-copy-to-acr.outputs.image_tag }} trtllm-copy-to-acr: name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI @@ -1092,6 +1096,8 @@ jobs: (needs.changed-files.outputs.snapshot == 'true' || needs.changed-files.outputs.snapshot_trtllm == 'true') runs-on: prod-default-v2 + outputs: + image_tag: ${{ format('{0}-ci-{1}-trtllm-placeholder', needs.trtllm-copy-to-acr.outputs.runtime_version, github.sha) }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -1110,9 +1116,9 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-trtllm-placeholder + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ format('{0}-ci-{1}-trtllm-placeholder', needs.trtllm-copy-to-acr.outputs.runtime_version, github.sha) }} extra_build_args: | - BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-trtllm-runtime + BASE_IMAGE=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.trtllm-copy-to-acr.outputs.image_tag }} # ============================================================================ # DEPLOY TEST PIPELINES @@ -1165,7 +1171,7 @@ jobs: with: framework: vllm profiles: '["agg", "agg_router", "disagg", "disagg_router"]' - image_suffix: vllm-runtime + image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -1183,7 +1189,7 @@ jobs: with: framework: sglang profiles: '["agg", "agg_router"]' - image_suffix: sglang-runtime + image_tag: ${{ needs.sglang-copy-to-acr.outputs.image_tag }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -1201,7 +1207,7 @@ jobs: with: framework: trtllm profiles: '["agg", "agg_router"]' - image_suffix: trtllm-runtime + image_tag: ${{ needs.trtllm-copy-to-acr.outputs.image_tag }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -1502,13 +1508,13 @@ jobs: operator_tag: ${{ needs.deploy-operator-checkpoint-vllm.outputs.operator_tag }} framework: checkpoint profile: dgd_restore - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-vllm-placeholder + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.snapshot-placeholder-vllm.outputs.image_tag }} test_name: checkpoint_dgd_restore_vllm test_file: tests/deploy/test_dynamocheckpoint.py extra_pytest_args: >- -m dynamocheckpoint --checkpoint-backend=vllm - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} deploy-test-checkpoint-sglang: name: SGLang DynamoCheckpoint Deploy Test @@ -1568,13 +1574,13 @@ jobs: operator_tag: ${{ needs.deploy-operator-checkpoint-sglang.outputs.operator_tag }} framework: checkpoint profile: dgd_restore - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-sglang-placeholder + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.snapshot-placeholder-sglang.outputs.image_tag }} test_name: checkpoint_dgd_restore_sglang test_file: tests/deploy/test_dynamocheckpoint.py extra_pytest_args: >- -m dynamocheckpoint --checkpoint-backend=sglang - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} deploy-test-checkpoint-trtllm: name: TRTLLM DynamoCheckpoint Deploy Test @@ -1634,13 +1640,13 @@ jobs: operator_tag: ${{ needs.deploy-operator-checkpoint-trtllm.outputs.operator_tag }} framework: checkpoint profile: dgd_restore - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-trtllm-placeholder + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.snapshot-placeholder-trtllm.outputs.image_tag }} test_name: checkpoint_dgd_restore_trtllm test_file: tests/deploy/test_dynamocheckpoint.py extra_pytest_args: >- -m dynamocheckpoint --checkpoint-backend=trtllm - --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-dynamo-frontend + --frontend-image=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.frontend-copy-to-acr.outputs.image_tag }} deploy-cleanup: if: always() diff --git a/.github/workflows/shared-copy.yml b/.github/workflows/shared-copy.yml index f0a1000232a8..995711c343aa 100644 --- a/.github/workflows/shared-copy.yml +++ b/.github/workflows/shared-copy.yml @@ -24,6 +24,13 @@ on: required: false type: string default: '' + outputs: + runtime_version: + description: 'Dynamo runtime compatibility version from Cargo.toml' + value: ${{ jobs.copy-to-acr.outputs.runtime_version }} + image_tag: + description: 'Image tag published to ACR' + value: ${{ jobs.copy-to-acr.outputs.image_tag }} secrets: AWS_DEFAULT_REGION: required: true @@ -38,6 +45,11 @@ on: jobs: copy-to-acr: + permissions: + contents: read + outputs: + runtime_version: ${{ steps.calculate-target-tag.outputs.runtime_version }} + image_tag: ${{ steps.calculate-target-tag.outputs.image_tag }} strategy: fail-fast: false matrix: @@ -56,17 +68,29 @@ jobs: CUDA_VERSION="${{ matrix.cuda_version }}" TARGET_TAG_PLAIN="${{ inputs.target_tag_plain }}" - IMAGE_TAG=${{ github.sha }}-${TARGET_TAG_PLAIN} + SOURCE_TAG=${{ github.sha }}-${TARGET_TAG_PLAIN} if [ -n "${CUDA_VERSION}" ]; then CUDA_MAJOR=${CUDA_VERSION%%.*} # cuda13 is the implicit default — the primary ECR tag carries # no cuda suffix. Non-13 majors (e.g. cuda12) stay explicit. # Keep in sync with shared-build-image.yml and shared-test.yml. if [ "${CUDA_MAJOR}" != "13" ]; then - IMAGE_TAG="${IMAGE_TAG}-cuda${CUDA_MAJOR}" + SOURCE_TAG="${SOURCE_TAG}-cuda${CUDA_MAJOR}" fi fi + DYNAMO_VERSION=$(sed -n \ + '/^\[workspace.package\]$/,/^\[/s/^version = "\([^"]*\)"/\1/p' \ + Cargo.toml) + DYNAMO_VERSION=${DYNAMO_VERSION%%[-+]*} + if [[ ! "${DYNAMO_VERSION}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "::error::Invalid Dynamo workspace version: ${DYNAMO_VERSION}" + exit 1 + fi + IMAGE_TAG="${DYNAMO_VERSION}-ci-${SOURCE_TAG}" + + echo "source_tag=${SOURCE_TAG}" >> $GITHUB_OUTPUT + echo "runtime_version=${DYNAMO_VERSION}" >> $GITHUB_OUTPUT echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT - name: Copy image to target registry timeout-minutes: ${{ inputs.copy_timeout_minutes }} @@ -74,7 +98,7 @@ jobs: with: source_registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com source_image: ai-dynamo/dynamo - source_tag: ${{ steps.calculate-target-tag.outputs.image_tag }} + source_tag: ${{ steps.calculate-target-tag.outputs.source_tag }} target_registry: ${{ secrets.AZURE_ACR_HOSTNAME }} target_image: ai-dynamo/dynamo target_tag: ${{ steps.calculate-target-tag.outputs.image_tag }} diff --git a/.github/workflows/shared-deploy-test.yml b/.github/workflows/shared-deploy-test.yml index 45176c986ae1..da6130006c57 100644 --- a/.github/workflows/shared-deploy-test.yml +++ b/.github/workflows/shared-deploy-test.yml @@ -14,8 +14,8 @@ on: description: 'JSON array of deployment profiles to test' type: string required: true - image_suffix: - description: 'Image suffix (e.g. vllm-runtime-cuda13-amd64)' + image_tag: + description: 'Runtime image tag' type: string required: true namespace: @@ -82,7 +82,7 @@ jobs: hf_token: ${{ secrets.HF_TOKEN }} framework: ${{ inputs.framework }} profile: ${{ matrix.profile }} - image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ github.sha }}-${{ inputs.image_suffix }} + image: ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ inputs.image_tag }} platform_arch: amd64 extra_pytest_args: -m framework_only # Mount the shared model cache only when both endpoint vars are set diff --git a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py b/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py index 0abcde031f79..7d277af852a9 100644 --- a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py +++ b/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py @@ -12,7 +12,11 @@ try: from dynamo.profiler.utils.config import update_image - from dynamo.profiler.utils.dgd_generation import add_planner_to_config + from dynamo.profiler.utils.dgd_generation import ( + add_planner_to_config, + apply_runtime_version_override, + assemble_final_config, + ) from dynamo.profiler.utils.dgdr_v1beta1_types import ( DynamoGraphDeploymentRequestSpec, HardwareSpec, @@ -20,6 +24,7 @@ WorkloadSpec, ) from dynamo.profiler.utils.profile_common import ( + ProfilerOperationalConfig, derive_backend_image, derive_planner_image, ) @@ -27,11 +32,14 @@ pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) -def _make_dgdr(image: str) -> DynamoGraphDeploymentRequestSpec: +def _make_dgdr( + image: str, runtime_version_override: str | None = None +) -> DynamoGraphDeploymentRequestSpec: return DynamoGraphDeploymentRequestSpec( model="Qwen/Qwen3-32B", backend="trtllm", image=image, + runtimeVersionOverride=runtime_version_override, hardware=HardwareSpec(gpuSku="h200_sxm", totalGpus=8, numGpusPerNode=8), workload=WorkloadSpec(isl=4000, osl=1000), sla=SLASpec(ttft=2000.0, itl=50.0), @@ -119,6 +127,30 @@ def test_add_planner_to_config_uses_dynamo_planner_image(): assert planner_image == "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3" +def test_assemble_final_config_applies_runtime_version_override(): + image = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:custom" + dgdr = _make_dgdr(image, runtime_version_override="1.2.3") + config = _base_dgd_config(image) + + result = assemble_final_config(dgdr, ProfilerOperationalConfig(), config) + + assert result["spec"]["services"]["Frontend"]["runtimeVersionOverride"] == "1.2.3" + + +def test_runtime_version_override_applies_to_injected_planner(): + image = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:custom" + dgdr = _make_dgdr(image, runtime_version_override="1.2.3") + config = _base_dgd_config(image) + add_planner_to_config(dgdr, config) + + apply_runtime_version_override(dgdr, config) + + assert { + name: service["runtimeVersionOverride"] + for name, service in config["spec"]["services"].items() + } == {"Frontend": "1.2.3", "Planner": "1.2.3"} + + def test_update_image_does_not_overwrite_planner_service_image(): profiler_image = "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.3" worker_image = "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3" diff --git a/components/src/dynamo/profiler/utils/dgd_generation.py b/components/src/dynamo/profiler/utils/dgd_generation.py index c1aeae58aafe..51ad0e9ffa18 100644 --- a/components/src/dynamo/profiler/utils/dgd_generation.py +++ b/components/src/dynamo/profiler/utils/dgd_generation.py @@ -113,6 +113,7 @@ def assemble_final_config( enable_trtllm_chunked_prefill(dgd_config) if not mocker and not planner: + apply_runtime_version_override(dgdr, dgd_config) return dgd_config # Save picked config for auditing @@ -156,11 +157,24 @@ def assemble_final_config( if profile_cm: config_maps.append(profile_cm) + apply_runtime_version_override(dgdr, base) if config_maps: return config_maps + [base] return base +def apply_runtime_version_override(dgdr, config_dict: dict) -> None: + """Apply the DGDR runtime version to every generated DGD service.""" + override = dgdr.runtimeVersionOverride + if not override: + return + + services = config_dict.get("spec", {}).get("services", {}) + for service_config in services.values(): + if isinstance(service_config, dict): + service_config["runtimeVersionOverride"] = override + + def _vllm_worker_roles() -> dict[str, str]: """Canonical DGD service name → DYN_BENCHMARK_MODE role. diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index e51c52492615..ed08c6340012 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -256,6 +256,10 @@ class DynamoGraphDeploymentRequestSpec(BaseModel): default=None, description='Image is the container image reference for the profiling job (planner image). Example: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.1". For Dynamo < 1.1.0, use dynamo-frontend.', ) + runtimeVersionOverride: Optional[str] = Field( + default=None, + description="RuntimeVersionOverride supplies the default Dynamo runtime version for generated DynamoGraphDeployment components that do not set their own override. Set this when Image uses a non-semantic-version tag or digest, or when its tag does not identify the Dynamo runtime version. An explicit component value in overrides.dgd takes precedence.", + ) modelCache: Optional[ModelCacheSpec] = Field( default=None, description="ModelCache provides optional PVC configuration for pre-downloaded model weights. When provided, weights are loaded from the PVC instead of downloading from HuggingFace.", diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 0b204aa975c4..65673a2fb152 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -41,6 +41,48 @@ The Dynamo Platform Helm chart deploys the complete Dynamo Kubernetes Platform i ## 🔄 Upgrading Notes +### Runtime version override for custom runtime images (v1.4.0+) + +Each `DynamoGraphDeployment` (DGD) or standalone `DynamoComponentDeployment` (DCD) +component now resolves its Dynamo runtime compatibility version from its main container image. +Use a semantic-version image tag such as `:1.4.0`, or set +`runtimeVersionOverride` to the compatible Dynamo runtime version when the image uses a digest, +custom build tag, or mutable non-semantic-version tag such as `:latest`. The override takes +precedence over the image-derived version only when the operator selects runtime-version-specific +flags, environment variables, and behavior. It never changes the image reference or rendered Pod: +the configured image, including `:latest`, runs unchanged. + +```yaml +components: + - name: worker + runtimeVersionOverride: 1.4.0 + podTemplate: + spec: + containers: + - name: main + image: registry.example/my-runtime:build-20260723 +``` + +Admission requires every new component's main-container image. In `v1beta1`, set +`spec.components[*].podTemplate.spec.containers[name=main].image` for a DGD, or +`spec.podTemplate.spec.containers[name=main].image` for a standalone DCD. In `v1alpha1`, set +`spec.services..extraPodSpec.mainContainer.image` for a DGD, or +`spec.extraPodSpec.mainContainer.image` for a standalone DCD. These fields were not previously +required by Dynamo admission, but were effectively required: Kubernetes rejects the rendered Pod +specification when its main container has no image. + +After upgrading the CRDs and operator, admission denies a new DGD component when its main image is +not tagged with a semantic version and `runtimeVersionOverride` is unset. The same applies to a +DGDR whose `spec.image` is not tagged with a semantic version. This includes digest references, +custom tags, and `:latest`. Controller-generated standalone DCDs may omit the override. + +These requirements are ratcheted on updates: a pre-existing component with an unchanged missing +pod configuration or main image remains admissible, as does an unchanged non-semantic-version +image without an override. Adding or changing an image applies the current validation, and removing +an existing pod configuration or main image is rejected. Changing the image to a +non-semantic-version tag requires setting `runtimeVersionOverride` to that image's Dynamo runtime +compatibility version in the same update. + ### Bundled NATS is now disabled by default The bundled NATS subchart is no longer installed by default because Dynamo's default request and diff --git a/deploy/helm/charts/platform/README.md.gotmpl b/deploy/helm/charts/platform/README.md.gotmpl index dbcf01ffec1a..24910e6f45ed 100644 --- a/deploy/helm/charts/platform/README.md.gotmpl +++ b/deploy/helm/charts/platform/README.md.gotmpl @@ -41,6 +41,48 @@ The Dynamo Platform Helm chart deploys the complete Dynamo Kubernetes Platform i ## 🔄 Upgrading Notes +### Runtime version override for custom runtime images (v1.4.0+) + +Each `DynamoGraphDeployment` (DGD) or standalone `DynamoComponentDeployment` (DCD) +component now resolves its Dynamo runtime compatibility version from its main container image. +Use a semantic-version image tag such as `:1.4.0`, or set +`runtimeVersionOverride` to the compatible Dynamo runtime version when the image uses a digest, +custom build tag, or mutable non-semantic-version tag such as `:latest`. The override takes +precedence over the image-derived version only when the operator selects runtime-version-specific +flags, environment variables, and behavior. It never changes the image reference or rendered Pod: +the configured image, including `:latest`, runs unchanged. + +```yaml +components: + - name: worker + runtimeVersionOverride: 1.4.0 + podTemplate: + spec: + containers: + - name: main + image: registry.example/my-runtime:build-20260723 +``` + +Admission requires every new component's main-container image. In `v1beta1`, set +`spec.components[*].podTemplate.spec.containers[name=main].image` for a DGD, or +`spec.podTemplate.spec.containers[name=main].image` for a standalone DCD. In `v1alpha1`, set +`spec.services..extraPodSpec.mainContainer.image` for a DGD, or +`spec.extraPodSpec.mainContainer.image` for a standalone DCD. These fields were not previously +required by Dynamo admission, but were effectively required: Kubernetes rejects the rendered Pod +specification when its main container has no image. + +After upgrading the CRDs and operator, admission denies a new DGD component when its main image is +not tagged with a semantic version and `runtimeVersionOverride` is unset. The same applies to a +DGDR whose `spec.image` is not tagged with a semantic version. This includes digest references, +custom tags, and `:latest`. Controller-generated standalone DCDs may omit the override. + +These requirements are ratcheted on updates: a pre-existing component with an unchanged missing +pod configuration or main image remains admissible, as does an unchanged non-semantic-version +image without an override. Adding or changing an image applies the current validation, and removing +an existing pod configuration or main image is rejected. Changing the image to a +non-semantic-version tag requires setting `runtimeVersionOverride` to that image's Dynamo runtime +compatibility version in the same update. + ### Bundled NATS is now disabled by default The bundled NATS subchart is no longer installed by default because Dynamo's default request and diff --git a/deploy/operator/AGENTS.md b/deploy/operator/AGENTS.md index 1016c6cd806c..0f9a88c9738d 100644 --- a/deploy/operator/AGENTS.md +++ b/deploy/operator/AGENTS.md @@ -9,6 +9,14 @@ SPDX-License-Identifier: Apache-2.0 - Keep chart-only grants in the manual section of the platform chart's `../helm/charts/platform/components/operator/templates/manager-rbac.yaml`. +## Go Code Style + +- Put a one-line story comment above every multi-line block of logically + connected code. +- Separate multi-line semantic blocks from surrounding code with one blank + line. Do not add trailing blank lines before a closing delimiter or between + a block-leading comment and its code. + ## Go Test Style - Use `t.Log` to tell the test's story, with one heading before each block that diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 90c1879770e6..3a15a1367e2b 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -67,6 +67,7 @@ DynamoComponentDeploymentSpec.multinode.nodeCount DynamoComponentDeploymentSpec.name DynamoComponentDeploymentSpec.podTemplate DynamoComponentDeploymentSpec.replicas +DynamoComponentDeploymentSpec.runtimeVersionOverride DynamoComponentDeploymentSpec.scalingAdapter DynamoComponentDeploymentSpec.sharedMemorySize DynamoComponentDeploymentSpec.topologyConstraint.packDomain @@ -125,6 +126,7 @@ DynamoGraphDeploymentSpec.components.multinode.nodeCount DynamoGraphDeploymentSpec.components.name DynamoGraphDeploymentSpec.components.podTemplate DynamoGraphDeploymentSpec.components.replicas +DynamoGraphDeploymentSpec.components.runtimeVersionOverride DynamoGraphDeploymentSpec.components.scalingAdapter DynamoGraphDeploymentSpec.components.sharedMemorySize DynamoGraphDeploymentSpec.components.topologyConstraint.packDomain diff --git a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go index 5de3e6aa04c2..3c3dd32fd273 100644 --- a/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamocomponentdeployment_types.go @@ -69,6 +69,15 @@ type DynamoComponentDeploymentSharedSpec struct { // SubComponentType indicates the sub-role of this component (for example, "prefill"). SubComponentType string `json:"subComponentType,omitempty"` + // RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's + // main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable + // semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is + // not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0". + // It does not change the image or rendered Pod, and changing only this field does not trigger a rollout. + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` + // +optional + RuntimeVersionOverride string `json:"runtimeVersionOverride,omitempty"` + // DynamoNamespace is deprecated and will be removed in a future version. // The DGD Kubernetes namespace and DynamoGraphDeployment name are used to construct the Dynamo namespace for each component // +kubebuilder:validation:Optional @@ -109,7 +118,9 @@ type DynamoComponentDeploymentSharedSpec struct { // +optional // ExtraPodSpec allows to override the main pod spec configuration. // It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field - // that allows overriding the main container configuration. + // that allows overriding the main container configuration. New components must set + // extraPodSpec and provide a non-empty mainContainer image. Existing components created + // without extraPodSpec may remain unchanged. ExtraPodSpec *ExtraPodSpec `json:"extraPodSpec,omitempty"` // LivenessProbe to detect and restart unhealthy containers. diff --git a/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion.go index d69e0e031917..6d2eb9fcf663 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion.go @@ -259,6 +259,7 @@ func restoreDGDRSpokeStatus(raw string) (DynamoGraphDeploymentRequestStatus, boo // v1alpha1 to v1beta1. func ConvertFromDynamoGraphDeploymentRequestSpec(src *DynamoGraphDeploymentRequestSpec, dst *v1beta1.DynamoGraphDeploymentRequestSpec, restored *v1beta1.DynamoGraphDeploymentRequestSpec, save *DynamoGraphDeploymentRequestSpec) error { dst.Model = src.Model + dst.RuntimeVersionOverride = src.RuntimeVersionOverride autoApply := src.AutoApply dst.AutoApply = &autoApply @@ -502,6 +503,7 @@ func projectProfilingConfigToProfilingJob(src *ProfilingConfigSpec, dst *v1beta1 // v1beta1 to v1alpha1. func ConvertToDynamoGraphDeploymentRequestSpec(src *v1beta1.DynamoGraphDeploymentRequestSpec, dst *DynamoGraphDeploymentRequestSpec, restored *DynamoGraphDeploymentRequestSpec, save *v1beta1.DynamoGraphDeploymentRequestSpec) { dst.Model = src.Model + dst.RuntimeVersionOverride = src.RuntimeVersionOverride if src.AutoApply != nil { dst.AutoApply = *src.AutoApply } else { diff --git a/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion_test.go index 9c248cbabd40..bcc9a8c09381 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_conversion_test.go @@ -63,10 +63,11 @@ func newV1alpha1DGDR() *DynamoGraphDeploymentRequest { Namespace: "default", }, Spec: DynamoGraphDeploymentRequestSpec{ - Model: "meta-llama/Llama-3.1-8B", - Backend: "vllm", - AutoApply: true, - UseMocker: true, + Model: "meta-llama/Llama-3.1-8B", + Backend: "vllm", + RuntimeVersionOverride: "1.2.3", + AutoApply: true, + UseMocker: true, ProfilingConfig: ProfilingConfigSpec{ ProfilerImage: "nvcr.io/nvidia/dynamo:latest", OutputPVC: "output-pvc", @@ -112,10 +113,11 @@ func newV1beta1DGDR() *v1beta1.DynamoGraphDeploymentRequest { Namespace: "default", }, Spec: v1beta1.DynamoGraphDeploymentRequestSpec{ - Model: "Qwen/Qwen3-32B", - Backend: v1beta1.BackendTypeVllm, - AutoApply: &autoApplyFalse, - Image: "nvcr.io/nvidia/dynamo:0.3.2", + Model: "Qwen/Qwen3-32B", + Backend: v1beta1.BackendTypeVllm, + AutoApply: &autoApplyFalse, + Image: "nvcr.io/nvidia/dynamo:custom", + RuntimeVersionOverride: "1.2.3", SLA: &v1beta1.SLASpec{ TTFT: &ttft, ITL: &itl, @@ -166,6 +168,9 @@ func TestConvertTo_SpecFields(t *testing.T) { if dst.Spec.AutoApply == nil || *dst.Spec.AutoApply != src.Spec.AutoApply { t.Errorf("AutoApply: got %v, want %v", dst.Spec.AutoApply, src.Spec.AutoApply) } + if dst.Spec.RuntimeVersionOverride != src.Spec.RuntimeVersionOverride { + t.Errorf("RuntimeVersionOverride: got %q, want %q", dst.Spec.RuntimeVersionOverride, src.Spec.RuntimeVersionOverride) + } // ProfilerImage → Image if dst.Spec.Image != src.Spec.ProfilingConfig.ProfilerImage { diff --git a/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_types.go index ee319f4e1caf..63e0dfe4ae98 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeploymentrequest_types.go @@ -176,6 +176,14 @@ type DynamoGraphDeploymentRequestSpec struct { // +kubebuilder:validation:Required ProfilingConfig ProfilingConfigSpec `json:"profilingConfig"` + // RuntimeVersionOverride explicitly sets the Dynamo runtime version for every + // component in the generated DynamoGraphDeployment. Set this when + // profilingConfig.profilerImage uses a non-semantic-version tag or digest, or + // when its tag does not identify the Dynamo runtime version. + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` + // +optional + RuntimeVersionOverride string `json:"runtimeVersionOverride,omitempty"` + // EnableGPUDiscovery controls whether the operator attempts to discover GPU hardware from cluster nodes. // DEPRECATED: This field is deprecated and will be removed in v1beta1. GPU discovery is now always // attempted automatically. Setting this field has no effect - the operator will always try to discover diff --git a/deploy/operator/api/v1alpha1/legacy_worker_hash.go b/deploy/operator/api/v1alpha1/legacy_worker_hash.go index c1116679fe9b..8b8feee521fe 100644 --- a/deploy/operator/api/v1alpha1/legacy_worker_hash.go +++ b/deploy/operator/api/v1alpha1/legacy_worker_hash.go @@ -59,6 +59,7 @@ func stripV1alpha1NonPodTemplateFields(spec *DynamoComponentDeploymentSharedSpec stripped.ServiceName = "" stripped.ComponentType = "" stripped.SubComponentType = "" + stripped.RuntimeVersionOverride = "" stripped.DynamoNamespace = nil stripped.Replicas = nil stripped.Autoscaling = nil //nolint:staticcheck // SA1019: intentionally matching the old v1alpha1 worker hash diff --git a/deploy/operator/api/v1alpha1/shared_spec_conversion.go b/deploy/operator/api/v1alpha1/shared_spec_conversion.go index 0f7dfc4781e4..296548c71d75 100644 --- a/deploy/operator/api/v1alpha1/shared_spec_conversion.go +++ b/deploy/operator/api/v1alpha1/shared_spec_conversion.go @@ -149,6 +149,7 @@ func ConvertFromDynamoComponentDeploymentSharedSpec(src *DynamoComponentDeployme // source of truth on v1alpha1); for standalone DCDs the caller falls // back to ObjectMeta.Name when src.ServiceName is empty. dst.ComponentName = src.ServiceName + dst.RuntimeVersionOverride = src.RuntimeVersionOverride dst.GlobalDynamoNamespace = src.GlobalDynamoNamespace dst.Replicas = src.Replicas @@ -258,9 +259,13 @@ func restoreSharedAlphaOnlyPodFields(dst *DynamoComponentDeploymentSharedSpec, p if dst.ExtraPodMetadata == nil && extraPodMetadataNeedsPreservation(preserved.ExtraPodMetadata) { dst.ExtraPodMetadata = preserved.ExtraPodMetadata.DeepCopy() } - if dst.ExtraPodSpec == nil && shouldRestorePreservedExtraPodSpec(dst, preserved) { - cp := *preserved.ExtraPodSpec.DeepCopy() - dst.ExtraPodSpec = &cp + if shouldRestorePreservedExtraPodSpec(dst, preserved) { + if dst.ExtraPodSpec == nil { + cp := *preserved.ExtraPodSpec.DeepCopy() + dst.ExtraPodSpec = &cp + } else { + restorePreservedFrontendSidecarConflict(dst.ExtraPodSpec, preserved.ExtraPodSpec) + } } restoreMainContainerFieldOrigins(dst, preserved, mainContainerPresent) if dst.ExtraPodSpec != nil && dst.ExtraPodSpec.MainContainer != nil && @@ -270,6 +275,23 @@ func restoreSharedAlphaOnlyPodFields(dst *DynamoComponentDeploymentSharedSpec, p } } +func restorePreservedFrontendSidecarConflict(dst, preserved *ExtraPodSpec) { + if dst == nil || preserved == nil || preserved.PodSpec == nil { + return + } + container, found := findContainerByName(preserved.PodSpec.Containers, defaultFrontendSidecarContainerName) + if !found { + return + } + if dst.PodSpec == nil { + dst.PodSpec = &corev1.PodSpec{} + } + if _, found := findContainerByName(dst.PodSpec.Containers, defaultFrontendSidecarContainerName); found { + return + } + dst.PodSpec.Containers = append(dst.PodSpec.Containers, container) +} + func restoreSharedAlphaOnlyDisabledFeatures(dst *DynamoComponentDeploymentSharedSpec, preserved *DynamoComponentDeploymentSharedSpec) { if dst.ScalingAdapter == nil && preserved.ScalingAdapter != nil && !preserved.ScalingAdapter.Enabled { dst.ScalingAdapter = preserved.ScalingAdapter.DeepCopy() @@ -520,6 +542,7 @@ func ConvertToDynamoComponentDeploymentSharedSpec(src *v1beta1.DynamoComponentDe ConvertToEPPConfig(src.EPPConfig, dst.EPPConfig) } + dst.RuntimeVersionOverride = src.RuntimeVersionOverride dst.ServiceName = src.ComponentName // sharedMemorySize -> SharedMemorySpec. diff --git a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go index ce9644d8336f..84afeedf3294 100644 --- a/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamocomponentdeployment_types.go @@ -37,12 +37,11 @@ const ( DynamoComponentDeploymentConditionTypeDynamoComponentReady = "DynamoComponentReady" // MainContainerName is the well-known name of the primary Dynamo workload - // container inside a component's `podTemplate.spec.containers`. The operator - // injects its defaults (image, command, env, ports, probes, resources, - // volume mounts) into this container. If no container with this name is - // present in the user-supplied `podTemplate`, the operator auto-generates - // it. Any other container in the `podTemplate` is treated as a user-managed - // sidecar. + // container inside a component's podTemplate.spec.containers. The + // podTemplate must include this container with a non-empty image. The + // operator injects its defaults (command, env, ports, probes, resources, + // volume mounts) into this container. Any other container in the + // podTemplate is treated as a user-managed sidecar. MainContainerName = "main" ) @@ -98,22 +97,31 @@ type DynamoComponentDeploymentSharedSpec struct { // +optional ComponentType ComponentType `json:"type,omitempty"` + // RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's + // main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has + // no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the + // parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for + // example "1.4.0". It does not change the image or rendered Pod, and changing only this field does + // not trigger a rollout. + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` + // +optional + RuntimeVersionOverride string `json:"runtimeVersionOverride,omitempty"` + // globalDynamoNamespace places the component in the global Dynamo // namespace rather than the per-deployment namespace derived from the // DGD name. // +optional GlobalDynamoNamespace bool `json:"globalDynamoNamespace,omitempty"` - // podTemplate is the pod template used to create the component's pods. - // The operator injects its defaults (image, command, env, ports, probes, - // resources, volume mounts) into the container named `"main"` inside - // `podTemplate.spec.containers`, merging user overrides by name. If no - // container named `"main"` is present, the operator auto-generates it - // with standard defaults. All other containers in `podTemplate.spec.containers` - // are treated as user-managed sidecars: the operator does not inject - // defaults into them, so sidecars must specify required fields (e.g. `image`) - // themselves. The validation webhook rejects pod templates where a - // non-`"main"` container is missing a required field such as `image`. + // podTemplate defines the component's Pod configuration. New components must + // include a container named "main" with a non-empty image. Existing components + // created without a podTemplate may remain unchanged. The operator merges + // defaults into the main container. + // For DGD components whose main image tag is not a Dynamo semantic version, + // set runtimeVersionOverride explicitly. + // + // All other containers are user-managed sidecars and must specify their + // required fields, including image. // +optional PodTemplate *corev1.PodTemplateSpec `json:"podTemplate,omitempty"` diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index e8616ffaee9d..cbba6a434fcf 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -461,6 +461,15 @@ type DynamoGraphDeploymentRequestSpec struct { // +optional Image string `json:"image,omitempty"` + // RuntimeVersionOverride supplies the default Dynamo runtime version for + // generated DynamoGraphDeployment components that do not set their own + // override. Set this when Image uses a non-semantic-version tag or digest, or + // when its tag does not identify the Dynamo runtime version. An explicit + // component value in overrides.dgd takes precedence. + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` + // +optional + RuntimeVersionOverride string `json:"runtimeVersionOverride,omitempty"` + // ModelCache provides optional PVC configuration for pre-downloaded model weights. // When provided, weights are loaded from the PVC instead of downloading from HuggingFace. // +optional diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml index ca49a34cba96..f02aa035a01e 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamocomponentdeployments.yaml @@ -1146,7 +1146,9 @@ spec: description: |- ExtraPodSpec allows to override the main pod spec configuration. It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field - that allows overriding the main container configuration. + that allows overriding the main container configuration. New components must set + extraPodSpec and provide a non-empty mainContainer image. Existing components created + without extraPodSpec may remain unchanged. properties: activeDeadlineSeconds: description: |- @@ -11447,6 +11449,15 @@ spec: type: string type: object type: object + runtimeVersionOverride: + description: |- + RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's + main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable + semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is + not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0". + It does not change the image or rendered Pod, and changing only this field does not trigger a rollout. + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ + type: string scalingAdapter: description: |- ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter. @@ -12262,16 +12273,15 @@ spec: type: string podTemplate: description: |- - podTemplate is the pod template used to create the component's pods. - The operator injects its defaults (image, command, env, ports, probes, - resources, volume mounts) into the container named `"main"` inside - `podTemplate.spec.containers`, merging user overrides by name. If no - container named `"main"` is present, the operator auto-generates it - with standard defaults. All other containers in `podTemplate.spec.containers` - are treated as user-managed sidecars: the operator does not inject - defaults into them, so sidecars must specify required fields (e.g. `image`) - themselves. The validation webhook rejects pod templates where a - non-`"main"` container is missing a required field such as `image`. + podTemplate defines the component's Pod configuration. New components must + include a container named "main" with a non-empty image. Existing components + created without a podTemplate may remain unchanged. The operator merges + defaults into the main container. + For DGD components whose main image tag is not a Dynamo semantic version, + set runtimeVersionOverride explicitly. + + All other containers are user-managed sidecars and must specify their + required fields, including image. properties: metadata: description: |- @@ -20403,6 +20413,16 @@ spec: format: int32 minimum: 0 type: integer + runtimeVersionOverride: + description: |- + RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's + main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has + no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the + parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for + example "1.4.0". It does not change the image or rendered Pod, and changing only this field does + not trigger a rollout. + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ + type: string scalingAdapter: description: |- scalingAdapter opts this component into using the diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 485fe3d1f59e..09e284cca165 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -314,6 +314,14 @@ spec: required: - profilerImage type: object + runtimeVersionOverride: + description: |- + RuntimeVersionOverride explicitly sets the Dynamo runtime version for every + component in the generated DynamoGraphDeployment. Set this when + profilingConfig.profilerImage uses a non-semantic-version tag or digest, or + when its tag does not identify the Dynamo runtime version. + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ + type: string useMocker: default: false description: |- @@ -9228,6 +9236,15 @@ spec: - template type: object type: object + runtimeVersionOverride: + description: |- + RuntimeVersionOverride supplies the default Dynamo runtime version for + generated DynamoGraphDeployment components that do not set their own + override. Set this when Image uses a non-semantic-version tag or digest, or + when its tag does not identify the Dynamo runtime version. An explicit + component value in overrides.dgd takes precedence. + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ + type: string searchStrategy: allOf: - enum: diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index f39545afcf05..7a8e63266a78 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -1446,7 +1446,9 @@ spec: description: |- ExtraPodSpec allows to override the main pod spec configuration. It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field - that allows overriding the main container configuration. + that allows overriding the main container configuration. New components must set + extraPodSpec and provide a non-empty mainContainer image. Existing components created + without extraPodSpec may remain unchanged. properties: activeDeadlineSeconds: description: |- @@ -11747,6 +11749,15 @@ spec: type: string type: object type: object + runtimeVersionOverride: + description: |- + RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's + main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable + semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is + not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0". + It does not change the image or rendered Pod, and changing only this field does not trigger a rollout. + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ + type: string scalingAdapter: description: |- ScalingAdapter configures whether this service uses the DynamoGraphDeploymentScalingAdapter. @@ -12720,16 +12731,15 @@ spec: type: string podTemplate: description: |- - podTemplate is the pod template used to create the component's pods. - The operator injects its defaults (image, command, env, ports, probes, - resources, volume mounts) into the container named `"main"` inside - `podTemplate.spec.containers`, merging user overrides by name. If no - container named `"main"` is present, the operator auto-generates it - with standard defaults. All other containers in `podTemplate.spec.containers` - are treated as user-managed sidecars: the operator does not inject - defaults into them, so sidecars must specify required fields (e.g. `image`) - themselves. The validation webhook rejects pod templates where a - non-`"main"` container is missing a required field such as `image`. + podTemplate defines the component's Pod configuration. New components must + include a container named "main" with a non-empty image. Existing components + created without a podTemplate may remain unchanged. The operator merges + defaults into the main container. + For DGD components whose main image tag is not a Dynamo semantic version, + set runtimeVersionOverride explicitly. + + All other containers are user-managed sidecars and must specify their + required fields, including image. properties: metadata: description: |- @@ -20861,6 +20871,16 @@ spec: format: int32 minimum: 0 type: integer + runtimeVersionOverride: + description: |- + RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's + main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has + no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the + parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for + example "1.4.0". It does not change the image or rendered Pod, and changing only this field does + not trigger a rollout. + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ + type: string scalingAdapter: description: |- scalingAdapter opts this component into using the diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index 9ddd8dfeae35..a6ad81a8d3c2 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -195,7 +195,7 @@ func TestClusterDynamoGraphDeploymentRequestProfilesAndCreatesWorkloadManifests( t.Fatalf("create profiler token secret: %v", err) } - t.Log("Apply the DGDR input manifest through Kubernetes admission and select the local profiler image") + t.Log("Apply the DGDR input manifest through Kubernetes admission and select the local profiler image with its runtime compatibility") objects := golden.ApplyManifests(t, filepath.Join(scenarioDir, "input.yaml"), env.Client(), env.Namespace()) if len(objects) != 1 || objects[0].GetKind() != "DynamoGraphDeploymentRequest" { t.Fatalf("scenario input contains %d objects, want one DynamoGraphDeploymentRequest", len(objects)) @@ -205,6 +205,7 @@ func TestClusterDynamoGraphDeploymentRequestProfilesAndCreatesWorkloadManifests( t.Fatalf("get admitted DGDR: %v", err) } dgdr.Spec.Image = profilerImage + dgdr.Spec.RuntimeVersionOverride = "1.4.0" if err := env.Client().Update(ctx, dgdr); err != nil { t.Fatalf("select profiler image: %v", err) } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 9dd7c224d4c8..206d6e1b887c 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -462,24 +462,35 @@ func (r *DynamoGraphDeploymentRequestReconciler) Reconcile(ctx context.Context, return ctrl.Result{}, nil } - // Check for spec changes (immutability enforcement) - if dgdr.Status.ObservedGeneration > 0 && dgdr.Status.ObservedGeneration != dgdr.Generation { - // Spec changed after initial processing - if dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseProfiling || dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseDeploying || - dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseReady || dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseDeployed { - logger.Info("Spec change detected in immutable phase", - "phase", dgdr.Status.Phase, - "observedGeneration", dgdr.Status.ObservedGeneration, - "currentGeneration", dgdr.Generation) - - r.Recorder.Event(dgdr, corev1.EventTypeWarning, nvidiacomv1beta1.EventReasonSpecChangeRejected, - fmt.Sprintf(MessageSpecChangeRejected, dgdr.Status.Phase)) - - // Keep the old observedGeneration to continue rejecting changes - // No phase transition - stay in current phase with old spec - return ctrl.Result{}, nil - } + // Admission permits deferred requests to select a runtime version while + // autoApply is disabled and Ready requests to enable autoApply. + immutablePhase := dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseProfiling || + dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseDeploying || + dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseReady || + dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseDeployed + autoApplyDisabled := dgdr.Spec.AutoApply != nil && !*dgdr.Spec.AutoApply + deferredRuntimeVersionUpdate := autoApplyDisabled && + (dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseProfiling || + dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseReady) + readyAutoApplyActivation := dgdr.Status.Phase == nvidiacomv1beta1.DGDRPhaseReady && + (dgdr.Spec.AutoApply == nil || *dgdr.Spec.AutoApply) + + // Reject unexpected generation changes after profiling starts. + if dgdr.Status.ObservedGeneration > 0 && + dgdr.Status.ObservedGeneration != dgdr.Generation && + immutablePhase && + !deferredRuntimeVersionUpdate && + !readyAutoApplyActivation { + logger.Info("Spec change detected in immutable phase", + "phase", dgdr.Status.Phase, + "observedGeneration", dgdr.Status.ObservedGeneration, + "currentGeneration", dgdr.Generation) + + r.Recorder.Event(dgdr, corev1.EventTypeWarning, nvidiacomv1beta1.EventReasonSpecChangeRejected, + fmt.Sprintf(MessageSpecChangeRejected, dgdr.Status.Phase)) + return ctrl.Result{}, nil } + // Phase machine: handle different phases switch dgdr.Status.Phase { case nvidiacomv1beta1.DGDRPhasePending, "": @@ -779,6 +790,12 @@ func (r *DynamoGraphDeploymentRequestReconciler) handleReadyPhase(ctx context.Co logger := log.FromContext(ctx) logger.Info("DGDR is ready", "name", dgdr.Name) + // Start deployment when autoApply is enabled after manual review. + if dgdr.Spec.AutoApply == nil || *dgdr.Spec.AutoApply { + logger.Info("AutoApply enabled, transitioning to Deploying phase") + return r.updatePhaseWithCondition(ctx, dgdr, nvidiacomv1beta1.DGDRPhaseDeploying, nvidiacomv1beta1.ConditionTypeSpecGenerated, metav1.ConditionTrue, nvidiacomv1beta1.EventReasonSpecGenerated, MessageSpecGenerated) + } + // Nothing to monitor in Ready phase - spec is available for manual application return ctrl.Result{}, nil } @@ -955,6 +972,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) createDGD(ctx context.Context, if err != nil { return ctrl.Result{}, fmt.Errorf("failed to unmarshal generated deployment from annotation: %w", err) } + applyDGDRRuntimeVersionOverride(dgdr, generatedDGD) // Determine DGD name and namespace from generated deployment dgdName := generatedDGD.Name @@ -2052,6 +2070,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) generateDGDSpec(ctx context.Con if err != nil { return nil, "", fmt.Errorf("failed to extract DGD from %s: %w", outputFile, err) } + applyDGDRRuntimeVersionOverride(dgdr, dgd) // Override the profiler-generated name with a DGDR-scoped unique name. // The profiler emits a static topology-derived name (e.g. "vllm-agg") which @@ -2105,6 +2124,25 @@ func (r *DynamoGraphDeploymentRequestReconciler) generateDGDSpec(ctx context.Con return profilingResults, dgd.Name, nil } +// applyDGDRRuntimeVersionOverride fills missing component overrides without replacing existing values. +func applyDGDRRuntimeVersionOverride( + dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) bool { + if dgdr.Spec.RuntimeVersionOverride == "" { + return false + } + + changed := false + for i := range dgd.Spec.Components { + if dgd.Spec.Components[i].RuntimeVersionOverride == "" { + dgd.Spec.Components[i].RuntimeVersionOverride = dgdr.Spec.RuntimeVersionOverride + changed = true + } + } + return changed +} + // encodeBetaDGDManifest returns JSON/YAML manifest bytes for a beta DGD. // The Kubernetes versioning encoder temporarily supplies apiVersion/kind from // the scheme during serialization and restores the typed object's TypeMeta after. diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go index 93fc753a6a48..02bcf4bd802b 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go @@ -109,7 +109,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", AutoApply: ptr.To(true), Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), @@ -162,7 +162,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -247,7 +247,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -362,7 +362,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -454,7 +454,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -533,7 +533,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "trtllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", SearchStrategy: "rapid", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), @@ -597,7 +597,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ ObjectMeta: metav1.ObjectMeta{Name: "test-dgdr-fast-output", Namespace: envtestNamespace}, Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ - Model: "test-model", Backend: "vllm", Image: "test-profiler:latest", + Model: "test-model", Backend: "vllm", Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -647,7 +647,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -720,7 +720,10 @@ metadata: spec: services: Frontend: - replicas: 1` + replicas: 1 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0` outputConfigMapName := getOutputConfigMapName(dgdr) cm := &corev1.ConfigMap{ @@ -879,9 +882,10 @@ spec: Namespace: namespace, }, Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ - Model: "test-model", - Backend: "vllm", - Image: "test-profiler:latest", + Model: "test-model", + Backend: "vllm", + Image: "test-profiler:custom", + RuntimeVersionOverride: "1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -948,7 +952,16 @@ metadata: spec: services: Frontend: - replicas: 1` + replicas: 1 + extraPodSpec: + mainContainer: + image: registry.example/runtime:custom + Worker: + replicas: 1 + runtimeVersionOverride: 1.2.0 + extraPodSpec: + mainContainer: + image: registry.example/runtime:other-custom` // expectedDGDName is the name the operator should assign: DGDR name + "-dgd", // not the static "vllm-agg" that the profiler emitted. @@ -977,6 +990,10 @@ spec: var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed()) Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseDeploying)) + Expect(updated.Annotations[AnnotationGeneratedDGDSpec]).Should(ContainSubstring("runtimeVersionOverride: 1.1.0")) + Expect(updated.Annotations[AnnotationGeneratedDGDSpec]).Should(ContainSubstring("runtimeVersionOverride: 1.2.0")) + Expect(string(updated.Status.ProfilingResults.SelectedConfig.Raw)).Should(ContainSubstring(`"runtimeVersionOverride":"1.1.0"`)) + Expect(string(updated.Status.ProfilingResults.SelectedConfig.Raw)).Should(ContainSubstring(`"runtimeVersionOverride":"1.2.0"`)) // Reconcile again to create DGD _, err = reconciler.Reconcile(ctx, reconcile.Request{ @@ -987,8 +1004,15 @@ spec: // Verify beta DGD was created with the DGDR-scoped name (not the profiler's "vllm-agg") dgd := &nvidiacomv1beta1.DynamoGraphDeployment{} Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expectedDGDName, Namespace: namespace}, dgd)).Should(Succeed()) - Expect(dgd.Spec.Components).Should(HaveLen(1)) - Expect(dgd.Spec.Components[0].ComponentName).Should(Equal("Frontend")) + Expect(dgd.Spec.Components).Should(HaveLen(2)) + componentOverrides := map[string]string{} + for _, component := range dgd.Spec.Components { + componentOverrides[component.ComponentName] = component.RuntimeVersionOverride + } + Expect(componentOverrides).Should(Equal(map[string]string{ + "Frontend": "1.1.0", + "Worker": "1.2.0", + })) // Get final DGDR status Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed()) @@ -999,6 +1023,44 @@ spec: _ = k8sClient.Delete(ctx, dgd) }) + It("Should apply the DGDR runtime version to a persisted legacy profiler result", func() { + ctx := context.Background() + dgdName := "legacy-profiler-result-dgd" + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-profiler-result", + Namespace: envtestNamespace, + Annotations: map[string]string{ + AnnotationGeneratedDGDSpec: `apiVersion: nvidia.com/v1alpha1 +kind: DynamoGraphDeployment +metadata: + name: legacy-profiler-result-dgd +spec: + services: + worker: + extraPodSpec: + mainContainer: + image: registry.example/runtime:custom`, + }, + }, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + RuntimeVersionOverride: "1.2.3", + }, + Status: nvidiacomv1beta1.DynamoGraphDeploymentRequestStatus{ + DGDName: dgdName, + }, + } + + _, err := reconciler.createDGD(ctx, dgdr) + Expect(err).NotTo(HaveOccurred()) + + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdName, Namespace: envtestNamespace}, dgd)).Should(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, dgd) }() + Expect(dgd.Spec.Components).Should(HaveLen(1)) + Expect(dgd.Spec.Components[0].RuntimeVersionOverride).Should(Equal("1.2.3")) + }) + It("Should create additional ConfigMaps without DGDR ownership and adopt them after DGD creation", func() { ctx := context.Background() dgdrName := "test-dgdr-additional-cm-owner" @@ -1014,7 +1076,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -1082,7 +1144,10 @@ metadata: spec: services: Frontend: - replicas: 1` + replicas: 1 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0` outputConfigMapName := getOutputConfigMapName(dgdr) outputCM := &corev1.ConfigMap{ @@ -1176,7 +1241,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", }, } Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) @@ -1196,6 +1261,9 @@ spec: ComponentName: "worker", ComponentType: nvidiacomv1beta1.ComponentTypeWorker, Replicas: ptr.To[int32](1), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }}, }, } @@ -1242,7 +1310,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", }, } Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) @@ -1259,6 +1327,9 @@ spec: ComponentName: "worker", ComponentType: nvidiacomv1beta1.ComponentTypeWorker, Replicas: ptr.To[int32](1), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }}, }, } @@ -1300,11 +1371,11 @@ spec: }) }) - Context("When enforcing spec immutability", func() { - It("Should reject spec changes after profiling starts", func() { + Context("When enabling autoApply after manual review", func() { + It("Should apply a newly added runtime version override only to the created DGD", func() { t := GinkgoT() ctx := context.Background() - dgdrName := "test-dgdr-immutable" + dgdrName := "test-dgdr-ready-auto-apply" namespace := envtestNamespace dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ @@ -1313,9 +1384,10 @@ spec: Namespace: namespace, }, Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ - Model: "test-model", - Backend: "vllm", - Image: "test-profiler:latest", + Model: "test-model", + Backend: "vllm", + Image: "test-profiler:custom", + AutoApply: ptr.To(false), Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -1329,50 +1401,148 @@ spec: }, } - t.Log("Create and reconcile the initial request") - Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) + t.Log("Seed the legacy request without reconciling it") + Expect(admissionBypassClient.Create(ctx, dgdr)).Should(Succeed()) defer func() { _ = k8sClient.Delete(ctx, dgdr) }() - _, err := reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace}, - }) - Expect(err).NotTo(HaveOccurred()) - - t.Log("Read the initialized generation") var current nvidiacomv1beta1.DynamoGraphDeploymentRequest Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, ¤t)).Should(Succeed()) initialGeneration := current.Generation - observedGeneration := current.Status.ObservedGeneration - t.Log("Move the request into the profiling phase") - current.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling + t.Log("Store legacy generated manifests and move the request into the ready phase") + generatedDGD := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: dgdrName + "-generated", + Namespace: namespace, + }, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + BackendFramework: "vllm", + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "worker", + ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + Replicas: ptr.To[int32](1), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:custom"}}, + }}, + }, + { + ComponentName: "frontend", + ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, + RuntimeVersionOverride: "1.2.0", + Replicas: ptr.To[int32](1), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:other-custom"}}, + }}, + }, + }, + }, + } + dgdJSON, dgdYAML, err := reconciler.encodeBetaDGDManifest(generatedDGD) + Expect(err).NotTo(HaveOccurred()) + current.Annotations = map[string]string{AnnotationGeneratedDGDSpec: string(dgdYAML)} + Expect(k8sClient.Update(ctx, ¤t)).Should(Succeed()) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, ¤t)).Should(Succeed()) + current.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + current.Status.DGDName = generatedDGD.Name + current.Status.ObservedGeneration = current.Generation + lastTransitionTime := metav1.Now() + current.Status.Conditions = []metav1.Condition{ + { + Type: nvidiacomv1beta1.ConditionTypeSpecGenerated, + Status: metav1.ConditionTrue, + ObservedGeneration: current.Generation, + LastTransitionTime: lastTransitionTime, + Reason: "SpecGenerated", + Message: "Spec is available", + }, + { + Type: nvidiacomv1beta1.ConditionTypeSucceeded, + Status: metav1.ConditionTrue, + ObservedGeneration: current.Generation, + LastTransitionTime: lastTransitionTime, + Reason: "SpecGenerated", + Message: "Profiling complete, spec available", + }, + } + current.Status.ProfilingResults = &nvidiacomv1beta1.ProfilingResultsStatus{ + SelectedConfig: &runtime.RawExtension{Raw: dgdJSON}, + } Expect(k8sClient.Status().Update(ctx, ¤t)).Should(Succeed()) - t.Log("Seed a spec change that validating admission normally rejects") + t.Log("Set the missing runtime version while autoApply remains disabled") Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, ¤t)).Should(Succeed()) - current.Spec.Model = "modified-model" - Expect(admissionBypassClient.Update(ctx, ¤t)).Should(Succeed()) + current.Spec.RuntimeVersionOverride = "1.0.0" + Expect(k8sClient.Update(ctx, ¤t)).Should(Succeed()) - t.Log("Reconcile the legacy invalid state") + t.Log("Change the deferred runtime version before deployment") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, ¤t)).Should(Succeed()) + current.Spec.RuntimeVersionOverride = "1.1.0" + Expect(k8sClient.Update(ctx, ¤t)).Should(Succeed()) + + t.Log("Enable autoApply after selecting the runtime version") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, ¤t)).Should(Succeed()) + current.Spec.AutoApply = ptr.To(true) + Expect(k8sClient.Update(ctx, ¤t)).Should(Succeed()) + + t.Log("Transition the reviewed request to deploying") _, err = reconciler.Reconcile(ctx, reconcile.Request{ NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace}, }) Expect(err).NotTo(HaveOccurred()) - t.Log("Verify that reconciliation preserves the previously observed state") + t.Log("Verify that the generated snapshots remain immutable") Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, ¤t)).Should(Succeed()) Expect(current.Generation).Should(BeNumerically(">", initialGeneration)) - Expect(current.Status.ObservedGeneration).Should(Equal(observedGeneration)) - Expect(current.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseProfiling)) + Expect(current.Status.ObservedGeneration).Should(Equal(current.Generation)) + Expect(current.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseDeploying)) + selectedDGD, err := reconciler.extractDGDFromYAML(current.Status.ProfilingResults.SelectedConfig.Raw) + Expect(err).NotTo(HaveOccurred()) + Expect(selectedDGD.Spec.Components).Should(HaveLen(2)) + Expect(selectedDGD.Spec.Components[0].RuntimeVersionOverride).Should(BeEmpty()) + Expect(selectedDGD.Spec.Components[1].RuntimeVersionOverride).Should(Equal("1.2.0")) + annotatedDGD, err := reconciler.extractDGDFromYAML([]byte(current.Annotations[AnnotationGeneratedDGDSpec])) + Expect(err).NotTo(HaveOccurred()) + Expect(annotatedDGD.Spec.Components).Should(HaveLen(2)) + Expect(annotatedDGD.Spec.Components[0].RuntimeVersionOverride).Should(BeEmpty()) + Expect(annotatedDGD.Spec.Components[1].RuntimeVersionOverride).Should(Equal("1.2.0")) - t.Log("Verify that reconciliation reports the rejected change") - Eventually(func() bool { - select { - case event := <-recorder.Events: - return strings.Contains(event, "DynamoGraphDeploymentRequest is immutable once profiling starts") - default: - return false - } - }, timeout, interval).Should(BeTrue()) + t.Log("Create the DGD with the runtime version applied lazily") + _, err = reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace}, + }) + Expect(err).NotTo(HaveOccurred()) + var createdDGD nvidiacomv1beta1.DynamoGraphDeployment + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: generatedDGD.Name, Namespace: namespace}, &createdDGD)).Should(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, &createdDGD) }() + Expect(createdDGD.Spec.Components).Should(HaveLen(2)) + Expect(createdDGD.Spec.Components[0].RuntimeVersionOverride).Should(Equal("1.1.0")) + Expect(createdDGD.Spec.Components[1].RuntimeVersionOverride).Should(Equal("1.2.0")) + }) + + It("Should fill only missing DGD runtime version overrides", func() { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + RuntimeVersionOverride: "1.1.0", + }, + } + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "worker", + RuntimeVersionOverride: "1.2.0", + }, + { + ComponentName: "frontend", + }, + }, + }, + } + + changed := applyDGDRRuntimeVersionOverride(dgdr, dgd) + Expect(changed).Should(BeTrue()) + Expect(dgd.Spec.Components[0].RuntimeVersionOverride).Should(Equal("1.2.0")) + Expect(dgd.Spec.Components[1].RuntimeVersionOverride).Should(Equal("1.1.0")) }) }) @@ -1390,7 +1560,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -1509,7 +1679,7 @@ var _ = Describe("DGDR Validation", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -1533,7 +1703,7 @@ var _ = Describe("DGDR Validation", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -1595,7 +1765,7 @@ var _ = Describe("DGDR Profiler Arguments", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "trtllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ GPUSKU: nvidiacomv1beta1.GPUSKUTypeH200SXM, NumGPUsPerNode: ptr.To[int32](8), @@ -1659,7 +1829,7 @@ var _ = Describe("DGDR Profiler Arguments", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "trtllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", SearchStrategy: "rapid", Hardware: &nvidiacomv1beta1.HardwareSpec{ GPUSKU: nvidiacomv1beta1.GPUSKUTypeH200SXM, @@ -1724,7 +1894,7 @@ var _ = Describe("DGDR Profiler Arguments", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "trtllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -1806,7 +1976,7 @@ var _ = Describe("DGDR Error Handling", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -2159,7 +2329,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", SLA: &nvidiacomv1beta1.SLASpec{ TTFT: ptr.To(100.0), ITL: ptr.To(1500.0), @@ -2225,7 +2395,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](4), GPUSKU: nvidiacomv1beta1.GPUSKUTypeA100SXM, @@ -2285,7 +2455,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", SLA: &nvidiacomv1beta1.SLASpec{ TTFT: ptr.To(100.0), ITL: ptr.To(1500.0), @@ -2338,7 +2508,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, VRAMMB: ptr.To(81920.0), @@ -2412,7 +2582,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", SLA: &nvidiacomv1beta1.SLASpec{ TTFT: ptr.To(100.0), ITL: ptr.To(1500.0), @@ -2472,7 +2642,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -2510,6 +2680,9 @@ spec: ComponentName: "worker", ComponentType: nvidiacomv1beta1.ComponentTypeWorker, Replicas: ptr.To[int32](1), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }}, }, } @@ -2558,7 +2731,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -2603,7 +2776,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -2661,7 +2834,10 @@ metadata: spec: services: Frontend: - replicas: 1` + replicas: 1 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0` cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -2700,7 +2876,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", AutoApply: ptr.To(true), Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), @@ -2756,7 +2932,10 @@ metadata: spec: services: Frontend: - replicas: 1` + replicas: 1 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0` // expectedDGDName is derived from the DGDR name, not from the profiler's output. expectedDGDName := dgdrName + "-dgd" @@ -2800,7 +2979,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -2863,7 +3042,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", AutoApply: ptr.To(false), Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), @@ -2924,8 +3103,14 @@ spec: services: Frontend: replicas: 1 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0 VllmWorker: - replicas: 2` + replicas: 2 + extraPodSpec: + mainContainer: + image: registry.example/runtime:1.1.0` cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -2993,7 +3178,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ GPUSKU: nvidiacomv1beta1.GPUSKUTypeA100SXM, }, @@ -3031,7 +3216,7 @@ spec: Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ GPUSKU: nvidiacomv1beta1.GPUSKUTypeA100SXM, }, @@ -3175,7 +3360,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3263,7 +3448,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3345,7 +3530,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3412,7 +3597,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3477,7 +3662,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3521,7 +3706,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3579,7 +3764,7 @@ var _ = Describe("DGDR Profiling Failure Attribution", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: "h100_sxm", @@ -3764,7 +3949,7 @@ var _ = Describe("DGDR Image Pull Error Detection", func() { Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "test-model", Backend: "vllm", - Image: "test-profiler:latest", + Image: "test-profiler:1.1.0", Hardware: &nvidiacomv1beta1.HardwareSpec{ NumGPUsPerNode: ptr.To[int32](8), GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, @@ -3801,6 +3986,9 @@ var _ = Describe("DGDR Image Pull Error Detection", func() { ComponentName: "worker", ComponentType: nvidiacomv1beta1.ComponentTypeWorker, Replicas: ptr.To[int32](1), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }}, }, } diff --git a/deploy/operator/internal/controller/testdata/dgd/components/input.yaml b/deploy/operator/internal/controller/testdata/dgd/components/input.yaml index 27f52e81d35b..911509c62425 100644 --- a/deploy/operator/internal/controller/testdata/dgd/components/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgd/components/input.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: main - image: registry.example/dynamo-frontend:test + image: registry.example/dynamo-frontend:1.4.0-test - name: decode type: decode replicas: 2 @@ -25,4 +25,4 @@ spec: spec: containers: - name: main - image: registry.example/dynamo-worker:test + image: registry.example/dynamo-worker:1.4.0-test diff --git a/deploy/operator/internal/controller/testdata/dgd/components/output.yaml b/deploy/operator/internal/controller/testdata/dgd/components/output.yaml index 02e641ecbb61..07c7120119e8 100644 --- a/deploy/operator/internal/controller/testdata/dgd/components/output.yaml +++ b/deploy/operator/internal/controller/testdata/dgd/components/output.yaml @@ -17,7 +17,7 @@ spec: nvidia.com/dynamo-worker-hash: $pattern:[0-9a-f]{8} spec: containers: - - image: registry.example/dynamo-worker:test + - image: registry.example/dynamo-worker:1.4.0-test name: main replicas: 2 type: decode @@ -38,7 +38,7 @@ spec: nvidia.com/dynamo-operator-origin-version: $pattern:1\.4\..* spec: containers: - - image: registry.example/dynamo-frontend:test + - image: registry.example/dynamo-frontend:1.4.0-test name: main replicas: 1 type: frontend @@ -124,7 +124,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.uid - image: registry.example/dynamo-worker:test + image: registry.example/dynamo-worker:1.4.0-test imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 1 @@ -250,7 +250,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.uid - image: registry.example/dynamo-frontend:test + image: registry.example/dynamo-frontend:1.4.0-test imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 diff --git a/deploy/operator/internal/controller/testdata/dgd/grove/input.yaml b/deploy/operator/internal/controller/testdata/dgd/grove/input.yaml index 9b9b4afc8ed9..bfc42e482f93 100644 --- a/deploy/operator/internal/controller/testdata/dgd/grove/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgd/grove/input.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: main - image: registry.example/dynamo-frontend:test + image: registry.example/dynamo-frontend:1.4.0-test - name: decode type: decode multinode: @@ -23,4 +23,4 @@ spec: spec: containers: - name: main - image: registry.example/dynamo-worker:test + image: registry.example/dynamo-worker:1.4.0-test diff --git a/deploy/operator/internal/controller/testdata/dgd/grove/output.yaml b/deploy/operator/internal/controller/testdata/dgd/grove/output.yaml index 6239cf427ef5..c62ac3487a8f 100644 --- a/deploy/operator/internal/controller/testdata/dgd/grove/output.yaml +++ b/deploy/operator/internal/controller/testdata/dgd/grove/output.yaml @@ -62,7 +62,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid - image: registry.example/dynamo-frontend:test + image: registry.example/dynamo-frontend:1.4.0-test livenessProbe: failureThreshold: 3 httpGet: @@ -164,7 +164,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: registry.example/dynamo-worker:test + image: registry.example/dynamo-worker:1.4.0-test livenessProbe: failureThreshold: 1 httpGet: @@ -274,7 +274,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: registry.example/dynamo-worker:test + image: registry.example/dynamo-worker:1.4.0-test name: main ports: - containerPort: 9090 diff --git a/deploy/operator/internal/controller/testdata/dgdr/deployment/input.yaml b/deploy/operator/internal/controller/testdata/dgdr/deployment/input.yaml index 90d179b935c6..7f5b6b070459 100644 --- a/deploy/operator/internal/controller/testdata/dgdr/deployment/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr/deployment/input.yaml @@ -8,7 +8,7 @@ metadata: spec: model: Qwen/Qwen3-0.6B backend: vllm - image: test.invalid/profiler:replaced-by-test + image: test.invalid/profiler:1.4.0-test searchStrategy: rapid autoApply: true hardware: diff --git a/deploy/operator/internal/controller/testdata/dgdr/deployment/output.yaml b/deploy/operator/internal/controller/testdata/dgdr/deployment/output.yaml index 2dac816205f1..199943aeb8ad 100644 --- a/deploy/operator/internal/controller/testdata/dgdr/deployment/output.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr/deployment/output.yaml @@ -15,6 +15,7 @@ spec: - image: $ignore name: main replicas: 1 + runtimeVersionOverride: 1.4.0 type: frontend - name: decode podTemplate: @@ -42,6 +43,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: decode - name: prefill podTemplate: @@ -69,6 +71,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: prefill status: $ignore: true @@ -110,6 +113,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: decode status: $ignore: true @@ -554,6 +558,7 @@ spec: - image: $ignore name: main replicas: 1 + runtimeVersionOverride: 1.4.0 type: frontend status: $ignore: true @@ -595,6 +600,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: prefill status: $ignore: true diff --git a/deploy/operator/internal/controller/testdata/dgdr/grove/input.yaml b/deploy/operator/internal/controller/testdata/dgdr/grove/input.yaml index ae4765981f9d..2c50c4c1a8bc 100644 --- a/deploy/operator/internal/controller/testdata/dgdr/grove/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr/grove/input.yaml @@ -8,7 +8,7 @@ metadata: spec: model: Qwen/Qwen3-0.6B backend: vllm - image: test.invalid/profiler:replaced-by-test + image: test.invalid/profiler:1.4.0-test searchStrategy: rapid autoApply: true hardware: diff --git a/deploy/operator/internal/controller/testdata/dgdr/grove/output.yaml b/deploy/operator/internal/controller/testdata/dgdr/grove/output.yaml index 2df3c873b4f1..668ee10ed54e 100644 --- a/deploy/operator/internal/controller/testdata/dgdr/grove/output.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr/grove/output.yaml @@ -16,6 +16,7 @@ spec: - image: $ignore name: main replicas: 1 + runtimeVersionOverride: 1.4.0 type: frontend - minAvailable: 1 name: decode @@ -44,6 +45,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: decode - minAvailable: 1 name: prefill @@ -72,6 +74,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: prefill status: $ignore: true diff --git a/deploy/operator/internal/controller/testdata/dgdr/lws/input.yaml b/deploy/operator/internal/controller/testdata/dgdr/lws/input.yaml index 3eb700631d4d..5326276f96bd 100644 --- a/deploy/operator/internal/controller/testdata/dgdr/lws/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr/lws/input.yaml @@ -8,7 +8,7 @@ metadata: spec: model: Qwen/Qwen3-0.6B backend: vllm - image: test.invalid/profiler:replaced-by-test + image: test.invalid/profiler:1.4.0-test searchStrategy: rapid autoApply: true hardware: diff --git a/deploy/operator/internal/controller/testdata/dgdr/lws/output.yaml b/deploy/operator/internal/controller/testdata/dgdr/lws/output.yaml index 72cf87ff3f80..fd97b1e805fb 100644 --- a/deploy/operator/internal/controller/testdata/dgdr/lws/output.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr/lws/output.yaml @@ -15,6 +15,7 @@ spec: - image: $ignore name: main replicas: 1 + runtimeVersionOverride: 1.4.0 type: frontend - multinode: nodeCount: 2 @@ -44,6 +45,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: decode - name: prefill podTemplate: @@ -71,6 +73,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: prefill status: $ignore: true @@ -114,6 +117,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: decode status: $ignore: true @@ -662,6 +666,7 @@ spec: - image: $ignore name: main replicas: 1 + runtimeVersionOverride: 1.4.0 type: frontend status: $ignore: true @@ -703,6 +708,7 @@ spec: name: main workingDir: /workspace replicas: 1 + runtimeVersionOverride: 1.4.0 type: prefill status: $ignore: true diff --git a/deploy/operator/internal/dynamo/hash.go b/deploy/operator/internal/dynamo/hash.go index 126f9b625563..445e71558b2c 100644 --- a/deploy/operator/internal/dynamo/hash.go +++ b/deploy/operator/internal/dynamo/hash.go @@ -101,5 +101,8 @@ func workerHashSpec(dcd *v1beta1.DynamoComponentDeployment) v1beta1.DynamoCompon spec.MinAvailable = nil spec.ScalingAdapter = nil + // RuntimeVersionOverride has no rendered Pod effect yet. + spec.RuntimeVersionOverride = "" + return *spec } diff --git a/deploy/operator/internal/dynamo/hash_test.go b/deploy/operator/internal/dynamo/hash_test.go index db835638885c..ef4fed320e71 100644 --- a/deploy/operator/internal/dynamo/hash_test.go +++ b/deploy/operator/internal/dynamo/hash_test.go @@ -296,6 +296,18 @@ func TestComputeBetaDGDWorkersSpecHash_IgnoresNonRolloutFields(t *testing.T) { assert.Equal(t, baseHash, mustComputeBetaDGDWorkersSpecHash(t, betaDGD(t, disabledScalingAdapter))) } +func TestComputeBetaDGDWorkersSpecHash_IgnoresRuntimeVersionOverride(t *testing.T) { + base := betaDGD(t, baseDGD(map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{ + "worker": {ComponentType: commonconsts.ComponentTypeWorker}, + })) + baseHash := mustComputeBetaDGDWorkersSpecHash(t, base) + + withOverride := base.DeepCopy() + withOverride.Spec.Components[0].RuntimeVersionOverride = "1.4.0" + + assert.Equal(t, baseHash, mustComputeBetaDGDWorkersSpecHash(t, withOverride)) +} + func TestComputeBetaDGDWorkersSpecHash_TracksPreservedAlphaResourceMetadata(t *testing.T) { base := func() *v1alpha1.DynamoGraphDeployment { return baseDGD(map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{ diff --git a/deploy/operator/internal/runtimeversion/runtimeversion.go b/deploy/operator/internal/runtimeversion/runtimeversion.go new file mode 100644 index 000000000000..a31ea7175de0 --- /dev/null +++ b/deploy/operator/internal/runtimeversion/runtimeversion.go @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package runtimeversion + +import ( + "fmt" + "regexp" + "strings" + + semver "github.com/Masterminds/semver/v3" +) + +var imageTagPattern = regexp.MustCompile(`^[vV]?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-((?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$`) + +// Version identifies a runtime compatibility version by semver core. +type Version struct { + Major uint64 + Minor uint64 + Patch uint64 +} + +func (v Version) String() string { + return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) +} + +// Parse returns the compatibility version represented by an explicit override. +func Parse(value string) (Version, error) { + version, err := semver.StrictNewVersion(value) + if err != nil { + return Version{}, fmt.Errorf("must be a semantic version such as \"1.1.0\"") + } + return fromSemver(version), nil +} + +// ParseImageVersion returns a normalized compatibility version from an image tag. +func ParseImageVersion(image string) (Version, error) { + tag := imageTag(image) + if tag == "" { + return Version{}, fmt.Errorf("image %q does not contain a tag", image) + } + trimmed := strings.TrimSpace(tag) + if !imageTagPattern.MatchString(trimmed) { + return Version{}, fmt.Errorf("image tag %q must contain a semantic version such as \"1.1.0\"", tag) + } + version, err := semver.StrictNewVersion(strings.TrimPrefix(strings.TrimPrefix(trimmed, "v"), "V")) + if err != nil { + return Version{}, fmt.Errorf("image tag %q must contain a semantic version such as \"1.1.0\"", tag) + } + return fromSemver(version), nil +} + +func fromSemver(version *semver.Version) Version { + return Version{Major: version.Major(), Minor: version.Minor(), Patch: version.Patch()} +} + +func imageTag(image string) string { + ref := strings.TrimSpace(image) + if digest := strings.Index(ref, "@"); digest >= 0 { + ref = ref[:digest] + } + lastSlash := strings.LastIndex(ref, "/") + lastColon := strings.LastIndex(ref, ":") + if lastColon <= lastSlash { + return "" + } + return ref[lastColon+1:] +} diff --git a/deploy/operator/internal/runtimeversion/runtimeversion_test.go b/deploy/operator/internal/runtimeversion/runtimeversion_test.go new file mode 100644 index 000000000000..5c2a72040b7f --- /dev/null +++ b/deploy/operator/internal/runtimeversion/runtimeversion_test.go @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package runtimeversion + +import "testing" + +func TestParse(t *testing.T) { + tests := []struct { + name string + value string + want Version + wantErr bool + }{ + { + name: "parses a canonical override", + value: "1.2.3", + want: Version{Major: 1, Minor: 2, Patch: 3}, + }, + { + name: "rejects an incomplete override", + value: "1.2", + wantErr: true, + }, + { + name: "rejects a uint64-overflowing override segment", + value: "18446744073709551616.0.0", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Parse(tt.value) + if (err != nil) != tt.wantErr { + t.Fatalf("Parse(%q) error = %v, wantErr %t", tt.value, err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Fatalf("Parse(%q) = %+v, want %+v", tt.value, got, tt.want) + } + }) + } +} + +func TestParseImageVersion(t *testing.T) { + tests := []struct { + name string + image string + want Version + wantErr bool + }{ + { + name: "parses a tag with a prefix and prerelease suffix", + image: "nvcr.io/nvidia/ai-dynamo/vllm-runtime:v1.2.3-cuda13", + want: Version{Major: 1, Minor: 2, Patch: 3}, + }, + { + name: "rejects an unparseable image tag", + image: "registry.example/runtime:sha-123", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseImageVersion(tt.image) + if (err != nil) != tt.wantErr { + t.Fatalf("ParseImageVersion(%q) error = %v, wantErr %t", tt.image, err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Fatalf("ParseImageVersion(%q) = %+v, want %+v", tt.image, got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index c21b7965651e..1154878441cb 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -21,6 +21,7 @@ import ( "context" "fmt" + semver "github.com/Masterminds/semver/v3" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" admissionv1 "k8s.io/api/admission/v1" @@ -99,13 +100,14 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { return nil } -// defaultImageFor returns the default image, or empty string when the operator version -// is unknown (e.g. local dev builds), in which case the user must provide spec.image explicitly. +// defaultImageFor returns the default image with a canonical semver tag, or an +// empty string when the operator version cannot be parsed. func (d *DGDRDefaulter) defaultImageFor() string { - if d.OperatorVersion == "" || d.OperatorVersion == "unknown" { + version, err := semver.NewVersion(d.OperatorVersion) + if err != nil { return "" } - return fmt.Sprintf("%s:%s", defaultImage, d.OperatorVersion) + return fmt.Sprintf("%s:%s", defaultImage, version.String()) } // RegisterWithManager registers the DGDR defaulting webhook with the manager. diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go index 52f911629b3a..7bfca339313b 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go @@ -45,6 +45,16 @@ func TestDGDRDefaulter_defaultImageFor(t *testing.T) { operatorVersion: "1.1.0-rc1", expectedImage: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.1.0-rc1", }, + { + name: "partial version is normalized", + operatorVersion: "1.2", + expectedImage: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.0", + }, + { + name: "leading v is normalized", + operatorVersion: "v1.2.3", + expectedImage: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", + }, { name: "unknown operator version cannot be defaulted", operatorVersion: "unknown", diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment.go index c3d8a9882aa6..26bb234ba315 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment.go @@ -45,9 +45,21 @@ type dynamoComponentDeploymentValidation struct { func (v *DynamoComponentDeploymentValidator) Validate( ctx context.Context, dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) (admission.Warnings, error) { + return v.validate(ctx, dcd, runtimeVersionSourceV1Beta1) +} + +func (v *DynamoComponentDeploymentValidator) validate( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, + runtimeVersionSource runtimeVersionValidationSource, ) (admission.Warnings, error) { validation := &dynamoComponentDeploymentValidation{ - sharedValidation: sharedValidation{ctx: ctx}, + sharedValidation: sharedValidation{ + ctx: ctx, + runtimeVersionSource: runtimeVersionSource, + allowMissingRuntimeVersionOverride: true, + }, } allErrs := validation.validateDynamoComponentDeployment(dcd) @@ -62,22 +74,42 @@ func (v *DynamoComponentDeploymentValidator) Validate( // ValidateUpdate performs complete validation of an updated v1beta1 DCD and // compares its state with the previous object. -// ctx, oldDCD, and newDCD must not be nil. +// ctx, oldDCD, and newDCD must not be nil. runtimeVersionSource identifies the request's source API. func (v *DynamoComponentDeploymentValidator) ValidateUpdate( ctx context.Context, oldDCD *nvidiacomv1beta1.DynamoComponentDeployment, newDCD *nvidiacomv1beta1.DynamoComponentDeployment, + runtimeVersionSource runtimeVersionValidationSource, ) (admission.Warnings, error) { validation := &dynamoComponentDeploymentValidation{ - sharedValidation: sharedValidation{ctx: ctx}, + sharedValidation: sharedValidation{ + ctx: ctx, + runtimeVersionSource: runtimeVersionSourceDisabled, + allowMissingRuntimeVersionOverride: true, + }, } allErrs := validation.validateDynamoComponentDeployment(newDCD) - alpha, err := alphaDynamoComponentDeploymentForValidation(newDCD) + newAlpha, err := alphaDynamoComponentDeploymentForValidation(newDCD) if err != nil { return nil, fmt.Errorf("cannot validate preserved v1alpha1 DynamoComponentDeployment fields: %w", err) } - allErrs = append(allErrs, validation.validateDynamoComponentDeploymentV1alpha1(alpha)...) + allErrs = append(allErrs, validation.validateDynamoComponentDeploymentV1alpha1(newAlpha)...) + + // Re-enable source-version runtime validation for the old/new ratchet. + validation.runtimeVersionSource = runtimeVersionSource + if validation.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { + oldAlpha, err := alphaDynamoComponentDeploymentForValidation(oldDCD) + if err != nil { + return nil, fmt.Errorf("cannot validate old preserved v1alpha1 DynamoComponentDeployment fields: %w", err) + } + allErrs = append(allErrs, validation.validateDynamoComponentDeploymentSharedSpecUpdateV1alpha1( + &newAlpha.Spec.DynamoComponentDeploymentSharedSpec, + &oldAlpha.Spec.DynamoComponentDeploymentSharedSpec, + field.NewPath("spec"), + )...) + } + allErrs = append(allErrs, validation.validateDynamoComponentDeploymentUpdate(newDCD, oldDCD)...) return validation.warnings, invalidDynamoComponentDeploymentError(newDCD, allErrs) } diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler.go index 76c849423209..c099225336c4 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler.go @@ -80,7 +80,7 @@ func (h *DynamoComponentDeploymentHandler) validateCreate( logger.Info("validate create", "name", deployment.Name, "namespace", deployment.Namespace) validator := NewDynamoComponentDeploymentValidator() - return validator.Validate(ctx, deployment) + return validator.validate(ctx, deployment, runtimeVersionValidationSourceForRequest(ctx, expectedGVK)) } // ValidateUpdate validates a DynamoComponentDeployment update request. @@ -118,7 +118,7 @@ func (h *DynamoComponentDeploymentHandler) validateUpdate( } validator := NewDynamoComponentDeploymentValidator() - return validator.ValidateUpdate(ctx, oldDeployment, newDeployment) + return validator.ValidateUpdate(ctx, oldDeployment, newDeployment, runtimeVersionValidationSourceForRequest(ctx, expectedGVK)) } // ValidateDelete validates a DynamoComponentDeployment delete request. diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler_test.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler_test.go index ee7e32762ff8..11415a68f6ac 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler_test.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -41,8 +42,10 @@ func TestDynamoComponentDeploymentV1Alpha1HandlerConvertsRequest(t *testing.T) { Spec: nvidiacomv1alpha1.DynamoComponentDeploymentSpec{ BackendFramework: "vllm", DynamoComponentDeploymentSharedSpec: nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ - ServiceName: "worker", - ComponentType: consts.ComponentTypeWorker, + ServiceName: "worker", + ComponentType: consts.ComponentTypeWorker, + RuntimeVersionOverride: "1.1.0", + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}}, }, }, } diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_v1alpha1.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_v1alpha1.go index f058ee4f7c94..4609bc3f3479 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_v1alpha1.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_v1alpha1.go @@ -26,7 +26,8 @@ import ( func (v *dynamoComponentDeploymentValidation) validateDynamoComponentDeploymentV1alpha1( dcd *nvidiacomv1alpha1.DynamoComponentDeployment, ) field.ErrorList { - if !hasDynamoComponentDeploymentV1alpha1CompatibilityFields(dcd) { + if !hasDynamoComponentDeploymentV1alpha1CompatibilityFields(dcd) && + !v.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { return nil } return v.validateDynamoComponentDeploymentSpecV1alpha1( diff --git a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go index 341cf01ec8fa..9a717acec318 100644 --- a/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamocomponentdeployment_validation_envtest_test.go @@ -71,8 +71,172 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { }), }, { - name: "checkpoint configuration requires operator feature gate", + name: "v1beta1 main image is required when pod template is absent on create", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.PodTemplate = nil + }), + wantWebhookErrs: []string{"spec.podTemplate.spec.containers: Required value: is required"}, + }, + { + name: "v1alpha1 main image is required when extra pod spec is absent on create", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.ExtraPodSpec = nil + }), + wantWebhookErrs: []string{"spec.extraPodSpec.mainContainer.image: Required value: is required"}, + }, + { + name: "v1beta1 main image cannot be removed by removing pod template", + oldDeployment: betaDCDForAdmission(nil), + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.PodTemplate = nil + }), + wantWebhookErrs: []string{"spec.podTemplate.spec.containers: Required value: is required"}, + }, + { + name: "v1alpha1 main image cannot be removed by removing extra pod spec", + oldDeployment: alphaDCDForAdmission(nil), + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.ExtraPodSpec = nil + }), + wantWebhookErrs: []string{"spec.extraPodSpec.mainContainer.image: Required value: is required"}, + }, + { + name: "v1beta1 main image is required", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName}}, + }} + }), + wantWebhookErrs: []string{"spec.podTemplate.spec.containers[0].image: Required value: is required"}, + }, + { + name: "v1alpha1 main image is required", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{ + MainContainer: &corev1.Container{}, + } + }), + wantWebhookErrs: []string{"spec.extraPodSpec.mainContainer.image: Required value: is required"}, + }, + { + name: "v1alpha1 custom image does not require runtime version override", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{ + MainContainer: &corev1.Container{Image: customRuntimeImage}, + } + }), + }, + { + name: "v1beta1 custom image does not require runtime version override", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + }), + }, + { + name: "v1beta1 metadata update with custom image does not require runtime version override", + oldDeployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + }), + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + dcd.Labels = map[string]string{"updated": "true"} + }), + }, + { + name: "v1alpha1 metadata update with custom image does not require runtime version override", + oldDeployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.ExtraPodSpec.MainContainer.Image = customRuntimeImage + }), + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.ExtraPodSpec.MainContainer.Image = customRuntimeImage + dcd.Labels = map[string]string{"updated": "true"} + }), + }, + { + name: "v1beta1 image change to custom does not require runtime version override", + oldDeployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + }), + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + }), + }, + { + name: "changing a v1alpha1 custom image does not require runtime version override", + oldDeployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.ExtraPodSpec.MainContainer.Image = customRuntimeImage + }), + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.ExtraPodSpec.MainContainer.Image = "registry.example/runtime:other-custom" + }), + }, + { + name: "v1alpha1 compatibility validation does not duplicate runtime version errors", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.ExtraPodSpec.MainContainer.Image = customRuntimeImage + dcd.Spec.Ingress = &nvidiacomv1alpha1.IngressSpec{Enabled: true} + }), + wantWebhookErrs: []string{ + "spec.ingress.host: Required value: is required when ingress is enabled", + }, + }, + { + name: "v1beta1 derives runtime version from a semver image tag", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "" + dcd.Spec.PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:v1.2.3-cuda12"}}, + }} + }), + }, + { + name: "runtime version override takes precedence over a semver image tag", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "1.1.0" + dcd.Spec.PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/vllm-opus:4.8.2"}}, + }} + }), + }, + { + name: "v1alpha1 accepts four-digit runtime version override segments", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "9999.9999.9999" + }), + }, + { + name: "v1alpha1 rejects runtime version override segments longer than four digits", + deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "10000.0.0" + }), + wantSchemaErr: `spec.runtimeVersionOverride: Invalid value: "10000.0.0": spec.runtimeVersionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta1 accepts four-digit runtime version override segments", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "9999.9999.9999" + }), + }, + { + name: "v1beta1 rejects runtime version override segments longer than four digits", + deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { + dcd.Spec.RuntimeVersionOverride = "10000.0.0" + }), + wantSchemaErr: `spec.runtimeVersionOverride: Invalid value: "10000.0.0": spec.runtimeVersionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { checkpointOff: true, + name: "checkpoint configuration requires operator feature gate", deployment: betaDCDForAdmission(func(dcd *nvidiacomv1beta1.DynamoComponentDeployment) { dcd.Spec.Experimental = &nvidiacomv1beta1.ExperimentalSpec{ Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true}, @@ -130,7 +294,7 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { }), wantWebhookErrs: []string{ `spec.sharedMemorySize: Invalid value: "-1Gi": must be non-negative`, - "spec.podTemplate.spec.containers: Required value: is required when frontendSidecar is set", + `spec.frontendSidecar: Invalid value: "frontend": must match a podTemplate.spec.containers name`, "spec.experimental.gpuMemoryService: Forbidden: GPU memory service is only supported for worker, prefill, or decode components", "spec.experimental.gpuMemoryService: Forbidden: GPU memory service requires podTemplate.spec.containers[main].resources.limits.nvidia.com/gpu >= 1", }, @@ -459,9 +623,12 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { name: "frontend sidecar container-name collision is rejected", deployment: alphaDCDWithSharedSpec(nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ FrontendSidecar: &nvidiacomv1alpha1.FrontendSidecarSpec{Image: "frontend:latest"}, - ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ - Containers: []corev1.Container{{Name: consts.FrontendSidecarContainerName, Image: "conflict:latest"}}, - }}, + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{ + PodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.FrontendSidecarContainerName, Image: "conflict:latest"}}, + }, + MainContainer: &corev1.Container{Name: consts.MainContainerName, Image: "main:1.1.0"}, + }, }), wantWebhookErrs: []string{`spec.frontendSidecar: Forbidden: cannot inject frontend sidecar: a container named "sidecar-frontend" already exists in extraPodSpec.containers`}, }, @@ -469,9 +636,12 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { name: "frontend sidecar with non-conflicting containers is accepted", deployment: alphaDCDWithSharedSpec(nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ FrontendSidecar: &nvidiacomv1alpha1.FrontendSidecarSpec{Image: "frontend:latest"}, - ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ - Containers: []corev1.Container{{Name: "other-sidecar", Image: "other:latest"}}, - }}, + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{ + PodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "other-sidecar", Image: "other:latest"}}, + }, + MainContainer: &corev1.Container{Name: consts.MainContainerName, Image: "main:1.1.0"}, + }, }), }, @@ -837,9 +1007,12 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { { name: "v1alpha1 sidecar without image reaches the webhook", deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { - dcd.Spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ - Containers: []corev1.Container{{Name: "metrics"}}, - }} + dcd.Spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{ + PodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "metrics"}}, + }, + MainContainer: &corev1.Container{Name: consts.MainContainerName, Image: "main:1.1.0"}, + } }), }, { @@ -855,9 +1028,12 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { { name: "v1alpha1 init container without image reaches the webhook", deployment: alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { - dcd.Spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ - InitContainers: []corev1.Container{{Name: "prepare"}}, - }} + dcd.Spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{ + PodSpec: &corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "prepare"}}, + }, + MainContainer: &corev1.Container{Name: consts.MainContainerName, Image: "main:1.1.0"}, + } }), }, { @@ -881,7 +1057,7 @@ func TestDynamoComponentDeploymentValidator_Validate(t *testing.T) { consts.KubeAnnotationVLLMDistributedExecutorBackend: "RaY", consts.KubeAnnotationDynamoKubeDiscoveryMode: "container", }}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName}}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "main:1.1.0"}}}, } }), wantPodAnnotations: map[string]string{ @@ -1054,8 +1230,12 @@ func alphaDCDForAdmission( Spec: nvidiacomv1alpha1.DynamoComponentDeploymentSpec{ BackendFramework: dcdAdmissionVLLMBackend, DynamoComponentDeploymentSharedSpec: nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ - ServiceName: "worker", - ComponentType: consts.ComponentTypeWorker, + ServiceName: "worker", + RuntimeVersionOverride: "1.1.0", + ComponentType: consts.ComponentTypeWorker, + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{ + MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, + }, }, }, } @@ -1069,7 +1249,12 @@ func alphaDCDWithSharedSpec( spec nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec, ) *nvidiacomv1alpha1.DynamoComponentDeployment { return alphaDCDForAdmission(func(dcd *nvidiacomv1alpha1.DynamoComponentDeployment) { + defaultExtraPodSpec := dcd.Spec.ExtraPodSpec dcd.Spec.DynamoComponentDeploymentSharedSpec = spec + // admission requires that the main image is set + if dcd.Spec.ExtraPodSpec == nil { + dcd.Spec.ExtraPodSpec = defaultExtraPodSpec + } }) } @@ -1085,8 +1270,12 @@ func betaDCDForAdmission( Spec: nvidiacomv1beta1.DynamoComponentDeploymentSpec{ BackendFramework: dcdAdmissionVLLMBackend, DynamoComponentDeploymentSharedSpec: nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "worker", - ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + ComponentName: "worker", + RuntimeVersionOverride: "1.1.0", + ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }, }, } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index 9aa511b107fb..eaa05efcde62 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -75,9 +75,10 @@ type dynamoGraphDeploymentSpecValidationOptions struct { func (v *DynamoGraphDeploymentValidator) Validate( ctx context.Context, deployment *nvidiacomv1beta1.DynamoGraphDeployment, + runtimeVersionSource runtimeVersionValidationSource, ) (admission.Warnings, error) { validation := &dynamoGraphDeploymentValidation{ - sharedValidation: sharedValidation{ctx: ctx, mgr: v.mgr}, + sharedValidation: sharedValidation{ctx: ctx, mgr: v.mgr, runtimeVersionSource: runtimeVersionSource}, } allErrs := validation.validateDynamoGraphDeployment(deployment) @@ -91,7 +92,7 @@ func (v *DynamoGraphDeploymentValidator) Validate( } // ValidateUpdate performs stateful validation comparing old and new v1beta1 DGD objects. -// ctx, oldDGD, and newDGD must not be nil. +// ctx, oldDGD, and newDGD must not be nil. runtimeVersionSource identifies the request's source API. // If userInfo is nil, replica changes for DGDSA-enabled components fail closed. func (v *DynamoGraphDeploymentValidator) ValidateUpdate( ctx context.Context, @@ -99,14 +100,30 @@ func (v *DynamoGraphDeploymentValidator) ValidateUpdate( newDGD *nvidiacomv1beta1.DynamoGraphDeployment, userInfo *authenticationv1.UserInfo, operatorPrincipal string, + runtimeVersionSource runtimeVersionValidationSource, ) (admission.Warnings, error) { validation := &dynamoGraphDeploymentValidation{ - sharedValidation: sharedValidation{ctx: ctx, mgr: v.mgr}, + sharedValidation: sharedValidation{ctx: ctx, mgr: v.mgr, runtimeVersionSource: runtimeVersionSource}, userInfo: userInfo, operatorPrincipal: operatorPrincipal, } allErrs := validation.validateDynamoGraphDeploymentUpdate(newDGD, oldDGD) + if validation.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { + newAlpha, err := alphaDynamoGraphDeploymentForValidation(newDGD) + if err != nil { + return nil, fmt.Errorf("cannot validate preserved v1alpha1 DynamoGraphDeployment fields: %w", err) + } + oldAlpha, err := alphaDynamoGraphDeploymentForValidation(oldDGD) + if err != nil { + return nil, fmt.Errorf("cannot validate old preserved v1alpha1 DynamoGraphDeployment fields: %w", err) + } + allErrs = append(allErrs, validation.validateDynamoGraphDeploymentSpecUpdateV1alpha1( + &newAlpha.Spec, + &oldAlpha.Spec, + field.NewPath("spec"), + )...) + } return validation.warnings, invalidDynamoGraphDeploymentError(newDGD, allErrs) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_handler.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_handler.go index f1fbc4d79ad5..c5ed5199fbb3 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_handler.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_handler.go @@ -93,7 +93,7 @@ func (h *DynamoGraphDeploymentHandler) validateCreate( // Create validator with manager for API group detection and perform validation validator := NewDynamoGraphDeploymentValidator(h.mgr) - return validator.Validate(ctx, deployment) + return validator.Validate(ctx, deployment, runtimeVersionValidationSourceForRequest(ctx, expectedGVK)) } // ValidateUpdate validates a DynamoGraphDeployment update request. @@ -132,7 +132,8 @@ func (h *DynamoGraphDeploymentHandler) validateUpdate( // Create validator with manager for API group detection and perform validation. validator := NewDynamoGraphDeploymentValidator(h.mgr) - warnings, err := validator.Validate(ctx, newDeployment) + runtimeVersionSource := runtimeVersionValidationSourceForRequest(ctx, expectedGVK) + warnings, err := validator.Validate(ctx, newDeployment, runtimeVersionSourceDisabled) if err != nil { return warnings, err } @@ -148,7 +149,14 @@ func (h *DynamoGraphDeploymentHandler) validateUpdate( } // Validate stateful rules (immutability + replicas protection) - updateWarnings, err := validator.ValidateUpdate(ctx, oldDeployment, newDeployment, userInfo, h.operatorPrincipal) + updateWarnings, err := validator.ValidateUpdate( + ctx, + oldDeployment, + newDeployment, + userInfo, + h.operatorPrincipal, + runtimeVersionSource, + ) if err != nil { username := "" if userInfo != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go index 0b9b0f1cd1ad..93b41a7b2d54 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_unit_test.go @@ -17,6 +17,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -37,7 +38,7 @@ func TestDynamoGraphDeploymentConversionFailureIsFatal(t *testing.T) { validator := newDynamoGraphDeploymentTestValidator(t) ctx := features.WithGate(context.Background(), features.Gates{Grove: true}) - _, err := validator.Validate(ctx, dgd) + _, err := validator.Validate(ctx, dgd, runtimeVersionSourceV1Beta1) if err == nil || !strings.Contains(err.Error(), "failed to reconstruct compatibility view") { t.Fatalf("Validate() error = %v, want fatal conversion error", err) } @@ -67,14 +68,22 @@ func newBetaDGDForValidation() *nvidiacomv1beta1.DynamoGraphDeployment { BackendFramework: "vllm", Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ { - ComponentName: "frontend", - ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, - Replicas: k8sptr.To(int32(1)), + ComponentName: "frontend", + ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, + RuntimeVersionOverride: "1.1.0", + Replicas: k8sptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }, { - ComponentName: "worker", - ComponentType: nvidiacomv1beta1.ComponentTypeWorker, - Replicas: k8sptr.To(int32(2)), + ComponentName: "worker", + ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + RuntimeVersionOverride: "1.1.0", + Replicas: k8sptr.To(int32(2)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }, }, }, @@ -91,8 +100,12 @@ func newAlphaDGDForCompatibilityValidation() *nvidiacomv1alpha1.DynamoGraphDeplo BackendFramework: "vllm", Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": { - ComponentType: consts.ComponentTypeWorker, - Replicas: k8sptr.To(int32(1)), + ComponentType: consts.ComponentTypeWorker, + RuntimeVersionOverride: "1.1.0", + Replicas: k8sptr.To(int32(1)), + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{ + MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, + }, }, }, }, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_v1alpha1.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_v1alpha1.go index 59fceef84454..ac6a588c683d 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_v1alpha1.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_v1alpha1.go @@ -26,7 +26,8 @@ import ( func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentV1alpha1( dgd *nvidiacomv1alpha1.DynamoGraphDeployment, ) field.ErrorList { - if !hasV1Alpha1CompatibilityFields(dgd) { + if !hasV1Alpha1CompatibilityFields(dgd) && + !v.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { return nil } return v.validateDynamoGraphDeploymentSpecV1alpha1( @@ -64,6 +65,30 @@ func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpecV1alp return allErrs } +// validateDynamoGraphDeploymentSpecUpdateV1alpha1 validates source-version runtime fields on update. +// newSpec, oldSpec, and fldPath must not be nil. +func (v *dynamoGraphDeploymentValidation) validateDynamoGraphDeploymentSpecUpdateV1alpha1( + newSpec *nvidiacomv1alpha1.DynamoGraphDeploymentSpec, + oldSpec *nvidiacomv1alpha1.DynamoGraphDeploymentSpec, + fldPath *field.Path, +) field.ErrorList { + allErrs := field.ErrorList{} + servicesPath := fldPath.Child("services") + for _, serviceName := range sortedV1Alpha1ServiceNames(newSpec.Services) { + newService := newSpec.Services[serviceName] + oldService, exists := oldSpec.Services[serviceName] + if !exists { + continue + } + allErrs = append(allErrs, v.validateDynamoComponentDeploymentSharedSpecUpdateV1alpha1( + newService, + oldService, + servicesPath.Key(serviceName), + )...) + } + return allErrs +} + // validatePVCV1alpha1 validates pvc. pvc and fldPath must not be nil. func (v *dynamoGraphDeploymentValidation) validatePVCV1alpha1( pvc *nvidiacomv1alpha1.PVC, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go index 6cc7f8e9d705..60706ea233b9 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go @@ -48,14 +48,15 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { tooLongComponentName := boundaryComponentName + "x" tests := []struct { - name string - deployment runtime.Object - oldDeployment runtime.Object - mutateRequest func(*testing.T, map[string]any) // mutates the source-version request map - withoutTopology bool // omits the default cluster topology fixture - groveDisabled bool // disables the configured Grove pathway - checkpointOff bool // disables checkpoint creation and restore - username string // supplies the admission request identity + name string + deployment runtime.Object + oldDeployment runtime.Object + mutateRequest func(*testing.T, map[string]any) // mutates the source-version request map + withoutTopology bool // omits the default cluster topology fixture + groveDisabled bool // disables the configured Grove pathway + checkpointOff bool // disables checkpoint creation and restore + seedWithoutWebhook bool // seeds oldDeployment without validating it + username string // supplies the admission request identity wantSchemaErr string wantCELErr string @@ -70,6 +71,98 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { name: "valid deployment with components", deployment: betaDGDForAdmission(nil), }, + { + name: "beta component main image is required when pod template is absent on create", + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + betaWorkerComponent(dgd).PodTemplate = nil + }), + wantWebhookErrs: []string{"spec.components[1].podTemplate.spec.containers: Required value: is required"}, + }, + { + name: "alpha service main image is required when extra pod spec is absent on create", + deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + dgd.Spec.Services[dgdAdmissionWorkerName].ExtraPodSpec = nil + }), + wantWebhookErrs: []string{"spec.services[worker].extraPodSpec.mainContainer.image: Required value: is required"}, + }, + { + name: "component custom image requires runtime version override", + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + worker.RuntimeVersionOverride = "" + worker.PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: customRuntimeImage}}, + }} + }), + wantWebhookErrs: []string{"spec.components[1].runtimeVersionOverride: Required value: is required when the specified main container image has no parseable semantic-version tag"}, + }, + { + name: "alpha component custom image uses source-version path", + deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + worker := dgd.Spec.Services["worker"] + worker.RuntimeVersionOverride = "" + worker.ExtraPodSpec.MainContainer.Image = customRuntimeImage + }), + wantWebhookErrs: []string{"spec.services[worker].runtimeVersionOverride: Required value: is required when the specified main container image has no parseable semantic-version tag"}, + }, + { + name: "unchanged legacy beta component runtime version is ratcheted on update", + seedWithoutWebhook: true, + oldDeployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + worker.RuntimeVersionOverride = "" + worker.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + }), + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + worker.RuntimeVersionOverride = "" + worker.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + dgd.Labels = map[string]string{"updated": "true"} + }), + }, + { + name: "unchanged legacy alpha service runtime version is ratcheted on update", + seedWithoutWebhook: true, + oldDeployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + worker := dgd.Spec.Services["worker"] + worker.RuntimeVersionOverride = "" + worker.ExtraPodSpec.MainContainer.Image = customRuntimeImage + }), + deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + worker := dgd.Spec.Services["worker"] + worker.RuntimeVersionOverride = "" + worker.ExtraPodSpec.MainContainer.Image = customRuntimeImage + dgd.Labels = map[string]string{"updated": "true"} + }), + }, + { + name: "beta component image change to custom requires runtime version override", + oldDeployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + worker.RuntimeVersionOverride = "" + }), + deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + worker := betaWorkerComponent(dgd) + worker.RuntimeVersionOverride = "" + worker.PodTemplate.Spec.Containers[0].Image = customRuntimeImage + }), + wantWebhookErrs: []string{"spec.components[1].runtimeVersionOverride: Required value: is required when the specified main container image has no parseable semantic-version tag"}, + }, + { + name: "changing a legacy alpha custom image requires runtime version override", + seedWithoutWebhook: true, + oldDeployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + worker := dgd.Spec.Services["worker"] + worker.RuntimeVersionOverride = "" + worker.ExtraPodSpec.MainContainer.Image = customRuntimeImage + }), + deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { + worker := dgd.Spec.Services["worker"] + worker.RuntimeVersionOverride = "" + worker.ExtraPodSpec.MainContainer.Image = "registry.example/runtime:other-custom" + }), + wantWebhookErrs: []string{"spec.services[worker].runtimeVersionOverride: Required value: is required when the specified main container image has no parseable semantic-version tag"}, + }, { name: "no components", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { @@ -244,7 +337,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { name: "v1beta1 sidecars must provide an image in CEL", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { betaWorkerComponent(dgd).PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: consts.MainContainerName}, {Name: "metrics"}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}, {Name: "metrics"}}, }} }), wantCELErr: "spec.components[1].podTemplate.spec.containers[1]: Invalid value: sidecar containers must specify a non-empty image", @@ -252,7 +345,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { { name: "v1alpha1 converted sidecar without image reaches the webhook", deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { - dgd.Spec.Services["worker"].ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ + dgd.Spec.Services["worker"].ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, PodSpec: &corev1.PodSpec{ Containers: []corev1.Container{{Name: "metrics"}}, }} }), @@ -261,7 +354,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { name: "v1alpha1 frontend sidecar without image reaches the webhook", deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["worker"].FrontendSidecar = &nvidiacomv1alpha1.FrontendSidecarSpec{} - dgd.Spec.Services["worker"].ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ + dgd.Spec.Services["worker"].ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, PodSpec: &corev1.PodSpec{ Containers: []corev1.Container{{Name: "metrics"}}, }} }), @@ -270,7 +363,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { name: "v1beta1 init containers must provide an image in CEL", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { betaWorkerComponent(dgd).PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: consts.MainContainerName}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, InitContainers: []corev1.Container{{Name: "prepare"}}, }} }), @@ -279,7 +372,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { { name: "v1alpha1 converted init container without image reaches the webhook", deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { - dgd.Spec.Services["worker"].ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ + dgd.Spec.Services["worker"].ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, PodSpec: &corev1.PodSpec{ InitContainers: []corev1.Container{{Name: "prep"}}, }} }), @@ -291,7 +384,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ consts.KubeAnnotationVLLMDistributedExecutorBackend: "invalid", }}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName}}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}}, } }), wantCELErr: "spec.components[1].podTemplate.metadata.annotations: Invalid value: podTemplate backend annotation must be mp or ray, case-insensitively", @@ -305,7 +398,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { consts.KubeAnnotationVLLMDistributedExecutorBackend: "RaY", consts.KubeAnnotationDynamoKubeDiscoveryMode: "container", }}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName}}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}}, } }), wantPodAnnotations: map[string]string{ @@ -336,7 +429,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { worker := betaWorkerComponent(dgd) worker.FrontendSidecar = k8sptr.To("missing") worker.PodTemplate = &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: consts.MainContainerName}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, }} }), wantWebhookErrs: []string{`spec.components[1].frontendSidecar: Invalid value: "missing": must match a podTemplate.spec.containers name`}, @@ -354,7 +447,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { name: "v1beta1 pod template container counts are not artificially bounded", deployment: betaDGDForAdmission(func(dgd *nvidiacomv1beta1.DynamoGraphDeployment) { podTemplate := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: consts.MainContainerName}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, }} for i := range 32 { podTemplate.Spec.Containers = append(podTemplate.Spec.Containers, corev1.Container{ @@ -752,7 +845,11 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { className := "nginx" dgd.Spec.Services["frontend"] = &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ - ComponentType: consts.ComponentTypeFrontend, + ComponentType: consts.ComponentTypeFrontend, + RuntimeVersionOverride: "1.1.0", + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{ + MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, + }, Ingress: &nvidiacomv1alpha1.IngressSpec{ Enabled: true, IngressControllerClassName: &className, @@ -802,11 +899,12 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { name: "alpha frontend sidecar rejects generated container name conflict", deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { dgd.Spec.Services["frontend"] = &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ - ComponentType: consts.ComponentTypeFrontend, + ComponentType: consts.ComponentTypeFrontend, + RuntimeVersionOverride: "1.1.0", FrontendSidecar: &nvidiacomv1alpha1.FrontendSidecarSpec{ Image: "custom/frontend:latest", }, - ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{ + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, PodSpec: &corev1.PodSpec{ Containers: []corev1.Container{{ Name: consts.FrontendSidecarContainerName, Image: "custom/frontend:latest", @@ -890,6 +988,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { deployment: alphaDGDForAdmission(func(dgd *nvidiacomv1alpha1.DynamoGraphDeployment) { service := dgd.Spec.Services["worker"] service.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{ + Image: "registry.example/runtime:1.1.0", Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{ corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), }}, @@ -1238,11 +1337,13 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { oldDeployment: newBetaDGDForValidation(), deployment: betaDGDWithSpec(func(spec *nvidiacomv1beta1.DynamoGraphDeploymentSpec) { spec.Components = append(spec.Components, nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "extra", - Replicas: k8sptr.To(int32(1)), + ComponentName: "extra", + Replicas: k8sptr.To(int32(1)), + RuntimeVersionOverride: "1.1.0", PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: consts.MainContainerName, - Env: []corev1.EnvVar{{Name: "TOKEN", Value: "do-not-leak-this-value"}}, + Name: consts.MainContainerName, + Image: "registry.example/runtime:1.1.0", + Env: []corev1.EnvVar{{Name: "TOKEN", Value: "do-not-leak-this-value"}}, }}}}, }) }), @@ -1264,8 +1365,10 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { spec.Components = []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ spec.Components[1], { - ComponentName: "extra", - Replicas: k8sptr.To(int32(1)), + ComponentName: "extra", + Replicas: k8sptr.To(int32(1)), + RuntimeVersionOverride: "1.1.0", + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}}}, }, } }), @@ -1644,18 +1747,19 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { gates := features.Gates{Checkpoint: !tt.checkpointOff, Grove: !tt.groveDisabled} test := admissionTestCase{ - object: tt.deployment, - oldObject: tt.oldDeployment, - mutateObject: tt.mutateRequest, - gates: gates, - withoutTopology: tt.withoutTopology, - username: tt.username, - wantSchemaError: tt.wantSchemaErr, - wantCELError: tt.wantCELErr, - wantAdmissionErrs: tt.wantAdmissionErrs, - wantWebhookErrors: tt.wantWebhookErrs, - wantWarnings: tt.wantWarnings, - notWantError: tt.notWantErr, + object: tt.deployment, + oldObject: tt.oldDeployment, + mutateObject: tt.mutateRequest, + gates: gates, + withoutTopology: tt.withoutTopology, + seedWithoutWebhook: tt.seedWithoutWebhook, + username: tt.username, + wantSchemaError: tt.wantSchemaErr, + wantCELError: tt.wantCELErr, + wantAdmissionErrs: tt.wantAdmissionErrs, + wantWebhookErrors: tt.wantWebhookErrs, + wantWarnings: tt.wantWarnings, + notWantError: tt.notWantErr, } if tt.oldDeployment != nil { test.oldBeforeUpdate = dgdBeforeRestart(t, tt.oldDeployment) @@ -1805,14 +1909,22 @@ func newBetaDGDForValidation() *nvidiacomv1beta1.DynamoGraphDeployment { BackendFramework: "vllm", Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ { - ComponentName: "frontend", - ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, - Replicas: k8sptr.To(int32(1)), + ComponentName: "frontend", + ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, + RuntimeVersionOverride: "1.1.0", + Replicas: k8sptr.To(int32(1)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }, { - ComponentName: "worker", - ComponentType: nvidiacomv1beta1.ComponentTypeWorker, - Replicas: k8sptr.To(int32(2)), + ComponentName: "worker", + ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + RuntimeVersionOverride: "1.1.0", + Replicas: k8sptr.To(int32(2)), + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}, + }}, }, }, }, @@ -1829,8 +1941,12 @@ func newAlphaDGDForCompatibilityValidation() *nvidiacomv1alpha1.DynamoGraphDeplo BackendFramework: "vllm", Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": { - ComponentType: consts.ComponentTypeWorker, - Replicas: k8sptr.To(int32(1)), + ComponentType: consts.ComponentTypeWorker, + RuntimeVersionOverride: "1.1.0", + Replicas: k8sptr.To(int32(1)), + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{ + MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, + }, }, }, }, @@ -1926,16 +2042,15 @@ func enableBetaInterPodGMS(component *nvidiacomv1beta1.DynamoComponentDeployment } component.PodTemplate = &corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: consts.MainContainerName, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), - }, + Containers: []corev1.Container{{ + Name: consts.MainContainerName, + Image: "registry.example/runtime:1.1.0", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), }, }, - }, + }}, }, } } @@ -1948,16 +2063,15 @@ func enableBetaIntraPodGMS(component *nvidiacomv1beta1.DynamoComponentDeployment } component.PodTemplate = &corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: consts.MainContainerName, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), - }, + Containers: []corev1.Container{{ + Name: consts.MainContainerName, + Image: "registry.example/runtime:1.1.0", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceName(consts.KubeResourceGPUNvidia): resource.MustParse("1"), }, }, - }, + }}, }, } } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index b54465914f34..e02f0e8495c1 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -23,6 +23,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -93,6 +94,13 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe )) } + if dgdrRuntimeVersionOverrideRequired(spec) { + allErrs = append(allErrs, field.Required( + fldPath.Child("runtimeVersionOverride"), + "is required when spec.image has no parseable semantic-version tag", + )) + } + return allErrs } @@ -110,7 +118,7 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe } // validateDynamoGraphDeploymentRequestSpecUpdate validates a spec update. -// newSpec, oldSpec, and fldPath must not be nil; oldPhase comes from the owning old resource status. +// newSpec, oldSpec, and fldPath must not be nil; oldPhase comes from the owning old resource. func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRequestSpecUpdate( newSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, oldSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, @@ -141,7 +149,22 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe )) } - if isImmutableDGDRPhase(oldPhase) && !apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + // Revalidate legacy image compatibility when autoApply creates a DGD. + autoApplyActivated := oldSpec.AutoApply != nil && !*oldSpec.AutoApply && + (newSpec.AutoApply == nil || *newSpec.AutoApply) + if dgdrRuntimeVersionOverrideRequired(newSpec) && + (newSpec.Image != oldSpec.Image || + newSpec.RuntimeVersionOverride != oldSpec.RuntimeVersionOverride || + autoApplyActivated) { + allErrs = append(allErrs, field.Required( + fldPath.Child("runtimeVersionOverride"), + "is required when spec.image has no parseable semantic-version tag", + )) + } + + if isImmutableDGDRPhase(oldPhase) && + !apiequality.Semantic.DeepEqual(newSpec, oldSpec) && + !isDGDRDeferredRuntimeVersionUpdate(newSpec, oldSpec, oldPhase) { allErrs = append(allErrs, field.Forbidden( fldPath, fmt.Sprintf("updates are forbidden while the resource is in phase %q; delete and recreate the resource to change its spec", oldPhase), @@ -150,3 +173,36 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe return allErrs } + +func dgdrRuntimeVersionOverrideRequired(spec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec) bool { + if spec.Image == "" || spec.RuntimeVersionOverride != "" { + return false + } + _, err := runtimeversion.ParseImageVersion(spec.Image) + return err != nil +} + +// isDGDRDeferredRuntimeVersionUpdate permits choosing the runtime version before deferred DGD creation. +func isDGDRDeferredRuntimeVersionUpdate( + newSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, + oldSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, + oldPhase nvidiacomv1beta1.DGDRPhase, +) bool { + if oldSpec.AutoApply == nil || *oldSpec.AutoApply || + (oldPhase != nvidiacomv1beta1.DGDRPhaseProfiling && + oldPhase != nvidiacomv1beta1.DGDRPhaseReady) { + return false + } + + // autoApply may only be activated once the generated snapshot is Ready. + autoApplyActivated := newSpec.AutoApply == nil || *newSpec.AutoApply + if autoApplyActivated && oldPhase != nvidiacomv1beta1.DGDRPhaseReady { + return false + } + + // Require every other field to remain unchanged. + newWithoutDeferredFields := newSpec.DeepCopy() + newWithoutDeferredFields.AutoApply = oldSpec.AutoApply + newWithoutDeferredFields.RuntimeVersionOverride = oldSpec.RuntimeVersionOverride + return apiequality.Semantic.DeepEqual(newWithoutDeferredFields, oldSpec) +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_helpers.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_helpers.go index 1e80974f99ff..ba82cd86712a 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_helpers.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_helpers.go @@ -32,6 +32,7 @@ func isImmutableDGDRPhase(phase nvidiacomv1beta1.DGDRPhase) bool { switch phase { case nvidiacomv1beta1.DGDRPhaseProfiling, nvidiacomv1beta1.DGDRPhaseDeploying, + nvidiacomv1beta1.DGDRPhaseReady, nvidiacomv1beta1.DGDRPhaseDeployed: return true default: diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 61387b6532f0..b8c88f0e45ee 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" ) const alternateAdmissionModel = "Qwen/Qwen3-8B" @@ -49,10 +50,29 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "valid v1beta1 request is defaulted", request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { request.Spec.Image = "" + request.Spec.RuntimeVersionOverride = "" }), gpuDiscovery: true, wantImage: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.1.0", }, + { + name: "custom image requires runtime version override", + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "" + }), + gpuDiscovery: true, + wantWebhook: []string{ + "spec.runtimeVersionOverride: Required value: is required when spec.image has no parseable semantic-version tag", + }, + }, + { + name: "runtime version override must be canonical semver core", + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "1.2" + }), + gpuDiscovery: true, + wantSchemaErr: `spec.runtimeVersionOverride: Invalid value: "1.2": spec.runtimeVersionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, { name: "valid v1alpha1 request converts through the production path", request: alphaDGDRForAdmission(nil), @@ -155,6 +175,106 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { `spec: Forbidden: updates are forbidden while the resource is in phase "Profiling"; delete and recreate the resource to change its spec`, }, }, + { + name: "auto apply can be enabled after reviewing a ready request", + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(true) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + gpuDiscovery: true, + }, + { + name: "auto apply activation can add a missing runtime version override", + seedWithoutWebhook: true, + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(true) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + gpuDiscovery: true, + }, + { + name: "auto apply activation requires a missing runtime version override", + seedWithoutWebhook: true, + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(true) + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + gpuDiscovery: true, + wantWebhook: []string{ + "spec.runtimeVersionOverride: Required value: is required when spec.image has no parseable semantic-version tag", + }, + }, + { + name: "runtime version override can be added while ready with auto apply disabled", + seedWithoutWebhook: true, + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + gpuDiscovery: true, + }, + { + name: "runtime version override can change while ready with auto apply disabled", + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Spec.RuntimeVersionOverride = "1.2.0" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + gpuDiscovery: true, + }, + { + name: "runtime version override can change while profiling with auto apply disabled", + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Spec.RuntimeVersionOverride = "1.2.0" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling + }), + gpuDiscovery: true, + }, + { + name: "other spec updates remain forbidden while ready with auto apply disabled", + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.AutoApply = ptr.To(false) + request.Spec.Model = alternateAdmissionModel + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseReady + }), + gpuDiscovery: true, + wantWebhook: []string{ + `spec: Forbidden: updates are forbidden while the resource is in phase "Ready"; delete and recreate the resource to change its spec`, + }, + }, { name: "spec update is rejected during deploying", oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { @@ -205,6 +325,65 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }), gpuDiscovery: true, }, + { + name: "unchanged legacy custom image without override is ratcheted on update", + seedWithoutWebhook: true, + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseDeployed + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseDeployed + request.Labels = map[string]string{"updated": "true"} + }), + gpuDiscovery: true, + }, + { + name: "adding runtime version override is rejected after deployment", + seedWithoutWebhook: true, + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "" + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseDeployed + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Status.Phase = nvidiacomv1beta1.DGDRPhaseDeployed + request.Labels = map[string]string{"updated": "true"} + }), + gpuDiscovery: true, + wantWebhook: []string{ + `spec: Forbidden: updates are forbidden while the resource is in phase "Deployed"; delete and recreate the resource to change its spec`, + }, + }, + { + name: "newly introduced custom image without override is rejected on update", + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.Image = "test-profiler:1.1.0" + request.Spec.RuntimeVersionOverride = "" + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "" + }), + gpuDiscovery: true, + wantWebhook: []string{ + "spec.runtimeVersionOverride: Required value: is required when spec.image has no parseable semantic-version tag", + }, + }, + { + name: "changing a legacy custom image without override is rejected on update", + seedWithoutWebhook: true, + oldRequest: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.RuntimeVersionOverride = "" + }), + request: betaDGDRForAdmission(func(request *nvidiacomv1beta1.DynamoGraphDeploymentRequest) { + request.Spec.Image = "test-profiler:other-custom" + request.Spec.RuntimeVersionOverride = "" + }), + gpuDiscovery: true, + wantWebhook: []string{ + "spec.runtimeVersionOverride: Required value: is required when spec.image has no parseable semantic-version tag", + }, + }, { name: "missing hardware is ratcheted when GPU discovery becomes disabled", oldRequest: betaDGDRForAdmission(nil), @@ -279,10 +458,11 @@ func betaDGDRForAdmission( }, ObjectMeta: metav1.ObjectMeta{Name: "test-dgdr", Namespace: "default"}, Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ - Model: "Qwen/Qwen3-0.6B", - Backend: nvidiacomv1beta1.BackendTypeVllm, - Image: "profiler:latest", - SearchStrategy: nvidiacomv1beta1.SearchStrategyRapid, + Model: "Qwen/Qwen3-0.6B", + Backend: nvidiacomv1beta1.BackendTypeVllm, + Image: "profiler:latest", + RuntimeVersionOverride: "1.1.0", + SearchStrategy: nvidiacomv1beta1.SearchStrategyRapid, }, } if mutate != nil { @@ -301,8 +481,9 @@ func alphaDGDRForAdmission( }, ObjectMeta: metav1.ObjectMeta{Name: "test-dgdr", Namespace: "default"}, Spec: nvidiacomv1alpha1.DynamoGraphDeploymentRequestSpec{ - Model: "Qwen/Qwen3-0.6B", - Backend: "vllm", + Model: "Qwen/Qwen3-0.6B", + Backend: "vllm", + RuntimeVersionOverride: "1.1.0", ProfilingConfig: nvidiacomv1alpha1.ProfilingConfigSpec{ ProfilerImage: "profiler:latest", }, diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index 0313eca7454d..69d224a317aa 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -22,12 +22,17 @@ import ( "fmt" "strings" + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/epp" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) const ( @@ -35,8 +40,87 @@ const ( vllmDistributedExecutorBackendMP = "mp" vllmDistributedExecutorBackendRay = "ray" + + runtimeVersionOverrideRequiredMessage = "is required when the specified main container image has no parseable semantic-version tag" +) + +// runtimeVersionValidationSource identifies the API representation whose field +// paths must be used for runtime-version validation errors. +type runtimeVersionValidationSource uint8 + +const ( + runtimeVersionSourceV1Beta1 runtimeVersionValidationSource = iota + runtimeVersionSourceV1Alpha1 + runtimeVersionSourceDisabled ) +// runtimeVersionValidationSourceForRequest uses RequestKind because it preserves +// the GVK the client submitted when the API server converts the object for +// an equivalent-version webhook. For unconverted requests, RequestKind is nil; +// the handler endpoint GVK is then the source representation. +func runtimeVersionValidationSourceForRequest(ctx context.Context, fallbackGVK schema.GroupVersionKind) runtimeVersionValidationSource { + request, err := admission.RequestFromContext(ctx) + if err == nil && request.RequestKind != nil { + return runtimeVersionValidationSourceForGVK(schema.GroupVersionKind{ + Group: request.RequestKind.Group, + Version: request.RequestKind.Version, + Kind: request.RequestKind.Kind, + }) + } + return runtimeVersionValidationSourceForGVK(fallbackGVK) +} + +func runtimeVersionValidationSourceForGVK(gvk schema.GroupVersionKind) runtimeVersionValidationSource { + if gvk.GroupVersion() == nvidiacomv1alpha1.GroupVersion { + return runtimeVersionSourceV1Alpha1 + } + return runtimeVersionSourceV1Beta1 +} + +func (v *sharedValidation) validatesRuntimeVersionFor(source runtimeVersionValidationSource) bool { + return v.runtimeVersionSource == source +} + +// runtimeVersionImageAndPath returns the main image and its v1beta1 field path. +// spec and fldPath must not be nil. +func runtimeVersionImageAndPath( + spec *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + fldPath *field.Path, +) (string, *field.Path) { + imagePath := fldPath.Child("podTemplate", "spec", "containers") + + // Resolve the exact container path when the named main container exists. + if spec.PodTemplate != nil { + if index := containerIndexByName(spec.PodTemplate.Spec.Containers, consts.MainContainerName); index >= 0 { + imagePath = imagePath.Index(index).Child("image") + return spec.PodTemplate.Spec.Containers[index].Image, imagePath + } + } + return "", imagePath +} + +// runtimeVersionImageAndPathV1Alpha1 returns the main image and its v1alpha1 field path. +// spec and fldPath must not be nil. +func runtimeVersionImageAndPathV1Alpha1( + spec *nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec, + fldPath *field.Path, +) (string, *field.Path) { + imagePath := fldPath.Child("extraPodSpec", "mainContainer", "image") + if spec.ExtraPodSpec != nil && spec.ExtraPodSpec.MainContainer != nil { + return spec.ExtraPodSpec.MainContainer.Image, imagePath + } + return "", imagePath +} + +// runtimeVersionOverrideRequired reports whether image cannot provide a version and override is absent. +func runtimeVersionOverrideRequired(image, override string) bool { + if override != "" { + return false + } + _, err := runtimeversion.ParseImageVersion(image) + return err != nil +} + func hasContainerNamed(containers []corev1.Container, name string) bool { for i := range containers { if containers[i].Name == name { diff --git a/deploy/operator/internal/webhook/validation/shared_test.go b/deploy/operator/internal/webhook/validation/shared_test.go index ca04ee926874..db6053b7d5eb 100644 --- a/deploy/operator/internal/webhook/validation/shared_test.go +++ b/deploy/operator/internal/webhook/validation/shared_test.go @@ -39,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -138,6 +139,36 @@ func admissionSourceVersion(t *testing.T, object runtime.Object) string { } } +func TestRuntimeVersionImageAbsenceRatcheting(t *testing.T) { + t.Run("v1beta1", func(t *testing.T) { + validation := &sharedValidation{runtimeVersionSource: runtimeVersionSourceV1Beta1} + oldSpec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{} + newSpec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{} + + errs := validation.validateDynamoComponentDeploymentSharedSpecUpdate( + newSpec, + oldSpec, + field.NewPath("spec"), + true, + schema.GroupKind{Group: nvidiacomv1beta1.GroupVersion.Group, Kind: "DynamoComponentDeployment"}, + ) + assertFieldPaths(t, errs, nil) + }) + + t.Run("v1alpha1", func(t *testing.T) { + validation := &sharedValidation{runtimeVersionSource: runtimeVersionSourceV1Alpha1} + oldSpec := &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{} + newSpec := &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{} + + errs := validation.validateDynamoComponentDeploymentSharedSpecUpdateV1alpha1( + newSpec, + oldSpec, + field.NewPath("spec"), + ) + assertFieldPaths(t, errs, nil) + }) +} + func assertWebhookErrors(t *testing.T, err error, want []string) { t.Helper() if len(want) == 0 { @@ -186,14 +217,15 @@ func TestValidateDynamoComponentDeploymentSharedSpecFieldPaths(t *testing.T) { frontendSidecar := "missing" sharedMemorySize := resource.MustParse("-1Gi") spec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ - ComponentName: "epp", - ComponentType: nvidiacomv1beta1.ComponentTypeEPP, + ComponentName: "epp", + ComponentType: nvidiacomv1beta1.ComponentTypeEPP, + RuntimeVersionOverride: "1.1.0", PodTemplate: &corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{consts.KubeAnnotationVLLMDistributedExecutorBackend: "invalid"}, }, Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: consts.MainContainerName}, {Name: "sidecar"}}, + Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}, {Name: "sidecar"}}, InitContainers: []corev1.Container{{Name: "init"}}, }, }, @@ -226,18 +258,20 @@ func TestValidateDynamoComponentDeploymentSharedSpecFrontendSidecar(t *testing.T t.Run("requires pod template", func(t *testing.T) { name := "frontend" - spec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{FrontendSidecar: &name} + spec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{RuntimeVersionOverride: "1.1.0", FrontendSidecar: &name} errs := validation.validateDynamoComponentDeploymentSharedSpec(spec, componentPath, true, true) assertFieldPaths(t, errs, []string{ "spec.components[0].podTemplate.spec.containers", + "spec.components[0].podTemplate.spec.containers", }) }) t.Run("rejects empty name", func(t *testing.T) { name := "" spec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ - PodTemplate: &corev1.PodTemplateSpec{}, - FrontendSidecar: &name, + PodTemplate: &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}}}}, + FrontendSidecar: &name, + RuntimeVersionOverride: "1.1.0", } errs := validation.validateDynamoComponentDeploymentSharedSpec(spec, componentPath, true, true) assertFieldPaths(t, errs, []string{ @@ -249,9 +283,10 @@ func TestValidateDynamoComponentDeploymentSharedSpecFrontendSidecar(t *testing.T name := "frontend" spec := &nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ PodTemplate: &corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: name, Image: "frontend:latest"}}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: consts.MainContainerName, Image: "registry.example/runtime:1.1.0"}, {Name: name, Image: "frontend:latest"}}}, }, - FrontendSidecar: &name, + FrontendSidecar: &name, + RuntimeVersionOverride: "1.1.0", } errs := validation.validateDynamoComponentDeploymentSharedSpec(spec, componentPath, true, true) assertFieldPaths(t, errs, nil) @@ -294,10 +329,10 @@ func TestValidateDynamoComponentDeploymentSharedSpecV1alpha1FrontendSidecarField Image: "frontend:latest", Envs: []corev1.EnvVar{{Name: "TOKEN", Value: "do-not-leak-this-value"}}, } - spec := &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{FrontendSidecar: frontendSidecar} + spec := &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{RuntimeVersionOverride: "1.1.0", FrontendSidecar: frontendSidecar, ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}}} errs := validation.validateDynamoComponentDeploymentSharedSpecV1alpha1(spec, fldPath, "dynamo") assertFieldPaths(t, errs, nil) - spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{PodSpec: &corev1.PodSpec{}} + spec.ExtraPodSpec = &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}, PodSpec: &corev1.PodSpec{}} errs = validation.validateDynamoComponentDeploymentSharedSpecV1alpha1(spec, fldPath, "dynamo") assertFieldPaths(t, errs, nil) spec.ExtraPodSpec.PodSpec.Containers = []corev1.Container{{Name: consts.FrontendSidecarContainerName}} @@ -342,7 +377,9 @@ func TestValidateExperimentalSpecDoesNotExposePodTemplate(t *testing.T) { func TestValidateDynamoComponentDeploymentSharedSpecV1alpha1WarningsAndErrors(t *testing.T) { legacyNamespace := "legacy" spec := &nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ - DynamoNamespace: &legacyNamespace, + DynamoNamespace: &legacyNamespace, + RuntimeVersionOverride: "1.1.0", + ExtraPodSpec: &nvidiacomv1alpha1.ExtraPodSpec{MainContainer: &corev1.Container{Image: "registry.example/runtime:1.1.0"}}, Annotations: map[string]string{ consts.KubeAnnotationVLLMDistributedExecutorBackend: "invalid", }, diff --git a/deploy/operator/internal/webhook/validation/shared_v1alpha1.go b/deploy/operator/internal/webhook/validation/shared_v1alpha1.go index d051bf8c4d21..65b35121dc7b 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1alpha1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1alpha1.go @@ -76,6 +76,49 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecV1alpha1( if spec.Failover != nil { allErrs = append(allErrs, v.validateFailoverSpecV1alpha1(spec.Failover, fldPath.Child("failover"))...) } + + // Validate runtime compatibility against the source-version fields. + if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { + image, imagePath := runtimeVersionImageAndPathV1Alpha1(spec, fldPath) + if image == "" { + allErrs = append(allErrs, field.Required(imagePath, "is required")) + } else if !v.allowMissingRuntimeVersionOverride && + runtimeVersionOverrideRequired(image, spec.RuntimeVersionOverride) { + allErrs = append(allErrs, field.Required( + fldPath.Child("runtimeVersionOverride"), + runtimeVersionOverrideRequiredMessage, + )) + } + } + + return allErrs +} + +// validateDynamoComponentDeploymentSharedSpecUpdateV1alpha1 validates a preserved v1alpha1 shared spec update. +// newSpec, oldSpec, and fldPath must not be nil. +func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdateV1alpha1( + newSpec *nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec, + oldSpec *nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec, + fldPath *field.Path, +) field.ErrorList { + allErrs := field.ErrorList{} + + // Ratchet legacy image absence or an unchanged legacy tuple, but reject a newly invalid tuple. + if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Alpha1) { + newImage, imagePath := runtimeVersionImageAndPathV1Alpha1(newSpec, fldPath) + oldImage, _ := runtimeVersionImageAndPathV1Alpha1(oldSpec, fldPath) + if newImage == "" && oldImage != "" { + allErrs = append(allErrs, field.Required(imagePath, "is required")) + } else if !v.allowMissingRuntimeVersionOverride && + runtimeVersionOverrideRequired(newImage, newSpec.RuntimeVersionOverride) && + (newImage != oldImage || newSpec.RuntimeVersionOverride != oldSpec.RuntimeVersionOverride) { + allErrs = append(allErrs, field.Required( + fldPath.Child("runtimeVersionOverride"), + runtimeVersionOverrideRequiredMessage, + )) + } + } + return allErrs } diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta1.go b/deploy/operator/internal/webhook/validation/shared_v1beta1.go index ae7fd4a8ea2b..a1fb90bad7f7 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta1.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta1.go @@ -36,9 +36,11 @@ import ( // sharedValidation carries request-wide dependencies and accumulation used by // validation for API types shared by multiple resources. type sharedValidation struct { - ctx context.Context - mgr ctrl.Manager - warnings admission.Warnings + ctx context.Context + mgr ctrl.Manager + warnings admission.Warnings + runtimeVersionSource runtimeVersionValidationSource + allowMissingRuntimeVersionOverride bool } func (v *sharedValidation) warn(message string) { @@ -124,6 +126,20 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpec( )...) } + // Validate runtime compatibility against the source-version fields. + if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Beta1) { + image, imagePath := runtimeVersionImageAndPath(spec, fldPath) + if image == "" { + allErrs = append(allErrs, field.Required(imagePath, "is required")) + } else if !v.allowMissingRuntimeVersionOverride && + runtimeVersionOverrideRequired(image, spec.RuntimeVersionOverride) { + allErrs = append(allErrs, field.Required( + fldPath.Child("runtimeVersionOverride"), + runtimeVersionOverrideRequiredMessage, + )) + } + } + return allErrs } @@ -389,6 +405,22 @@ func (v *sharedValidation) validateDynamoComponentDeploymentSharedSpecUpdate( )) } } + + // Ratchet legacy image absence or an unchanged legacy tuple, but reject a newly invalid tuple. + if v.validatesRuntimeVersionFor(runtimeVersionSourceV1Beta1) { + newImage, imagePath := runtimeVersionImageAndPath(newComponent, fldPath) + oldImage, _ := runtimeVersionImageAndPath(oldComponent, fldPath) + if newImage == "" && oldImage != "" { + allErrs = append(allErrs, field.Required(imagePath, "is required")) + } else if !v.allowMissingRuntimeVersionOverride && + runtimeVersionOverrideRequired(newImage, newComponent.RuntimeVersionOverride) && + (newImage != oldImage || newComponent.RuntimeVersionOverride != oldComponent.RuntimeVersionOverride) { + allErrs = append(allErrs, field.Required( + fldPath.Child("runtimeVersionOverride"), + runtimeVersionOverrideRequiredMessage, + )) + } + } return allErrs } diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 2424a60299da..4c71c2d32bb3 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -31,6 +31,7 @@ import ( const ( admissionOperatorPrincipal = "system:serviceaccount:dynamo-system:dynamo-operator" + customRuntimeImage = "registry.example/runtime:custom" legacySeedUsername = "operatorenv-legacy-seeder" ) diff --git a/deploy/operator/samples/dgd-gms-failover.yaml b/deploy/operator/samples/dgd-gms-failover.yaml index 0b4052a7333e..3dc46f1c1e9b 100644 --- a/deploy/operator/samples/dgd-gms-failover.yaml +++ b/deploy/operator/samples/dgd-gms-failover.yaml @@ -49,7 +49,7 @@ spec: # numShadows: 1 # 1 primary + 1 shadow = 2 engine pods per PCSG replica # extraPodSpec: # mainContainer: - # image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest + # image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 # command: ["python3", "-m", "dynamo.vllm"] # args: ["--model", "Qwen/Qwen3-0.6B", "--tensor-parallel-size", "1", "--enforce-eager", "--gpu-memory-utilization", "0.85"] # sharedMemory: @@ -72,7 +72,7 @@ spec: numShadows: 1 # 1 primary + 1 shadow = 2 engine pods per rank extraPodSpec: mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 command: ["python3", "-m", "dynamo.vllm"] # args: ["--model", "Qwen/Qwen3-235B-A22B", "--tensor-parallel-size", "8", "--enforce-eager", "--gpu-memory-utilization", "0.85"] args: ["--model", "Qwen/Qwen3-0.6B", "--tensor-parallel-size", "2", "--enforce-eager", "--gpu-memory-utilization", "0.85"] @@ -87,5 +87,5 @@ spec: replicas: 1 extraPodSpec: mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 # command: ["python3", "-m", "dynamo.frontend"] diff --git a/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml b/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml index d0ad609cd9e9..636f0e4712d6 100644 --- a/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml +++ b/deploy/operator/samples/v1beta1/dgd-gms-failover.yaml @@ -59,7 +59,7 @@ spec: # - python3 # - -m # - dynamo.vllm - # image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest + # image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 # name: main # resources: # limits: @@ -98,7 +98,7 @@ spec: envFrom: - secretRef: name: hf-token-secret - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 name: main resources: limits: @@ -116,7 +116,7 @@ spec: - secretRef: name: hf-token-secret # command: ["python3", "-m", "dynamo.frontend"] - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 name: main replicas: 1 type: frontend diff --git a/docs/fern/components/profiler/profiler-guide.md b/docs/fern/components/profiler/profiler-guide.md index 22eb82b97769..b8ce155c5448 100644 --- a/docs/fern/components/profiler/profiler-guide.md +++ b/docs/fern/components/profiler/profiler-guide.md @@ -202,15 +202,31 @@ The recommended deployment method is through DGDRs. See [Profiler Examples](prof #### Container Images -Each DGDR requires a container image for profiling and deployment: - -- **`image`** (Optional): Container image for the profiling job. Must contain the profiler code and dependencies. +The DGDR `image` field selects the container image for the profiling job. The +image must contain the profiler code and dependencies. If `image` is omitted, +the operator defaults it to +`nvcr.io/nvidia/ai-dynamo/dynamo-planner:`. ```yaml spec: image: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.1" # dynamo-frontend for Dynamo < 1.1.0 ``` +> [!NOTE] +> The DGDR-level `spec.runtimeVersionOverride` supplies a default for generated +> DGD components. The operator applies it after processing profiler output and +> DGD overrides only when a component does not already set an explicit value, +> including when a profiler based on Dynamo 1.3.0 or earlier discards the field +> while parsing the DGDR. Set the override when the effective generated runtime +> images do not use tags that identify their Dynamo runtime versions. +> The profiler also applies the field when its output is consumed directly, +> outside the operator-managed DGDR workflow. For DGDR-managed deployments, an +> explicit component value in `spec.overrides.dgd` takes precedence over the +> DGDR-level default. +> +> [Profiler Image Version Compatibility](../../kubernetes/dgdr-reference.mdx#profiler-image-version-compatibility) +> for details. + #### Quick Start: Deploy with DGDR **Step 1: Create Your DGDR** diff --git a/docs/fern/development/custom-containers.md b/docs/fern/development/custom-containers.md index 54d7a0ec18a2..56fdf3e9468f 100644 --- a/docs/fern/development/custom-containers.md +++ b/docs/fern/development/custom-containers.md @@ -69,6 +69,52 @@ COPY --from=backend-builder /src/target/release/my-backend /usr/local/bin/my-bac ENTRYPOINT ["my-backend"] ``` +## Declare the Runtime Version for Kubernetes + +When you deploy a custom image with a `DynamoGraphDeployment` (DGD) or standalone +`DynamoComponentDeployment` (DCD), admission determines the Dynamo runtime compatibility version +from the component's main-container image tag. The tag itself must be a semantic version. Tags such +as `1.4.0`, `v1.4.0`, and `1.4.0-cuda13` can provide the version. Set +`runtimeVersionOverride` when the image is tagless, digest-only, uses a tag such as `latest`, +`main`, `sha-abc`, or `cuda13-1.4.0`, or when a semantic-version tag does not identify the Dynamo +runtime version packaged in the image. + +Set the override to the canonical `MAJOR.MINOR.PATCH` Dynamo runtime version without a `v` prefix, +prerelease suffix, or build metadata. Each numeric segment may contain at most four digits. +Admission can verify that an image tag is parseable, but it cannot determine whether that tag +actually represents the Dynamo runtime version. Set the override when a valid semantic-version +tag describes another part of the image stack: + +```yaml +spec: + backendFramework: vllm + components: + - name: worker + runtimeVersionOverride: "1.4.0" + podTemplate: + spec: + containers: + - name: main + image: registry.example/my-runtime:build-20260723 +``` + +The override is optional in the CRD schema but conditionally required by admission: + +| API and resource | Main image | Runtime version override | +| --- | --- | --- | +| `v1beta1` DGD component | `spec.components[*].podTemplate.spec.containers[name=main].image` | `spec.components[*].runtimeVersionOverride` | +| `v1beta1` standalone DCD | `spec.podTemplate.spec.containers[name=main].image` | `spec.runtimeVersionOverride` | +| `v1alpha1` DGD service | `spec.services..extraPodSpec.mainContainer.image` | `spec.services..runtimeVersionOverride` | +| `v1alpha1` standalone DCD | `spec.extraPodSpec.mainContainer.image` | `spec.runtimeVersionOverride` | + +The main image remains required when an override is set. Sidecar image tags are not used for +runtime-version detection. The override declares the Dynamo runtime packaged in the image; it is +not the CUDA, inference-engine, operator, Git, or image-build version. It does not rewrite the +image or change the rendered Pod, and changing only this field does not trigger a worker rollout. +For compatibility, an existing component created without its pod configuration or main image can +still be updated while that field remains missing. New components must provide both, and adding, +changing, or removing the image applies current admission validation. + ## Run Locally Use `container/run.sh` to launch the image with the same GPU and mount defaults used by Dynamo development workflows: diff --git a/docs/fern/kubernetes/api-reference.md b/docs/fern/kubernetes/api-reference.md index ced095646d97..0cf003ba1521 100644 --- a/docs/fern/kubernetes/api-reference.md +++ b/docs/fern/kubernetes/api-reference.md @@ -427,6 +427,7 @@ _Appears in:_ | `serviceName` _string_ | The name of the component | | | | `componentType` _string_ | ComponentType indicates the role of this component (for example, "main"). | | | | `subComponentType` _string_ | SubComponentType indicates the sub-role of this component (for example, "prefill"). | | | +| `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
It does not change the image or rendered Pod, and changing only this field does not trigger a rollout. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | GlobalDynamoNamespace indicates that the Component will be placed in the global Dynamo namespace | | | | `resources` _[Resources](#resources)_ | Resources requested and limits for this component, including CPU, memory,
GPUs/devices, and any runtime-specific resources. | | | | `autoscaling` _[Autoscaling](#autoscaling)_ | Deprecated: This field is deprecated and ignored. Use DynamoGraphDeploymentScalingAdapter
with HPA, KEDA, or Planner for autoscaling instead. See docs/kubernetes/autoscaling.md
for migration guidance. This field will be removed in a future API version. | | | @@ -437,7 +438,7 @@ _Appears in:_ | `modelRef` _[ModelReference](#modelreference)_ | ModelRef references a model that this component serves
When specified, a headless service will be created for endpoint discovery | | Optional: \{\}
| | `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | | | `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | Optional: \{\}
| -| `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. | | Optional: \{\}
| +| `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. New components must set
extraPodSpec and provide a non-empty mainContainer image. Existing components created
without extraPodSpec may remain unchanged. | | Optional: \{\}
| | `livenessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | LivenessProbe to detect and restart unhealthy containers. | | | | `readinessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | ReadinessProbe to signal when the container is ready to receive traffic. | | | | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| @@ -471,6 +472,7 @@ _Appears in:_ | `serviceName` _string_ | The name of the component | | | | `componentType` _string_ | ComponentType indicates the role of this component (for example, "main"). | | | | `subComponentType` _string_ | SubComponentType indicates the sub-role of this component (for example, "prefill"). | | | +| `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
It does not change the image or rendered Pod, and changing only this field does not trigger a rollout. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | GlobalDynamoNamespace indicates that the Component will be placed in the global Dynamo namespace | | | | `resources` _[Resources](#resources)_ | Resources requested and limits for this component, including CPU, memory,
GPUs/devices, and any runtime-specific resources. | | | | `autoscaling` _[Autoscaling](#autoscaling)_ | Deprecated: This field is deprecated and ignored. Use DynamoGraphDeploymentScalingAdapter
with HPA, KEDA, or Planner for autoscaling instead. See docs/kubernetes/autoscaling.md
for migration guidance. This field will be removed in a future API version. | | | @@ -481,7 +483,7 @@ _Appears in:_ | `modelRef` _[ModelReference](#modelreference)_ | ModelRef references a model that this component serves
When specified, a headless service will be created for endpoint discovery | | Optional: \{\}
| | `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | | | `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | Optional: \{\}
| -| `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. | | Optional: \{\}
| +| `extraPodSpec` _[ExtraPodSpec](#extrapodspec)_ | ExtraPodSpec allows to override the main pod spec configuration.
It is a k8s standard PodSpec. It also contains a MainContainer (standard k8s Container) field
that allows overriding the main container configuration. New components must set
extraPodSpec and provide a non-empty mainContainer image. Existing components created
without extraPodSpec may remain unchanged. | | Optional: \{\}
| | `livenessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | LivenessProbe to detect and restart unhealthy containers. | | | | `readinessProbe` _[Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core)_ | ReadinessProbe to signal when the container is ready to receive traffic. | | | | `replicas` _integer_ | Replicas is the desired number of Pods for this component.
When scalingAdapter is enabled, this field is managed by the
DynamoGraphDeploymentScalingAdapter and should not be modified directly. | | Minimum: 0
| @@ -588,6 +590,7 @@ _Appears in:_ | `backend` _string_ | Backend specifies the inference backend for profiling.
The controller automatically sets this value in profilingConfig.config.engine.backend.
Profiling runs on real GPUs or via AIC simulation to collect performance data. | | Enum: [auto vllm sglang trtllm]
Required: \{\}
| | `useMocker` _boolean_ | UseMocker indicates whether to deploy a mocker DynamoGraphDeployment instead of
a real backend deployment. When true, the deployment uses simulated engines that
don't require GPUs, using the profiling data to simulate realistic timing behavior.
Mocker is available in all backend images and useful for large-scale experiments.
Profiling still runs against the real backend (specified above) to collect performance data. | false | | | `profilingConfig` _[ProfilingConfigSpec](#profilingconfigspec)_ | ProfilingConfig provides the complete configuration for the profiling job.
Note: GPU discovery is automatically attempted to detect GPU resources from Kubernetes
cluster nodes. If the operator has node read permissions (cluster-wide or explicitly granted),
discovered GPU configuration is used as defaults when hardware configuration is not manually
specified (minNumGpusPerEngine, maxNumGpusPerEngine, numGpusPerNode). User-specified values
always take precedence over auto-discovered values. If GPU discovery fails (e.g.,
namespace-restricted operator without node permissions), manual hardware config is required.
This configuration is passed directly to the profiler.
The structure matches the profile_sla config format exactly (see ProfilingConfigSpec for schema).
Note: deployment.model and engine.backend are automatically set from the high-level
modelName and backend fields and should not be specified in this config. | | Required: \{\}
| +| `runtimeVersionOverride` _string_ | RuntimeVersionOverride explicitly sets the Dynamo runtime version for every
component in the generated DynamoGraphDeployment. Set this when
profilingConfig.profilerImage uses a non-semantic-version tag or digest, or
when its tag does not identify the Dynamo runtime version. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `enableGpuDiscovery` _boolean_ | EnableGPUDiscovery controls whether the operator attempts to discover GPU hardware from cluster nodes.
DEPRECATED: This field is deprecated and will be removed in v1beta1. GPU discovery is now always
attempted automatically. Setting this field has no effect - the operator will always try to discover
GPU hardware when node read permissions are available. If discovery is unavailable (e.g., namespace-scoped
operator without permissions), manual hardware configuration is required regardless of this setting. | true | Optional: \{\}
| | `autoApply` _boolean_ | AutoApply indicates whether to automatically create a DynamoGraphDeployment
after profiling completes. If false, only the spec is generated and stored in status.
Users can then manually create a DGD using the generated spec. | false | | | `deploymentOverrides` _[DeploymentOverridesSpec](#deploymentoverridesspec)_ | DeploymentOverrides allows customizing metadata for the auto-created DGD.
Only applicable when AutoApply is true. | | Optional: \{\}
| @@ -2120,8 +2123,9 @@ _Appears in:_ | --- | --- | --- | --- | | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| | `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp]
Optional: \{\}
| +| `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image or rendered Pod, and changing only this field does
not trigger a rollout. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate is the pod template used to create the component's pods.
The operator injects its defaults (image, command, env, ports, probes,
resources, volume mounts) into the container named `"main"` inside
`podTemplate.spec.containers`, merging user overrides by name. If no
container named `"main"` is present, the operator auto-generates it
with standard defaults. All other containers in `podTemplate.spec.containers`
are treated as user-managed sidecars: the operator does not inject
defaults into them, so sidecars must specify required fields (e.g. `image`)
themselves. The validation webhook rejects pod templates where a
non-`"main"` container is missing a required field such as `image`. | | Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures multinode components. | | Optional: \{\}
| @@ -2151,8 +2155,9 @@ _Appears in:_ | `backendFramework` _string_ | backendFramework specifies the backend framework. | | Enum: [sglang vllm trtllm]
| | `name` _string_ | name is the stable logical identifier for this component within its
DynamoGraphDeployment. It must be unique within the parent's
`spec.components` list.
For standalone DynamoComponentDeployment objects, the defaulting webhook
populates `name` from `metadata.name` on admission, so users
typically do not need to set it explicitly.
`name` is decoupled from the underlying Kubernetes resource name so that
the operator can rename child workloads (e.g. suffixing worker DCDs with
a hash during rolling updates) without losing the stable identity that
downstream consumers (labels, status maps, DGDSA references, planner
RBAC, EPP filters) depend on. | | MaxLength: 63
MinLength: 1
Pattern: `^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$`
Required: \{\}
| | `type` _[ComponentType](#componenttype)_ | type indicates the role of this component within a Dynamo graph. Drives
port mapping, frontend detection, planner RBAC, and the pod label
`nvidia.com/dynamo-component-type`. Because `prefill` and `decode` are
first-class values, users can set them directly. | | Enum: [frontend worker prefill decode planner epp]
Optional: \{\}
| +| `runtimeVersionOverride` _string_ | RuntimeVersionOverride declares the Dynamo runtime compatibility version in this component's
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image or rendered Pod, and changing only this field does
not trigger a rollout. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `globalDynamoNamespace` _boolean_ | globalDynamoNamespace places the component in the global Dynamo
namespace rather than the per-deployment namespace derived from the
DGD name. | | Optional: \{\}
| -| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate is the pod template used to create the component's pods.
The operator injects its defaults (image, command, env, ports, probes,
resources, volume mounts) into the container named `"main"` inside
`podTemplate.spec.containers`, merging user overrides by name. If no
container named `"main"` is present, the operator auto-generates it
with standard defaults. All other containers in `podTemplate.spec.containers`
are treated as user-managed sidecars: the operator does not inject
defaults into them, so sidecars must specify required fields (e.g. `image`)
themselves. The validation webhook rejects pod templates where a
non-`"main"` container is missing a required field such as `image`. | | Optional: \{\}
| +| `podTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | podTemplate defines the component's Pod configuration. New components must
include a container named "main" with a non-empty image. Existing components
created without a podTemplate may remain unchanged. The operator merges
defaults into the main container.
For DGD components whose main image tag is not a Dynamo semantic version,
set runtimeVersionOverride explicitly.
All other containers are user-managed sidecars and must specify their
required fields, including image. | | Optional: \{\}
| | `replicas` _integer_ | replicas is the desired number of Pods for this component. When
`scalingAdapter` is set on this component, this field is managed by
the DynamoGraphDeploymentScalingAdapter and should not be modified
directly. | | Minimum: 0
Optional: \{\}
| | `minAvailable` _integer_ | minAvailable maps to Grove PodClique minAvailable for single-node and
Grove PodCliqueScalingGroup minAvailable for multi-node components.
This field determines 1) the minimum number of replicas guaranteed to be
gang-scheduled, and 2) when violating minAvailable replicas triggers gang
termination.
For Grove-backed DynamoGraphDeployment components, minAvailable defaults to
1 when omitted and is immutable after creation. Positive replica counts must
be greater than or equal to minAvailable. Replicas may be scaled to 0 as a
special scale-to-zero state; minAvailable remains configured but is not
enforced again until replicas is scaled back to a positive value.
For non-Grove deployments, setting this field will result in a validation error. | | Minimum: 1
Optional: \{\}
| | `multinode` _[MultinodeSpec](#multinodespec)_ | multinode configures multinode components. | | Optional: \{\}
| @@ -2276,6 +2281,7 @@ _Appears in:_ | `model` _string_ | Model specifies the model to deploy (e.g., "Qwen/Qwen3-0.6B", "meta-llama/Llama-3-70b").
Can be a HuggingFace ID or a private model name. | | MinLength: 1
Required: \{\}
| | `backend` _[BackendType](#backendtype)_ | Backend specifies the inference backend to use for profiling and deployment. | auto | Enum: [auto sglang trtllm vllm]
Optional: \{\}
| | `image` _string_ | Image is the container image reference for the profiling job (planner image).
Example: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.1".
For Dynamo < 1.1.0, use dynamo-frontend. | | Optional: \{\}
| +| `runtimeVersionOverride` _string_ | RuntimeVersionOverride supplies the default Dynamo runtime version for
generated DynamoGraphDeployment components that do not set their own
override. Set this when Image uses a non-semantic-version tag or digest, or
when its tag does not identify the Dynamo runtime version. An explicit
component value in overrides.dgd takes precedence. | | Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `modelCache` _[ModelCacheSpec](#modelcachespec)_ | ModelCache provides optional PVC configuration for pre-downloaded model weights.
When provided, weights are loaded from the PVC instead of downloading from HuggingFace. | | Optional: \{\}
| | `hardware` _[HardwareSpec](#hardwarespec)_ | Hardware describes the hardware resources available for profiling and deployment.
Typically auto-filled by the operator from cluster discovery. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#workloadspec)_ | Workload defines the expected workload characteristics for SLA-based profiling. | | Optional: \{\}
| diff --git a/docs/fern/kubernetes/dgdr-guide.md b/docs/fern/kubernetes/dgdr-guide.md index 20c2a2ec2e11..2cf7d97bb5d6 100644 --- a/docs/fern/kubernetes/dgdr-guide.md +++ b/docs/fern/kubernetes/dgdr-guide.md @@ -266,6 +266,21 @@ kubectl get dgdr my-model -n \ kubectl apply -f my-dgd.yaml -n ``` +While `autoApply` is disabled, you may set or change +`runtimeVersionOverride` during `Profiling` or `Ready`. To have the operator +deploy the reviewed snapshot, enable `autoApply`: + +```bash +kubectl patch dgdr my-model -n --type=merge \ + -p '{"spec":{"runtimeVersionOverride":"1.4.0"}}' +kubectl patch dgdr my-model -n --type=merge \ + -p '{"spec":{"autoApply":true}}' +``` + +You may also set or change the override in the update that enables +`autoApply`. The operator applies the current value to the new DGD without +changing the stored snapshot. + @@ -278,7 +293,7 @@ A DGDR progresses through these phases. Profiling failures are terminal — they |---|---| | `Pending` | Spec validated; operator is discovering GPU hardware and preparing the profiling job | | `Profiling` | Profiling job running (sub-phases: `Initializing`, `SweepingPrefill`, `SweepingDecode`, `SelectingConfig`, `BuildingCurves`, `GeneratingDGD`, `Done`) | -| `Ready` | Profiling complete; config stored in `.status.profilingResults.selectedConfig`. Terminal when `autoApply: false`. | +| `Ready` | Profiling complete; config stored in `.status.profilingResults.selectedConfig`. Waits for manual application or for `autoApply` to be enabled. | | `Deploying` | Creating the DGD (only when `autoApply: true`) | | `Deployed` | DGD is running and healthy | | `Failed` | Unrecoverable error — check events and conditions | @@ -311,7 +326,7 @@ For the full lifecycle, conditions, and monitoring command reference, see [DGDR | **OOM during profiling or serving** | The model doesn't fit in GPU memory at the selected TP. Raise `hardware.totalGpus`; edge cases (long context, KV overhead) need more than the minimum. | | **Profiler ignores extra GPUs** | Auto-detection caps at 32. Set `hardware.totalGpus` explicitly. | | **Profiling job won't schedule** | GPU nodes are tainted. Add tolerations through `overrides.profilingJob`. | -| **Spec edits rejected** | The DGDR spec is immutable once it enters `Profiling`. Delete and recreate the DGDR. | +| **Spec edits rejected** | The DGDR spec is immutable once it enters `Profiling`, except that a request with `autoApply: false` may set or change `runtimeVersionOverride` during `Profiling` or `Ready`. In `Ready`, it may also enable `autoApply`. Delete and recreate the DGDR for other changes. | | **Multinode deployment errors out** | Grove or LWS is missing. See [Multinode Orchestration](multinode-installation.md). | ### Profiling Job Fails to Schedule diff --git a/docs/fern/kubernetes/dgdr-reference.mdx b/docs/fern/kubernetes/dgdr-reference.mdx index f160c1703694..527360de94ca 100644 --- a/docs/fern/kubernetes/dgdr-reference.mdx +++ b/docs/fern/kubernetes/dgdr-reference.mdx @@ -23,6 +23,10 @@ Only `model` is required; every other field has a default or is auto-detected. Container image for the profiling job (the planner image), for example `nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.1`. For Dynamo < 1.1.0, use `dynamo-frontend`. + + Dynamo runtime version propagated to generated DGD components when `image` has no parseable semantic-version tag or its tag does not identify the intended runtime. + + Inference backend used for profiling and deployment. @@ -39,6 +43,46 @@ Only `model` is required; every other field has a default or is auto-detected. Automatically create a DGD after profiling completes. When `false`, the generated spec is stored in `status` for manual review instead of being applied. +### Profiler image version compatibility + +When `spec.image` is omitted, the operator defaults it on creation to +`nvcr.io/nvidia/ai-dynamo/dynamo-planner:`. The operator +requires `operatorVersion` to be valid semantic versioning, so the default +image has a parseable version tag. + +The DGDR-level `spec.runtimeVersionOverride` supplies a default for generated +DGD components. After the profiler materializes the DGD and applies +`spec.overrides.dgd`, the operator copies the DGDR value only to components that +do not already have an explicit override. This behavior applies across profiler +versions. Profilers through Dynamo 1.3.0 may discard the field while parsing the +DGDR, but the operator reapplies the default from the stored DGDR. + +Set `spec.runtimeVersionOverride` when an effective generated component image +uses a non-semantic-version tag or digest, or when its tag does not identify the +intended Dynamo runtime version. The DGDR value also covers images replaced +through `spec.overrides.dgd` when that component does not set its own +`runtimeVersionOverride`. An explicit component-level value takes precedence. + +While a DGDR has `autoApply: false`, you may set or change +`runtimeVersionOverride` during `Profiling` or `Ready`. After it reaches +`Ready`, you may enable `autoApply`, either separately or while changing the +override. The operator applies the current value while creating the new DGD. +These updates do not modify `status.profilingResults.selectedConfig` or the +`nvidia.com/generated-dgd-spec` annotation. Other spec updates remain +forbidden after profiling starts. + +When the DGDR-level field is unset, each effective generated DGD component +follows the DGD component rules independently. An overridden image without a +parseable semantic-version tag requires `runtimeVersionOverride` on that +component. + +Profilers through Dynamo 1.3.0 discard the embedded DGD override's `apiVersion` +and `kind`, then directly merge the remaining dictionary into a generated +`v1alpha1` DGD. Raw storage preserves the override fields but does not convert +between DGD schemas. Use a `v1alpha1`-shaped override under `spec.services`; a +`v1beta1` `spec.components` override is not translated into the generated +service. + Expected workload characteristics for SLA-based profiling. @@ -850,7 +894,7 @@ When you create a DGDR, it progresses through these phases (`status.phase`): |---|---| | `Pending` | Spec validated; operator is discovering GPU hardware and preparing the profiling job | | `Profiling` | Profiling job running — sub-phases in `status.profilingPhase`: `Initializing`, `SweepingPrefill`, `SweepingDecode`, `SelectingConfig`, `BuildingCurves`, `GeneratingDGD`, `Done` | -| `Ready` | Profiling complete; optimal config stored in `status.profilingResults.selectedConfig`. Terminal state when `autoApply: false`. | +| `Ready` | Profiling complete; optimal config stored in `status.profilingResults.selectedConfig`. Waits for manual application or for `autoApply` to be enabled. | | `Deploying` | Creating the DGD (only when `autoApply: true`) | | `Deployed` | DGD is running and healthy | | `Failed` | Unrecoverable error — profiling failures are not retried (`backoffLimit: 0`); check events and conditions for details | diff --git a/tests/fault_tolerance/deploy/templates/vllm/moe_agg.yaml b/tests/fault_tolerance/deploy/templates/vllm/moe_agg.yaml index afea31bf2aa1..a46fa09adce9 100644 --- a/tests/fault_tolerance/deploy/templates/vllm/moe_agg.yaml +++ b/tests/fault_tolerance/deploy/templates/vllm/moe_agg.yaml @@ -12,7 +12,7 @@ spec: replicas: 1 extraPodSpec: mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker @@ -48,7 +48,7 @@ spec: imagePullSecrets: - name: nvcr-imagepullsecret mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 workingDir: /workspace/examples/backends/vllm command: - python3 diff --git a/tests/fault_tolerance/deploy/templates/vllm/moe_disagg.yaml b/tests/fault_tolerance/deploy/templates/vllm/moe_disagg.yaml index 50767ef44745..97ea199b8f3d 100644 --- a/tests/fault_tolerance/deploy/templates/vllm/moe_disagg.yaml +++ b/tests/fault_tolerance/deploy/templates/vllm/moe_disagg.yaml @@ -14,7 +14,7 @@ spec: imagePullSecrets: - name: nvcr-imagepullsecret mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker @@ -51,7 +51,7 @@ spec: imagePullSecrets: - name: nvcr-imagepullsecret mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 workingDir: /workspace/examples/backends/vllm command: - python3 @@ -116,7 +116,7 @@ spec: imagePullSecrets: - name: nvcr-imagepullsecret mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 workingDir: /workspace/examples/backends/vllm command: - python3 diff --git a/tests/fault_tolerance/deploy/templates/vllm/moe_elastic_ep_demo.yaml b/tests/fault_tolerance/deploy/templates/vllm/moe_elastic_ep_demo.yaml index 3433cfea78fe..87ef3ee80b8f 100644 --- a/tests/fault_tolerance/deploy/templates/vllm/moe_elastic_ep_demo.yaml +++ b/tests/fault_tolerance/deploy/templates/vllm/moe_elastic_ep_demo.yaml @@ -28,7 +28,7 @@ spec: extraPodSpec: mainContainer: # Update tag to match VllmDecodeWorker image below - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 VllmDecodeWorker: envFromSecret: hf-token-secret componentType: worker @@ -74,7 +74,7 @@ spec: persistentVolumeClaim: claimName: model-cache mainContainer: - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:my-tag + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.4.0 imagePullPolicy: Always workingDir: /workspace/examples/backends/vllm volumeMounts: