Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions components/src/dynamo/profiler/rapid.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,17 @@

import pandas as pd
import yaml

from aiconfigurator.cli.main import _execute_task_configs, build_default_task_configs
from aiconfigurator.generator.api import (
generate_backend_artifacts,
generate_naive_config,
)
from aiconfigurator.generator.module_bridge import task_config_to_generator_config
from aiconfigurator.sdk.task import TaskConfig, TaskRunner

from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS
from dynamo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec
from dynamo.profiler.utils.profile_common import derive_backend_image

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -56,7 +57,7 @@ def _generate_dgd_from_pick(
generator_overrides: dict = {}

k8s_overrides: dict = {}
k8s_overrides["k8s_image"] = dgdr.image
k8s_overrides["k8s_image"] = derive_backend_image(dgdr.image, tc.backend_name)
if dgdr.modelCache:
if dgdr.modelCache.pvcName:
k8s_overrides["k8s_pvc_name"] = dgdr.modelCache.pvcName
Expand Down Expand Up @@ -115,7 +116,9 @@ def run_rapid(
dgd_config = yaml.safe_load(dgd_yaml) if dgd_yaml else None
if dgd_config:
config_modifier = CONFIG_MODIFIERS[backend]
dgd_config = config_modifier.update_image(dgd_config, dgdr.image)
dgd_config = config_modifier.update_image(
dgd_config, derive_backend_image(dgdr.image, backend)
)
if dgdr.modelCache and dgdr.modelCache.pvcName:
dgd_config = config_modifier.update_model_from_pvc(
dgd_config,
Expand Down
9 changes: 6 additions & 3 deletions components/src/dynamo/profiler/thorough.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@

import pandas as pd
import yaml

from aiconfigurator.generator.enumerate import enumerate_profiling_configs
from aiconfigurator.sdk.picking import pick_autoscale, pick_default, pick_load_match
from aiconfigurator.sdk.task import TaskConfig

from deploy.utils.dynamo_deployment import DynamoDeploymentClient
from dynamo.planner.defaults import SubComponentType
from dynamo.profiler.rapid import _generate_dgd_from_pick
Expand All @@ -40,7 +40,10 @@
from dynamo.profiler.utils.config import Config, get_service_name_by_type
from dynamo.profiler.utils.config_modifiers import CONFIG_MODIFIERS
from dynamo.profiler.utils.dgdr_v1beta1_types import DynamoGraphDeploymentRequestSpec
from dynamo.profiler.utils.profile_common import ProfilerOperationalConfig
from dynamo.profiler.utils.profile_common import (
ProfilerOperationalConfig,
derive_backend_image,
)
from dynamo.profiler.utils.profile_decode import get_num_request_range

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -69,7 +72,7 @@ async def run_thorough(
model_path=model,
system=system,
backend=backend,
image=dgdr.image,
image=derive_backend_image(dgdr.image, backend),
isl=isl,
osl=osl,
num_gpus_per_node=dgdr.hardware.numGpusPerNode,
Expand Down
34 changes: 25 additions & 9 deletions components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,6 @@

from pydantic import BaseModel, Field, model_validator

# Import canonical planner types – do NOT redefine them here.
from dynamo.planner.utils.planner_config import ( # noqa: F401 (re-exported)
PlannerConfig,
PlannerPreDeploymentSweepMode,
)


class DGDRPhase(str, Enum):
Pending = "Pending"
Expand Down Expand Up @@ -69,6 +63,12 @@ class BackendType(str, Enum):
Vllm = "vllm"


class PlannerPreDeploymentSweepMode(str, Enum):
None_ = "none"
Rapid = "rapid"
Thorough = "thorough"


class WorkloadSpec(BaseModel):
"""WorkloadSpec defines the workload characteristics for SLA-based profiling."""

Expand Down Expand Up @@ -161,6 +161,22 @@ class OverridesSpec(BaseModel):
)


class PlannerSpec(BaseModel):
"""PlannerSpec configures the SLA planner for autoscaling in the generated DGD."""

enabled: Optional[bool] = Field(
default=None, description="Enabled indicates whether the planner is enabled."
)
plannerPreDeploymentSweeping: Optional[PlannerPreDeploymentSweepMode] = Field(
default=None,
description='PlannerPreDeploymentSweeping controls pre-deployment sweeping mode for planner in-depth profiling. "none" means no pre-deployment sweep (only load-based scaling). "rapid" uses AI Configurator to simulate engine performance. "thorough" uses real GPUs to measure engine performance (takes several hours).',
)
plannerArgsList: Optional[List[str]] = Field(
default=None,
description="PlannerArgsList is a list of additional planner arguments.",
)


class MockerSpec(BaseModel):
"""MockerSpec configures the simulated (mocker) backend."""

Expand All @@ -182,9 +198,9 @@ class KVRouterSpec(BaseModel):
class FeaturesSpec(BaseModel):
"""FeaturesSpec controls optional Dynamo platform features in the generated deployment."""

planner: Optional[PlannerConfig] = Field(
planner: Optional[PlannerSpec] = Field(
default=None,
description="Planner is the raw SLA planner configuration passed to the planner service. Its schema is defined by dynamo.planner.utils.planner_config.PlannerConfig. Go treats this as opaque bytes; the Planner service validates it at startup. The presence of this field (non-null) enables the planner in the generated DGD.",
description="Planner configures the SLA planner for autoscaling in the generated DGD.",
)
mocker: Optional[MockerSpec] = Field(
default=None,
Expand Down Expand Up @@ -223,7 +239,7 @@ class DynamoGraphDeploymentRequestSpec(BaseModel):
)
image: Optional[str] = Field(
default=None,
description='Image is the container image reference for the profiling job (frontend image). Example: "nvcr.io/nvidia/dynamo-runtime:latest" TODO: In a future MR, the operator will derive the backend inference image from the backend type automatically; backend images can be overridden via overrides.dgd.',
description='Image is the container image reference for the profiling job (frontend image). Example: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.0".',
)
modelCache: Optional[ModelCacheSpec] = Field(
default=None,
Expand Down
60 changes: 60 additions & 0 deletions components/src/dynamo/profiler/utils/profile_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,66 @@

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Published container image naming conventions
# ---------------------------------------------------------------------------

# Mapping from backend name to the image-name component of the published
# backend runtime image.
# e.g. vllm → nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.0.0
BACKEND_IMAGE_NAMES: dict[str, str] = {
"vllm": "vllm-runtime",
"sglang": "sglang-runtime",
"trtllm": "tensorrtllm-runtime",
}


def derive_backend_image(profiler_image: str, backend: str) -> str:
"""Derive the backend worker image from the profiler image.

Replaces the image name (the last ``/``-delimited component, before any
``:tag``) with the backend-specific runtime image name, preserving the
registry path and tag unchanged.

Examples::

derive_backend_image(
"nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.0", "vllm"
)
# → "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.0.0"

derive_backend_image("myregistry.io/sglang-runtime:1.0.0", "sglang")
# → "myregistry.io/sglang-runtime:1.0.0"

Args:
profiler_image: Any Docker image reference of the form
``[REGISTRY/]NAME[:TAG]``.
backend: The resolved backend type (``'vllm'``, ``'sglang'``, or
``'trtllm'``).

Returns:
The backend container image string.

Raises:
ValueError: If *backend* is not a recognised backend.
"""
backend_image_name = BACKEND_IMAGE_NAMES.get(backend)
if backend_image_name is None:
raise ValueError(
f"Cannot derive backend image for unknown backend '{backend}'. "
f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}"
)

# Split off the last path component: "registry/path/name:tag" → "name:tag"
slash_idx = profiler_image.rfind("/")
prefix = profiler_image[: slash_idx + 1] if slash_idx >= 0 else ""
suffix = profiler_image[slash_idx + 1 :]
colon_idx = suffix.find(":")
tag = suffix[colon_idx:] if colon_idx >= 0 else ""

return f"{prefix}{backend_image_name}{tag}"


# ---------------------------------------------------------------------------
# Operational defaults not part of DynamoGraphDeploymentRequestSpec
# ---------------------------------------------------------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -626,8 +626,7 @@ spec:
image:
description: |-
Image is the container image reference for the profiling job (frontend image).
Example: "nvcr.io/nvidia/dynamo-runtime:latest"
backend type automatically; backend images can be overridden via overrides.dgd.
Example: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.0".
type: string
model:
description: |-
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -381,9 +381,7 @@ type DynamoGraphDeploymentRequestSpec struct {
Backend BackendType `json:"backend,omitempty"`

// Image is the container image reference for the profiling job (frontend image).
// Example: "nvcr.io/nvidia/dynamo-runtime:latest"
// TODO: In a future MR, the operator will derive the backend inference image from the
// backend type automatically; backend images can be overridden via overrides.dgd.
// Example: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.0".
// +optional
Image string `json:"image,omitempty"`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -626,8 +626,7 @@ spec:
image:
description: |-
Image is the container image reference for the profiling job (frontend image).
Example: "nvcr.io/nvidia/dynamo-runtime:latest"
backend type automatically; backend images can be overridden via overrides.dgd.
Example: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.0".
type: string
model:
description: |-
Expand Down
2 changes: 1 addition & 1 deletion docs/pages/kubernetes/api-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -1326,7 +1326,7 @@ _Appears in:_
| --- | --- | --- | --- |
| `model` _string_ | Model specifies the model to deploy (e.g., "Qwen/Qwen3-0.6B", "meta-llama/Llama-3-70b").<br />Can be a HuggingFace ID or a private model name. | | MinLength: 1 <br />Required: \{\} <br /> |
| `backend` _[BackendType](#backendtype)_ | Backend specifies the inference backend to use for profiling and deployment. | auto | Enum: [auto sglang trtllm vllm] <br />Optional: \{\} <br /> |
| `image` _string_ | Image is the container image reference for the profiling job (frontend image).<br />Example: "nvcr.io/nvidia/dynamo-runtime:latest"<br />backend type automatically; backend images can be overridden via overrides.dgd. | | Optional: \{\} <br /> |
| `image` _string_ | Image is the container image reference for the profiling job (frontend image).<br />Example: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.0.0". | | Optional: \{\} <br /> |
| `modelCache` _[ModelCacheSpec](#modelcachespec)_ | ModelCache provides optional PVC configuration for pre-downloaded model weights.<br />When provided, weights are loaded from the PVC instead of downloading from HuggingFace. | | Optional: \{\} <br /> |
| `hardware` _[HardwareSpec](#hardwarespec)_ | Hardware describes the hardware resources available for profiling and deployment.<br />Typically auto-filled by the operator from cluster discovery. | | Optional: \{\} <br /> |
| `workload` _[WorkloadSpec](#workloadspec)_ | Workload defines the expected workload characteristics for SLA-based profiling. | | Optional: \{\} <br /> |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 1: AIC supported model, rapid, without planner, no input load
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 2: AIC supported model, rapid, without planner, input load (request rate)
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 64
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 2b: AIC supported model, rapid, without planner, input load, with PVC model cache
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 64
Expand Down
2 changes: 1 addition & 1 deletion tests/profiler/configs/2c_rapid_supported_e2e_latency.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 2c: AIC supported model, rapid, without planner, e2eLatency instead of ttft/itl
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 64
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 2d: Both concurrency and requestRate specified — should fail validation
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 64
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 3: AIC supported model, rapid, with planner, rapid pre-deployment sweeping
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 3b: AIC supported model, rapid, with planner, rapid pre-deployment sweeping, enable mocker
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 8
Expand Down
2 changes: 1 addition & 1 deletion tests/profiler/configs/4_rapid_unsupported_no_planner.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# l40s + vllm has no disagg support in AIC
model: "Qwen/Qwen3-32B"
backend: vllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: l40s
totalGpus: 4
Expand Down
2 changes: 1 addition & 1 deletion tests/profiler/configs/5_rapid_unsupported_planner.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# This config uses load-based scaling only to test the non-error path.
model: "Qwen/Qwen3-32B"
backend: vllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: l40s
totalGpus: 4
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# requires AIC support.
model: "Qwen/Qwen3-32B"
backend: vllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: l40s
totalGpus: 4
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# Case 6: Thorough, without planner, input load (test with --dry-run)
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 64
Expand Down
2 changes: 1 addition & 1 deletion tests/profiler/configs/7_thorough_planner_rapid_sweep.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# This uses a supported combo so rapid sweeping should work.
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# Both the config picking and the interpolation use real GPUs (mocked in tests).
model: "Qwen/Qwen3-32B"
backend: trtllm
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: h200_sxm
totalGpus: 8
Expand Down
2 changes: 1 addition & 1 deletion tests/profiler/configs/8_thorough_empty_candidates.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# enumerate_profiling_configs should return empty lists.
model: "deepseek-ai/DeepSeek-R1"
backend: sglang
image: "nvcr.io/nvidia/dynamo:latest"
image: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:latest"
hardware:
gpuSku: l40s
totalGpus: 1
Expand Down
Loading