Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
479 changes: 291 additions & 188 deletions components/src/dynamo/profiler/profile_sla.py

Large diffs are not rendered by default.

215 changes: 137 additions & 78 deletions components/src/dynamo/profiler/rapid.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,45 @@ def _generate_dgd_from_pick(
return None


def run_rapid(
def _run_naive_fallback(
dgdr: DynamoGraphDeploymentRequestSpec,
model: str,
total_gpus: int,
system: str,
backend: str,
) -> dict:
"""Handle the AIC-unsupported path via naive config generation."""
logger.info(
"AIC does not support this combo — falling back to naive config generation."
)
naive_result = generate_naive_config(model, total_gpus, system, backend)

dgd_yaml = naive_result.get("artifacts", {}).get("k8s_deploy.yaml", "")
dgd_config = yaml.safe_load(dgd_yaml) if dgd_yaml else None
if dgd_config:
config_modifier = CONFIG_MODIFIERS[backend]
dgd_config = config_modifier.update_image(
dgd_config, derive_backend_image(dgdr.image, backend)
)
if dgdr.modelCache and dgdr.modelCache.pvcName:
dgd_config = config_modifier.update_model_from_pvc(
dgd_config,
model_name=model,
pvc_name=dgdr.modelCache.pvcName,
pvc_mount_path=dgdr.modelCache.pvcMountPath,
pvc_path=dgdr.modelCache.pvcModelPath or "",
)

return {
"best_config_df": pd.DataFrame(),
"best_latencies": {"ttft": 0.0, "tpot": 0.0, "request_latency": 0.0},
"dgd_config": dgd_config,
"chosen_exp": None,
}


def _run_autoscale_sim(
dgdr: DynamoGraphDeploymentRequestSpec,
picking_mode: str,
aic_supported: bool,
model: str,
system: str,
backend: str,
Expand All @@ -101,82 +136,60 @@ def run_rapid(
target_tpot: float,
request_latency: float | None,
) -> dict:
"""Run AIC simulation and picking. Returns a result dict with
``best_config_df``, ``best_latencies``, and ``dgd_config``.
"""

# --- Stage 1: AIC support check ---
if not aic_supported:
logger.info(
"AIC does not support this combo — falling back to naive config generation."
"""Build a TaskConfig, run autoscale simulation, collect latencies, generate DGD."""
planner_cfg = dgdr.features.planner if dgdr.features else None
if planner_cfg and planner_cfg.enable_throughput_scaling:
logger.warning(
"Throughput-based scaling enabled — only disagg mode is supported."
)
naive_result = generate_naive_config(model, total_gpus, system, backend)

dgd_yaml = naive_result.get("artifacts", {}).get("k8s_deploy.yaml", "")
dgd_config = yaml.safe_load(dgd_yaml) if dgd_yaml else None
if dgd_config:
config_modifier = CONFIG_MODIFIERS[backend]
dgd_config = config_modifier.update_image(
dgd_config, derive_backend_image(dgdr.image, backend)
)
if dgdr.modelCache and dgdr.modelCache.pvcName:
dgd_config = config_modifier.update_model_from_pvc(
dgd_config,
model_name=model,
pvc_name=dgdr.modelCache.pvcName,
pvc_mount_path=dgdr.modelCache.pvcMountPath or "/opt/model-cache",
pvc_path=dgdr.modelCache.pvcModelPath or "",
)

return {
"best_config_df": pd.DataFrame(),
"best_latencies": {"ttft": 0.0, "tpot": 0.0, "request_latency": 0.0},
"dgd_config": dgd_config,
"chosen_exp": None,
}

# --- Stage 2: Simulation + Picking ---
if picking_mode == "autoscale":
planner_cfg = dgdr.features.planner if dgdr.features else None
if planner_cfg and planner_cfg.enable_throughput_scaling:
logger.warning(
"Throughput-based scaling enabled — only disagg mode is supported."
)

task = TaskConfig(
serving_mode="disagg",
model_path=model,
system_name=system,
backend_name=backend,
total_gpus=total_gpus,
isl=isl,
osl=osl,
ttft=target_ttft,
tpot=target_tpot,
request_latency=request_latency,
)
runner = TaskRunner()
sim_result = runner.run(task, autoscale=True)
pareto_df = sim_result.get("pareto_df", pd.DataFrame())
best_latencies = {"ttft": 0.0, "tpot": 0.0, "request_latency": 0.0}
if pareto_df is not None and not pareto_df.empty:
row = pareto_df.iloc[0]
best_latencies["ttft"] = float(row.get("ttft", 0.0))
best_latencies["tpot"] = float(row.get("tpot", 0.0))
best_latencies["request_latency"] = float(row.get("request_latency", 0.0))

# --- Stage 3: DGD generation ---
task_configs = {"disagg": task}
dgd_config = _generate_dgd_from_pick(dgdr, pareto_df, "disagg", task_configs)
return {
"best_config_df": pareto_df,
"best_latencies": best_latencies,
"dgd_config": dgd_config,
"chosen_exp": "disagg",
"task_configs": task_configs,
}

# default or load_match: simulation sweeps both agg + disagg
task = TaskConfig(
serving_mode="disagg",
model_path=model,
system_name=system,
backend_name=backend,
total_gpus=total_gpus,
isl=isl,
osl=osl,
ttft=target_ttft,
tpot=target_tpot,
request_latency=request_latency,
)
runner = TaskRunner()
sim_result = runner.run(task, autoscale=True)
pareto_df = sim_result.get("pareto_df", pd.DataFrame())
best_latencies = {"ttft": 0.0, "tpot": 0.0, "request_latency": 0.0}
if pareto_df is not None and not pareto_df.empty:
row = pareto_df.iloc[0]
best_latencies["ttft"] = float(row.get("ttft", 0.0))
best_latencies["tpot"] = float(row.get("tpot", 0.0))
best_latencies["request_latency"] = float(row.get("request_latency", 0.0))

task_configs = {"disagg": task}
dgd_config = _generate_dgd_from_pick(dgdr, pareto_df, "disagg", task_configs)
return {
"best_config_df": pareto_df,
"best_latencies": best_latencies,
"dgd_config": dgd_config,
"chosen_exp": "disagg",
"task_configs": task_configs,
}


def _run_default_sim(
dgdr: DynamoGraphDeploymentRequestSpec,
model: str,
system: str,
backend: str,
total_gpus: int,
isl: int,
osl: int,
target_ttft: float,
target_tpot: float,
request_latency: float | None,
picking_mode: str,
) -> dict:
"""Build default task_configs, apply load_match kwargs, run simulation, generate DGD."""
task_configs = build_default_task_configs(
model_path=model,
total_gpus=total_gpus,
Expand Down Expand Up @@ -207,7 +220,6 @@ def run_rapid(
chosen, {"ttft": 0.0, "tpot": 0.0, "request_latency": 0.0}
)

# --- Stage 3: DGD generation ---
dgd_config = _generate_dgd_from_pick(dgdr, best_config_df, chosen, task_configs)

return {
Expand All @@ -217,3 +229,50 @@ def run_rapid(
"chosen_exp": chosen,
"task_configs": task_configs,
}


def run_rapid(
dgdr: DynamoGraphDeploymentRequestSpec,
picking_mode: str,
aic_supported: bool,
model: str,
system: str,
backend: str,
total_gpus: int,
isl: int,
osl: int,
target_ttft: float,
target_tpot: float,
request_latency: float | None,
) -> dict:
"""Run AIC simulation and picking. Returns a result dict with
``best_config_df``, ``best_latencies``, and ``dgd_config``.
"""
if not aic_supported:
return _run_naive_fallback(dgdr, model, total_gpus, system, backend)
if picking_mode == "autoscale":
return _run_autoscale_sim(
dgdr,
model,
system,
backend,
total_gpus,
isl,
osl,
target_ttft,
target_tpot,
request_latency,
)
return _run_default_sim(
dgdr,
model,
system,
backend,
total_gpus,
isl,
osl,
target_ttft,
target_tpot,
request_latency,
picking_mode,
)
Loading
Loading