From 4cb33d13fb5c6ffbf52698532142255cc4f44df9 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 14 Sep 2026 21:27:00 +0800 Subject: [PATCH 01/27] feat: support local systems roots in predict and recommend Carry engine.systems_path through lookup, simulation, recommendation and exported configs without changing process-wide defaults. Reject modes whose consumers cannot use the root. Refs: AIC-1963 Signed-off-by: Yiming Liu --- python/aisimulate/src/aisimulate/compiler.py | 3 + .../src/aisimulate/config/common.py | 11 +- .../src/aisimulate/config/engine.py | 15 +- python/aisimulate/src/aisimulate/recommend.py | 3 + .../src/aisimulate/sweeper/config.py | 8 + .../src/aisimulate/sweeper/deploy.py | 4 + .../src/aisimulate/sweeper/kv_estimate.py | 10 +- .../src/aisimulate/sweeper/kv_load.py | 3 + .../src/aisimulate/sweeper/model_hw.py | 10 +- .../src/aisimulate/sweeper/sample.py | 2 + .../src/aisimulate/sweeper/search.py | 14 +- .../src/aisimulate/sweeper/search_space.py | 7 +- .../tests/unit/test_systems_path.py | 299 ++++++++++++++++++ tests/sweeper/test_epd.py | 6 +- tests/sweeper/test_model_hw.py | 2 +- tests/sweeper/test_result.py | 10 +- tests/sweeper/test_search.py | 6 +- tests/sweeper/test_search_providers.py | 4 +- tests/sweeper/test_search_space.py | 10 +- tests/sweeper/test_unified_optimizer.py | 10 +- tests/test_epd_cli.py | 4 +- 21 files changed, 406 insertions(+), 35 deletions(-) create mode 100644 python/aisimulate/tests/unit/test_systems_path.py diff --git a/python/aisimulate/src/aisimulate/compiler.py b/python/aisimulate/src/aisimulate/compiler.py index cd3f9cf7b9..82a351d4c1 100644 --- a/python/aisimulate/src/aisimulate/compiler.py +++ b/python/aisimulate/src/aisimulate/compiler.py @@ -285,6 +285,7 @@ def _worker_performance_model_metadata( "moe_ep_size": parallel.moe_expert if sharded_moe else None, "nextn": None, "forward_model": worker.timing.forward_model, + **({"systems_path": engine.systems_path} if engine.systems_path is not None else {}), }, } @@ -323,6 +324,8 @@ def _worker_engine_args( } if engine.backend_version is not None: payload["aic_backend_version"] = engine.backend_version + if engine.systems_path is not None: + payload["systems_path"] = engine.systems_path if parallel.pipeline != 1: payload["aic_pp_size"] = parallel.pipeline if parallel.moe_tensor * parallel.moe_expert > 1: diff --git a/python/aisimulate/src/aisimulate/config/common.py b/python/aisimulate/src/aisimulate/config/common.py index c57b73bf82..e86e174bac 100644 --- a/python/aisimulate/src/aisimulate/config/common.py +++ b/python/aisimulate/src/aisimulate/config/common.py @@ -11,13 +11,22 @@ from typing import Annotated, Any, Generic, Literal, TypeVar import yaml -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator class StrictModel(BaseModel): model_config = ConfigDict(extra="forbid") +def _absolute_systems_path(value: str) -> str: + if not value.strip() or "," in value: + raise ValueError("systems_path must name one nonempty local directory, not a comma-separated list") + return str(Path(value).expanduser().resolve()) + + +SystemsPath = Annotated[str, Field(strict=True, min_length=1), AfterValidator(_absolute_systems_path)] + + T = TypeVar("T") PositiveFiniteFloat = Annotated[float, Field(strict=True, gt=0, allow_inf_nan=False)] PositiveStrictInt = Annotated[int, Field(strict=True, gt=0)] diff --git a/python/aisimulate/src/aisimulate/config/engine.py b/python/aisimulate/src/aisimulate/config/engine.py index 4ba44811e0..a8b774f96d 100644 --- a/python/aisimulate/src/aisimulate/config/engine.py +++ b/python/aisimulate/src/aisimulate/config/engine.py @@ -9,7 +9,7 @@ from pydantic import Field, field_validator, model_validator -from .common import Choices, IntegerRange, NumericRange, StrictModel +from .common import Choices, IntegerRange, NumericRange, StrictModel, SystemsPath PositiveInt = Annotated[int, Field(strict=True, gt=0)] CudaGraphReservedBytes = Annotated[int, Field(strict=True, ge=0, le=1 << 53)] @@ -185,6 +185,7 @@ class EnginePredictionConfig(StrictModel): hardware: str backend: Backend = "vllm" backend_version: str | None = None + systems_path: SystemsPath | None = None context_length: PositiveInt | Literal["max"] = "max" workers: WorkersPredictionConfig = Field(default_factory=WorkersPredictionConfig) kv_transfer: KvTransferConfig | None = None @@ -206,6 +207,7 @@ def _reject_auto_hardware(cls, value: str) -> str: @model_validator(mode="after") def _validate_roles(self) -> EnginePredictionConfig: + _validate_systems_path_modes(self, {self.mode}) if self.mode == "afd": _validate_prediction_afd(self) else: @@ -357,6 +359,7 @@ class EngineRecommendationConfig(StrictModel): hardware: str backend: Backend | Choices[Backend] = Field(default_factory=lambda: Choices[Backend](choices=["vllm", "sglang"])) backend_version: str | None = None + systems_path: SystemsPath | None = None context_length: PositiveInt | Literal["max"] = "max" workers: WorkersRecommendationConfig = Field(default_factory=WorkersRecommendationConfig) kv_transfer: KvTransferConfig | None = None @@ -372,6 +375,7 @@ def _validate_nonempty(cls, value: str) -> str: @model_validator(mode="after") def _validate_roles(self) -> EngineRecommendationConfig: modes = set(self.mode.choices) if isinstance(self.mode, Choices) else {self.mode} + _validate_systems_path_modes(self, modes) if "afd" in modes: if modes != {"afd"}: raise ValueError("AFD recommendation mode cannot be mixed with aggregated/disaggregated modes") @@ -390,6 +394,15 @@ def _validate_roles(self) -> EngineRecommendationConfig: return self +def _validate_systems_path_modes(engine, modes: set[str]) -> None: + if engine.systems_path is None: + return + if "afd" in modes: + raise ValueError("engine.systems_path does not support AFD") + if engine.workers.encoder is not None: + raise ValueError("engine.systems_path does not support analytical encoder pools") + + def _workers_with_host_offload(workers) -> list[tuple[str, Any]]: return [ (role, worker) diff --git a/python/aisimulate/src/aisimulate/recommend.py b/python/aisimulate/src/aisimulate/recommend.py index 59102ddf3d..31be1023c1 100644 --- a/python/aisimulate/src/aisimulate/recommend.py +++ b/python/aisimulate/src/aisimulate/recommend.py @@ -112,6 +112,7 @@ def recommendation_to_sweeper( "deployment_mode": modes, "backend": [str(value) for value in backend_values], "backend_version": engine.get("backend_version"), + "systems_path": engine.get("systems_path"), "model_name": model, "hardware_sku": hardware, "gpu_budget": optimization.constraints.max_candidate_gpus, @@ -705,6 +706,8 @@ def _candidate_prediction( "context_length": sample.get("context_length") or "max", "workers": {}, } + if sample.get("systems_path") is not None: + engine["systems_path"] = sample["systems_path"] raw_engine = source.engine.model_dump(mode="python", exclude_none=True) if deployment.encoder is not None: from .config.epd import encoder_prediction_fields diff --git a/python/aisimulate/src/aisimulate/sweeper/config.py b/python/aisimulate/src/aisimulate/sweeper/config.py index 0942be8326..b92682614b 100644 --- a/python/aisimulate/src/aisimulate/sweeper/config.py +++ b/python/aisimulate/src/aisimulate/sweeper/config.py @@ -27,6 +27,8 @@ import yaml from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from ..config.common import SystemsPath + class OptimizationTarget(str, Enum): """What the search optimizes for. @@ -482,6 +484,7 @@ class SearchSpace(BaseModel): # pinned model_name: str # HF id or private model name hardware_sku: str # e.g. "h200_sxm" + systems_path: SystemsPath | None = None prefill_hardware_sku: str | None = Field(default=None, min_length=1) decode_hardware_sku: str | None = Field(default=None, min_length=1) gpu_budget: int = 32 # max GPUs per candidate @@ -558,6 +561,11 @@ class SearchSpace(BaseModel): @model_validator(mode="after") def _validate_search_choices(self) -> SearchSpace: """Every backend dimension is a non-empty subset of its allowed choices.""" + if self.systems_path is not None: + if any(mode in {"afd", "afd+pd"} for mode in self.deployment_mode): + raise ValueError("systems_path does not support AFD") + if self.encoder is not None: + raise ValueError("systems_path does not support analytical encoder pools") for field_name, allowed in SEARCH_CHOICES.items(): values = getattr(self, field_name) if not values: diff --git a/python/aisimulate/src/aisimulate/sweeper/deploy.py b/python/aisimulate/src/aisimulate/sweeper/deploy.py index 74f82a2ebd..30edbfbe1a 100644 --- a/python/aisimulate/src/aisimulate/sweeper/deploy.py +++ b/python/aisimulate/src/aisimulate/sweeper/deploy.py @@ -44,6 +44,8 @@ def _performance_model_metadata(sample: dict[str, Any], role: str, *, backend_ve else "op_level" ), } + if sample.get("systems_path") is not None: + config["systems_path"] = sample["systems_path"] return {"provider": "aic", "config": config} @@ -81,6 +83,8 @@ def _engine_args_payload(sample: dict[str, Any], role: str, *, backend_version: memory_fraction_field: float(memory_fraction), "enable_prefix_caching": bool(sample[f"{role}_enable_prefix_caching"]), } + if sample.get("systems_path") is not None: + payload["systems_path"] = sample["systems_path"] if backend == "vllm" and sample.get("context_length") is not None: payload["max_model_len"] = int(sample["context_length"]) if moe_tp * moe_ep > 1: diff --git a/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py b/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py index 878d6a4079..4154936bb1 100644 --- a/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py +++ b/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py @@ -46,9 +46,9 @@ def memory_fraction_kind(backend: str) -> str: return "of_free" if backend == "trtllm" else "of_total" -def resolve_backend_version(hardware_sku: str, backend: str) -> str: +def resolve_backend_version(hardware_sku: str, backend: str, *, systems_path: str | None = None) -> str: """Latest perf-DB version for the SKU/backend (required by the native estimate).""" - version = get_latest_database_version(hardware_sku, backend) + version = get_latest_database_version(hardware_sku, backend, systems_paths=systems_path) if version is None: raise NoPerfDatabase( f"no perf database for hardware_sku={hardware_sku!r}, backend={backend!r}; " @@ -68,6 +68,7 @@ def estimate_kv_tokens( max_batch_size: int = DEFAULT_MAX_BATCH_SIZE, memory_fraction: float = DEFAULT_MEMORY_FRACTION, nextn: int = 0, + systems_path: str | None = None, ) -> int | None: """Per-rank KV-cache capacity (in tokens) for ``shape``, or ``None`` when the shape leaves no KV budget (weights + activations already fill VRAM -> OOM). @@ -90,6 +91,7 @@ def estimate_kv_tokens( moe_tp_size=shape.moe_tp, moe_ep_size=shape.moe_ep, nextn=nextn, + systems_path=systems_path, allow_naive_fallback=False, ) except ValueError as exc: @@ -117,6 +119,7 @@ def feasible_shape_tokens( max_num_tokens: int = DEFAULT_MAX_NUM_TOKENS, max_batch_size: int = DEFAULT_MAX_BATCH_SIZE, memory_fraction: float = DEFAULT_MEMORY_FRACTION, + systems_path: str | None = None, ) -> dict[ParallelShape, int]: """Map each *feasible* shape to its KV-cache token capacity. @@ -125,7 +128,7 @@ def feasible_shape_tokens( per distinct shape, so repeated shapes across replica counts are free. """ if backend_version is None: - backend_version = resolve_backend_version(hardware_sku, backend) + backend_version = resolve_backend_version(hardware_sku, backend, systems_path=systems_path) feasible: dict[ParallelShape, int] = {} for shape in dict.fromkeys(shapes): # dedup, preserve first-seen order tokens = estimate_kv_tokens( @@ -137,6 +140,7 @@ def feasible_shape_tokens( max_num_tokens=max_num_tokens, max_batch_size=max_batch_size, memory_fraction=memory_fraction, + systems_path=systems_path, ) if tokens is not None and tokens > max_seq_len: feasible[shape] = tokens diff --git a/python/aisimulate/src/aisimulate/sweeper/kv_load.py b/python/aisimulate/src/aisimulate/sweeper/kv_load.py index 6a6a8ddce9..bbdfbf1954 100644 --- a/python/aisimulate/src/aisimulate/sweeper/kv_load.py +++ b/python/aisimulate/src/aisimulate/sweeper/kv_load.py @@ -40,6 +40,7 @@ def _per_rank_capacity_tokens( max_batch_size: int, memory_fraction: float, nextn: int, + systems_path: str | None = None, ) -> int: tokens = estimate_kv_tokens( shape, @@ -51,6 +52,7 @@ def _per_rank_capacity_tokens( max_batch_size=max_batch_size, memory_fraction=memory_fraction, nextn=nextn, + systems_path=systems_path, ) if tokens is None: raise InfeasibleKVCapacity( @@ -85,6 +87,7 @@ def _role_capacity_tokens( max_batch_size=int(sample[f"{role}_max_num_seqs"]), memory_fraction=float(sample[f"{role}_gpu_memory_utilization"]), nextn=int(sample.get("aic_nextn") or 0), + systems_path=sample.get("systems_path"), ) # Dynamo's AIC estimator returns per-rank blocks. Offline replay models one # engine-wide KV pool, so attention-DP ranks contribute independent capacity; diff --git a/python/aisimulate/src/aisimulate/sweeper/model_hw.py b/python/aisimulate/src/aisimulate/sweeper/model_hw.py index 72672267b4..064aa43c04 100644 --- a/python/aisimulate/src/aisimulate/sweeper/model_hw.py +++ b/python/aisimulate/src/aisimulate/sweeper/model_hw.py @@ -65,7 +65,9 @@ class ModelHardware: num_experts: int = 0 -def resolve_model_hardware(model_name: str, hardware_sku: str, *, backend: str) -> ModelHardware: +def resolve_model_hardware( + model_name: str, hardware_sku: str, *, backend: str, systems_path: str | None = None +) -> ModelHardware: """Read the model weights + SKU spec (via AIC) to derive is_moe / mla / wideep and the model's max context length.""" model_config = get_model_config_from_model_path(model_name) @@ -76,7 +78,7 @@ def resolve_model_hardware(model_name: str, hardware_sku: str, *, backend: str) max_context = model_config.get("context") num_experts = int(model_config.get("num_experts") or model_config.get("n_routed_experts") or 0) - system_spec = perf_database.load_system_spec(hardware_sku) + system_spec = perf_database.load_system_spec(hardware_sku, systems_paths=systems_path) if not system_spec: raise ValueError( f"unknown hardware_sku {hardware_sku!r}: no system config found on AIConfigurator Core's systems path" @@ -117,6 +119,7 @@ def parallel_configs_for( max_batch_size: int = DEFAULT_MAX_BATCH_SIZE, memory_fraction: float = DEFAULT_MEMORY_FRACTION, role_runtime: dict[str, tuple[int, int, float] | tuple[int, int, float, int | None]] | None = None, + systems_path: str | None = None, ) -> list[ReplicaParallelConfig] | list[DisaggParallelConfig]: """Resolve the model/hardware, then enumerate the parallel configs that fit the GPU budget and can hold a ``max_seq_len``-token sequence. @@ -136,7 +139,7 @@ def parallel_configs_for( :class:`NoViableParallelConfig` when no shape can hold the sequence within the budget. """ - mh = resolve_model_hardware(model_name, hardware_sku, backend=backend) + mh = resolve_model_hardware(model_name, hardware_sku, backend=backend, systems_path=systems_path) seq_len = max_seq_len if max_seq_len is not None else mh.max_context if seq_len is None: raise ValueError(f"max_seq_len is required: {model_name} config exposes no max context length") @@ -184,6 +187,7 @@ def feasible_for(role: str, shapes): max_num_tokens=role_tokens, max_batch_size=role_batch, memory_fraction=role_memory, + systems_path=systems_path, ) if deployment_mode == "agg": diff --git a/python/aisimulate/src/aisimulate/sweeper/sample.py b/python/aisimulate/src/aisimulate/sweeper/sample.py index efa8fdd4ae..157354caa2 100644 --- a/python/aisimulate/src/aisimulate/sweeper/sample.py +++ b/python/aisimulate/src/aisimulate/sweeper/sample.py @@ -130,6 +130,8 @@ def unroll_sample( for key in _DEPLOYMENT_PINNED: sample[key] = getattr(search_space, key) + if search_space.systems_path is not None: + sample["systems_path"] = search_space.systems_path if mode == "disagg": sample["prefill_hardware_sku"] = search_space.hardware_sku_for("prefill") diff --git a/python/aisimulate/src/aisimulate/sweeper/search.py b/python/aisimulate/src/aisimulate/sweeper/search.py index d9721f724c..7e9399dae4 100644 --- a/python/aisimulate/src/aisimulate/sweeper/search.py +++ b/python/aisimulate/src/aisimulate/sweeper/search.py @@ -630,10 +630,14 @@ def _materialize_one( if sample["deployment_mode"] == "disagg": role_hardware = {role: sample[f"{role}_hardware_sku"] for role in ("prefill", "decode")} if len(set(role_hardware.values())) == 1: - backend_version = resolve_backend_version(role_hardware["prefill"], selection["backend"]) + backend_version = resolve_backend_version( + role_hardware["prefill"], selection["backend"], systems_path=config.search_space.systems_path + ) else: role_versions = { - role: resolve_backend_version(hardware, selection["backend"]) + role: resolve_backend_version( + hardware, selection["backend"], systems_path=config.search_space.systems_path + ) for role, hardware in role_hardware.items() } if len(set(role_versions.values())) != 1: @@ -644,7 +648,11 @@ def _materialize_one( ) backend_version = role_versions["prefill"] else: - backend_version = resolve_backend_version(config.search_space.hardware_sku, selection["backend"]) + backend_version = resolve_backend_version( + config.search_space.hardware_sku, + selection["backend"], + systems_path=config.search_space.systems_path, + ) # The resolved perf-model version is part of the evaluated contract. Keep it # on the candidate so downstream artifact generation cannot independently # select a different backend version. diff --git a/python/aisimulate/src/aisimulate/sweeper/search_space.py b/python/aisimulate/src/aisimulate/sweeper/search_space.py index 46f26a4b52..3a34eeb18e 100644 --- a/python/aisimulate/src/aisimulate/sweeper/search_space.py +++ b/python/aisimulate/src/aisimulate/sweeper/search_space.py @@ -246,7 +246,10 @@ def _heterogeneous_disagg_configs( role_hardware = {role: search_space.hardware_sku_for(role) for role in ("prefill", "decode")} backend_version = search_space.backend_version if backend_version is None: - role_versions = {role: resolve_backend_version(hardware, backend) for role, hardware in role_hardware.items()} + role_versions = { + role: resolve_backend_version(hardware, backend, systems_path=search_space.systems_path) + for role, hardware in role_hardware.items() + } if len(set(role_versions.values())) != 1: raise NoPerfDatabase( "heterogeneous P/D hardware requires one common backend_version; " @@ -268,6 +271,7 @@ def _heterogeneous_disagg_configs( min_gpu_budget=None, max_seq_len=max_seq_len, role_runtime={"agg": _role_runtime(search_space, backend, role)}, + systems_path=search_space.systems_path, ) except (NoPerfDatabase, NoViableParallelConfig) as exc: raise type(exc)(f"{role} hardware_sku={hardware!r}: {exc}") from exc @@ -591,6 +595,7 @@ def matches_custom(config: _ParallelConfig) -> bool: min_gpu_budget=ss.min_gpu_budget, max_seq_len=max_seq_len, role_runtime=_runtime_by_role(ss, backend, deployment_mode), + systems_path=ss.systems_path, ) except (NoPerfDatabase, NoViableParallelConfig): continue # backend unusable for this mode -> drop it from the search diff --git a/python/aisimulate/tests/unit/test_systems_path.py b/python/aisimulate/tests/unit/test_systems_path.py new file mode 100644 index 0000000000..84f9d84706 --- /dev/null +++ b/python/aisimulate/tests/unit/test_systems_path.py @@ -0,0 +1,299 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Public prediction and recommendation use an isolated local systems root.""" + +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from copy import deepcopy +from importlib.resources import files + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import yaml +from pydantic import ValidationError + +from aiconfigurator_core.sdk import models, perf_database +from aiconfigurator_core.sdk.config import ModelConfig +from aiconfigurator_core.sdk.operations.fpm_forward import _CELL_MATCH_COLUMNS +from aisimulate.compiler import prediction_to_replay_spec +from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig +from aisimulate.output import write_recommendations +from aisimulate.recommend import run_recommendation +from aisimulate.runner import EngineReplayRunnerFactory + +pytestmark = pytest.mark.unit + +_SYSTEM = "local_fpm_test" +_VERSION = "0.0.1" + + +def _request() -> dict: + return { + "engine": { + "mode": "aggregated", + "model": "Qwen/Qwen3-0.6B", + "hardware": "h200_sxm", + "backend": "vllm", + "context_length": 128, + "workers": {"aggregated": {}}, + }, + } + + +@pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) +def test_systems_root_is_absolute_and_reloadable_before_collection(config_type, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + raw = _request() + raw["engine"]["systems_path"] = "local profiles" + if config_type is CoreRecommendationConfig: + raw["optimization"] = {"target": "throughput"} + + config = config_type.model_validate(raw) + saved = config.model_dump(mode="json", exclude_none=True) + monkeypatch.chdir(tmp_path.parent) + + assert saved["engine"]["systems_path"] == str(tmp_path / "local profiles") + assert config_type.model_validate(saved).engine.systems_path == config.engine.systems_path + + +@pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) +@pytest.mark.parametrize("value", ["", " ", 7, ["one"], {"choices": ["one"]}, "one,two"]) +def test_systems_root_rejects_empty_or_multiple_roots(config_type, value): + raw = _request() + raw["engine"]["systems_path"] = value + if config_type is CoreRecommendationConfig: + raw["optimization"] = {"target": "throughput"} + + with pytest.raises(ValidationError, match="systems_path"): + config_type.model_validate(raw) + + +@pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) +@pytest.mark.parametrize("unsupported", ["AFD", "analytical encoder pools"]) +def test_systems_root_rejects_modes_that_cannot_consume_it(config_type, unsupported, tmp_path): + raw = _request() + if unsupported == "AFD": + raw["engine"]["mode"] = "afd" + raw["engine"]["workers"] = {} + raw["engine"]["afd"] = {"phase": "both", "combined_with_pd": False, "a_batch_size": 1} + if config_type is CorePredictionConfig: + raw["engine"]["afd"].update(n_a_nodes=1, n_f_nodes=1, tp_a=1) + else: + raw["engine"]["workers"]["encoder"] = {} + raw["traffic"] = { + "source": {"type": "synthetic", "images": {"height": 32, "width": 32}}, + "load": {"type": "concurrency", "concurrency": 1}, + "stop": {"requests": 1}, + } + if config_type is CoreRecommendationConfig: + raw["optimization"] = {"target": "throughput"} + + config_type.model_validate(raw) + raw["engine"]["systems_path"] = str(tmp_path) + with pytest.raises(ValidationError, match=f"systems_path.*{unsupported}"): + config_type.model_validate(raw) + + +@pytest.fixture +def local_profiles(tmp_path): + """Two roots share an identity but have distinct timings and GPU memory.""" + model = models.get_model("Qwen/Qwen3-0.6B", ModelConfig(forward_model="fpm"), "vllm") + identity = dict(zip(_CELL_MATCH_COLUMNS, model.context_ops[0]._match_identity, strict=True)) + for name in ("tp", "pp", "dp", "moe_tp", "moe_ep", "cp"): + identity[name] = int(identity[name]) + for name in ("enable_wideep", "enable_eplb"): + identity[name] = identity[name] == "True" + roots = [] + for label, latency, memory in (("fast", 20.0, 16 << 30), ("slow", 200.0, 32 << 30)): + root = tmp_path / label + data = root / "data" / _SYSTEM / "vllm" / _VERSION + data.mkdir(parents=True) + system = yaml.safe_load((files("aiconfigurator_core") / "systems/h200_sxm.yaml").read_text()) + system["data_dir"] = f"data/{_SYSTEM}" + system["gpu"]["mem_capacity"] = memory + (root / f"{_SYSTEM}.yaml").write_text(yaml.safe_dump(system)) + rows = [] + for phase, tokens, kv in (("prefill", 16, 0), ("decode", 0, 0), ("decode", 0, 128)): + rows.append( + identity + | { + "cell_id": phase, + "model_path": model.model_path, + "system": _SYSTEM, + "backend": "vllm", + "backend_version": _VERSION, + "weight_quantization": identity["gemm_quant_mode"], + "workload_kind": phase, + "batch_size": 1, + "total_prefill_tokens": tokens, + "total_kv_read_tokens": kv, + "partition_policy": "balanced_v1", + "latency_ms": latency, + } + ) + parquet = data / "fpm_forward_perf.parquet" + pq.write_table(pa.Table.from_pylist(rows), parquet) + (data / "fpm_forward_perf.metadata.json").write_text( + json.dumps( + { + "schema_name": "aic_fpm_forward_perf", + "schema_version": 6, + "coordinate_system": "iteration_totals_balanced_v1", + "measurement_policy": "dynamo_native_single_sample_v1", + "row_count": len(rows), + "parquet_sha256": hashlib.sha256(parquet.read_bytes()).hexdigest(), + "system": _SYSTEM, + "backend": "vllm", + "backend_version": _VERSION, + } + ) + ) + roots.append(root) + return roots + + +def _local_request(root, mode="aggregated") -> dict: + raw = _request() + raw["engine"].update(mode=mode, hardware=_SYSTEM, systems_path=str(root), backend_version=_VERSION) + worker = { + "scheduler": {"max_batched_tokens": 16, "max_sequences": 1}, + "kv_cache": {"prefix_caching": False}, + "timing": {"forward_model": "fpm"}, + } + roles = ("aggregated",) if mode == "aggregated" else ("prefill", "decode") + raw["engine"]["workers"] = {role: deepcopy(worker) for role in roles} + raw["traffic"] = { + "source": {"type": "synthetic", "input_tokens": 16, "output_tokens": 2}, + "load": {"type": "concurrency", "concurrency": 1}, + "stop": {"requests": 1}, + } + return raw + + +def _predict(raw): + spec = prediction_to_replay_spec(CorePredictionConfig.model_validate(raw)) + runner = EngineReplayRunnerFactory().create(0) + try: + return runner.run(spec) + finally: + runner.close() + + +@pytest.mark.parametrize("mode", ["aggregated", "disaggregated"]) +def test_prediction_consumes_local_fpm_timings_without_changing_default_roots(local_profiles, mode): + default_roots = perf_database.get_systems_paths() + fast, slow = local_profiles + + fast_report = _predict(_local_request(fast, mode)) + slow_report = _predict(_local_request(slow, mode)) + repeat_report = _predict(_local_request(fast, mode)) + + assert fast_report.metrics["completed_requests"] == slow_report.metrics["completed_requests"] == 1 + assert slow_report.metrics["mean_ttft_ms"] > 5 * fast_report.metrics["mean_ttft_ms"] + assert repeat_report.metrics["duration_ms"] == fast_report.metrics["duration_ms"] + assert perf_database.get_systems_paths() == default_roots + deployment = prediction_to_replay_spec( + CorePredictionConfig.model_validate(_local_request(fast, mode)) + ).backend_deployment + assert deployment.performance_model_metadata + for metadata in deployment.performance_model_metadata.values(): + assert metadata["config"]["systems_path"] == str(fast) + + +@pytest.mark.parametrize("timing", [{"type": "fixed", "prefill_ms": 1, "decode_ms": 2}, {"type": "polynomial"}]) +def test_custom_timing_can_use_local_profiles_for_inferred_kv_capacity(local_profiles, timing): + raw = _local_request(local_profiles[0]) + raw["engine"]["workers"]["aggregated"]["timing"] = timing + + assert _predict(raw).metrics["completed_requests"] == 1 + + +def test_default_prediction_still_uses_bundled_op_level_data_after_a_local_request(local_profiles): + local = _local_request(local_profiles[0]) + default = deepcopy(local) + default["engine"].pop("systems_path") + default["engine"]["hardware"] = "h200_sxm" + default["engine"]["backend_version"] = perf_database.get_latest_database_version("h200_sxm", "vllm") + default["engine"]["workers"]["aggregated"].pop("timing") + + before = _predict(default) + assert _predict(local).metrics["completed_requests"] == 1 + after = _predict(default) + + assert before.metrics["completed_requests"] == after.metrics["completed_requests"] == 1 + assert after.metrics["duration_ms"] == before.metrics["duration_ms"] + config = CorePredictionConfig.model_validate(default) + assert "systems_path" not in config.model_dump(mode="json", exclude_none=True)["engine"] + assert config.engine.workers.aggregated.timing.forward_model == "op_level" + + +@pytest.mark.parametrize("mode", ["aggregated", "disaggregated"]) +def test_recommendation_resolves_local_capacity_and_exports_replayable_profiles(local_profiles, mode, tmp_path): + default_roots = perf_database.get_systems_paths() + candidates = [] + for root in local_profiles: + raw = _local_request(root, mode) + del raw["engine"]["backend_version"] + raw["traffic"]["load"] = {"type": "kv_capacity_fraction", "fraction": 0.000001} + raw["optimization"] = { + "target": "throughput", + "constraints": {"max_candidate_gpus": 1 if mode == "aggregated" else 2}, + } + raw["optimizer"] = {"algorithm": "random", "max_trials": 1, "parallelism": 1} + for worker in raw["engine"]["workers"].values(): + worker["kv_cache"].update(block_size=64, capacity={"memory_fraction": 0.9}) + worker["parallelism"] = { + "preset": [ + {"replicas": 1, "tensor": 1, "pipeline": 1, "attention_data": 1, "moe_tensor": 1, "moe_expert": 1} + ] + } + + result = run_recommendation( + CoreRecommendationConfig.model_validate(raw), + stack="engine", + runner_factory=EngineReplayRunnerFactory(), + show_progress=False, + ) + + assert result.counts.feasible == 1, [record.reason for record in result.candidates] + candidate = result.selected_candidates[0] + assert candidate.config["systems_path"] == str(root) + assert candidate.prediction_config["engine"]["systems_path"] == str(root) + assert candidate.prediction_config["engine"]["backend_version"] == _VERSION + assert result.candidates[0].provenance.performance_data + for metadata in result.candidates[0].provenance.performance_data: + assert metadata["config"]["systems_path"] == str(root) + [saved] = write_recommendations(tmp_path / f"export-{root.name}", [candidate.prediction_config]) + rerun = subprocess.run( + [ + sys.executable, + "-m", + "aisimulate", + "predict", + "--config", + str(saved), + "--output-dir", + str(tmp_path / f"predict-{root.name}"), + "--format", + "json", + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert rerun.returncode == 0, rerun.stderr + assert json.loads(rerun.stdout)["mean_ttft_ms"] == candidate.metrics["mean_ttft_ms"] + candidates.append(candidate) + + fast, slow = candidates + assert slow.metrics["mean_ttft_ms"] > 5 * fast.metrics["mean_ttft_ms"] + assert slow.config["kv_load_capacity_tokens"] > fast.config["kv_load_capacity_tokens"] + assert perf_database.get_systems_paths() == default_roots diff --git a/tests/sweeper/test_epd.py b/tests/sweeper/test_epd.py index bb39dc45c9..9393f9c4f1 100644 --- a/tests/sweeper/test_epd.py +++ b/tests/sweeper/test_epd.py @@ -282,7 +282,7 @@ def test_complete_sweeper_selection_and_serialization(monkeypatch, gpu_budget, f catalog = {"one": _encoder(), "two": _encoder(workers=2)} monkeypatch.setattr(search_mod, "enumerate_branches", lambda *a, **kw: [branch]) monkeypatch.setattr(search_mod, "resolve_encoder_catalog", lambda c: catalog) - monkeypatch.setattr(search_mod, "resolve_backend_version", lambda *a: "0.5.14") + monkeypatch.setattr(search_mod, "resolve_backend_version", lambda *a, systems_path=None: "0.5.14") if failure in {"build", "version", "kv"}: def fail_build(*args, **kwargs): @@ -440,10 +440,10 @@ def test_encoder_catalog_preserves_version_precedence( version_calls = [] database_calls = [] - def latest_version(system, backend): + def latest_version(system, backend, systems_paths=None): assert implicit, "explicit encoder versions must not depend on latest-version availability" version_calls.append((system, backend)) - return original_version(system, backend) + return original_version(system, backend, systems_paths=systems_paths) def database(system, backend, version, **kwargs): database_calls.append((system, backend, version)) diff --git a/tests/sweeper/test_model_hw.py b/tests/sweeper/test_model_hw.py index c1cb64f24a..629151045b 100644 --- a/tests/sweeper/test_model_hw.py +++ b/tests/sweeper/test_model_hw.py @@ -59,7 +59,7 @@ def test_aic_core_system_spec_contract(monkeypatch): monkeypatch.setattr( mh_mod.perf_database, "load_system_spec", - lambda hardware: { + lambda hardware, systems_paths=None: { "gpu": {"mem_capacity": 80}, "node": {"num_gpus_per_node": 8}, }, diff --git a/tests/sweeper/test_result.py b/tests/sweeper/test_result.py index fb10db1834..6d394fc215 100644 --- a/tests/sweeper/test_result.py +++ b/tests/sweeper/test_result.py @@ -486,7 +486,7 @@ def test_optimizer_guided_run_emits_complete_ledger_and_top_n(monkeypatch): monkeypatch.setattr( search_module, "resolve_backend_version", - lambda hardware, backend: "1.0", + lambda hardware, backend, systems_path=None: "1.0", ) result = Sweeper( @@ -545,7 +545,7 @@ def test_strict_sla_rejection_is_preserved_in_the_candidate_ledger(monkeypatch): monkeypatch.setattr( search_module, "resolve_backend_version", - lambda hardware, backend: "1.0", + lambda hardware, backend, systems_path=None: "1.0", ) config_data = _config().model_dump(mode="python") config_data["goal"] = { @@ -577,7 +577,7 @@ def test_same_batch_failed_duplicates_are_counted_as_coalesced_hits(monkeypatch) knob_choices={"backend": ["trtllm"]}, ) monkeypatch.setattr(search_module, "enumerate_branches", lambda *args, **kwargs: [branch]) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "1.0") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "1.0") result = Sweeper( runner_factory=_FailingRunnerFactory(), @@ -603,7 +603,7 @@ def test_zero_or_missing_sample_latency_preserves_ranked_sampler_feedback( knob_choices={"backend": ["trtllm"]}, ) monkeypatch.setattr(search_module, "enumerate_branches", lambda *args, **kwargs: [branch]) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "1.0") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "1.0") seen = {} @@ -660,7 +660,7 @@ def test_optimizer_guided_result_separates_unsupported_and_runtime_failure(monke monkeypatch.setattr( search_module, "resolve_backend_version", - lambda hardware, backend: "1.0", + lambda hardware, backend, systems_path=None: "1.0", ) result = Sweeper( diff --git a/tests/sweeper/test_search.py b/tests/sweeper/test_search.py index 0005a4c1d9..b88380512d 100644 --- a/tests/sweeper/test_search.py +++ b/tests/sweeper/test_search.py @@ -151,7 +151,7 @@ def _stub(monkeypatch, branch): "enumerate_branches", lambda config, *, max_seq_len=None, runner_capabilities=None: [branch], ) - monkeypatch.setattr(search_mod, "resolve_backend_version", lambda hw, be: "1.3.0rc10") + monkeypatch.setattr(search_mod, "resolve_backend_version", lambda hw, be, systems_path=None: "1.3.0rc10") def _pc(*, tp=4, replicas=2): @@ -229,7 +229,7 @@ def test_heterogeneous_disagg_materializes_role_hardware_and_provenance(monkeypa ) version_calls = [] - def resolve_version(hardware, backend): + def resolve_version(hardware, backend, *, systems_path=None): version_calls.append((hardware, backend)) return "0.24.0" @@ -885,7 +885,7 @@ def test_projection_stall_only_stops_current_branch(monkeypatch): "enumerate_branches", lambda config, *, max_seq_len=None, runner_capabilities=None: [agg, disagg], ) - monkeypatch.setattr(search_mod, "resolve_backend_version", lambda hw, be: "1.3.0rc10") + monkeypatch.setattr(search_mod, "resolve_backend_version", lambda hw, be, systems_path=None: "1.3.0rc10") seen = [] class RepeatingSampler(_FakeSampler): diff --git a/tests/sweeper/test_search_providers.py b/tests/sweeper/test_search_providers.py index e06f19d26a..6120b9d1b0 100644 --- a/tests/sweeper/test_search_providers.py +++ b/tests/sweeper/test_search_providers.py @@ -245,7 +245,7 @@ def _stub_branch(monkeypatch) -> None: monkeypatch.setattr( search_module, "resolve_backend_version", - lambda hardware, backend: "0.11.0", + lambda hardware, backend, systems_path=None: "0.11.0", ) @@ -373,7 +373,7 @@ def materialize_replay(self, plan, selection, context): monkeypatch.setattr( search_module, "resolve_backend_version", - lambda hardware, backend: "0.11.0", + lambda hardware, backend, systems_path=None: "0.11.0", ) prepared, result = search_module._materialize_one( { diff --git a/tests/sweeper/test_search_space.py b/tests/sweeper/test_search_space.py index 6667c5b9a2..102354722d 100644 --- a/tests/sweeper/test_search_space.py +++ b/tests/sweeper/test_search_space.py @@ -236,6 +236,7 @@ def fake_parallel_configs( min_gpu_budget=None, max_seq_len=None, role_runtime=None, + systems_path=None, ): calls.append((deployment_mode, backend)) return [_DISAGG_DP1_CFG] @@ -271,12 +272,15 @@ def fake_parallel_configs( min_gpu_budget=None, max_seq_len=None, role_runtime=None, + systems_path=None, ): calls.append((hardware, deployment_mode, backend_version, role_runtime)) return [_AGG_CFG if hardware == "h200_sxm" else _DP8_CFG] monkeypatch.setattr("aisimulate.sweeper.search_space.parallel_configs_for", fake_parallel_configs) - monkeypatch.setattr("aisimulate.sweeper.search_space.resolve_backend_version", lambda *args: "1.0") + monkeypatch.setattr( + "aisimulate.sweeper.search_space.resolve_backend_version", lambda *args, systems_path=None: "1.0" + ) config = _config( deployment_mode=["disagg"], hardware_sku="gb200", @@ -297,7 +301,7 @@ def fake_parallel_configs( def test_heterogeneous_disagg_requires_one_common_implicit_backend_version(monkeypatch): - def role_version(hardware, backend): + def role_version(hardware, backend, *, systems_path=None): del backend return {"h200_sxm": "prefill-version", "gb200": "decode-version"}[hardware] @@ -698,6 +702,7 @@ def fake_parallel_configs( min_gpu_budget=None, max_seq_len=None, role_runtime=None, + systems_path=None, ): if deployment_mode == "disagg": raise NoViableParallelConfig("disagg does not fit") @@ -745,6 +750,7 @@ def fake_parallel_configs( min_gpu_budget=None, max_seq_len=None, role_runtime=None, + systems_path=None, ): if backend == "vllm": raise NoPerfDatabase("no vLLM perf database") diff --git a/tests/sweeper/test_unified_optimizer.py b/tests/sweeper/test_unified_optimizer.py index 086e940946..a194bb43c0 100644 --- a/tests/sweeper/test_unified_optimizer.py +++ b/tests/sweeper/test_unified_optimizer.py @@ -146,7 +146,7 @@ def test_global_trial_budget_is_split_across_branches(monkeypatch) -> None: _CountingSampler.suggestion_batches = [] _CountingSampler.suggested = 0 monkeypatch.setattr(search_module, "enumerate_branches", lambda *args, **kwargs: _branches()) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "test") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "test") config = SmartSearchConfig.model_validate( { "search_space": { @@ -191,7 +191,7 @@ def test_global_trial_budget_runs_branch_batches_round_robin(monkeypatch) -> Non _CountingSampler.suggestion_batches = [] _CountingSampler.suggested = 0 monkeypatch.setattr(search_module, "enumerate_branches", lambda *args, **kwargs: _branches()) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "test") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "test") config = SmartSearchConfig.model_validate( { "search_space": { @@ -241,7 +241,7 @@ def test_legacy_rounds_remain_branch_major_without_max_trials(monkeypatch) -> No _CountingSampler.suggestion_batches = [] _CountingSampler.suggested = 0 monkeypatch.setattr(search_module, "enumerate_branches", lambda *args, **kwargs: _branches()) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "test") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "test") config = SmartSearchConfig.model_validate( { "search_space": { @@ -286,7 +286,7 @@ def test_branch_seed_is_stable_when_branch_order_changes(monkeypatch) -> None: "enumerate_branches", lambda *args, **kwargs: list(reversed(_branches())), ) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "test") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "test") config = SmartSearchConfig.model_validate( { "search_space": { @@ -351,7 +351,7 @@ def test_candidate_timeout_applies_with_parallelism_one(monkeypatch) -> None: "enumerate_branches", lambda *args, **kwargs: [_branches()[0]], ) - monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args: "test") + monkeypatch.setattr(search_module, "resolve_backend_version", lambda *args, systems_path=None: "test") config = SmartSearchConfig.model_validate( { "search_space": { diff --git a/tests/test_epd_cli.py b/tests/test_epd_cli.py index bbbae7ce75..e778c42447 100644 --- a/tests/test_epd_cli.py +++ b/tests/test_epd_cli.py @@ -383,8 +383,8 @@ def test_epd_native_search_preserves_available_backends(monkeypatch, caplog, bac def database(system, backend, version, **kwargs): return None if backend == "vllm" else original_database(system, backend, version, **kwargs) - def latest_version(system, backend): - return None if backend == "vllm" else original_version(system, backend) + def latest_version(system, backend, systems_paths=None): + return None if backend == "vllm" else original_version(system, backend, systems_paths=systems_paths) raw = _recommendation() raw["engine"]["workers"]["encoder"]["replicas"] = 1 From dcb73f99c378125074d87ba2c02ac53f258b84d8 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 14 Sep 2026 22:00:04 +0800 Subject: [PATCH 02/27] feat: add independent guided FPM onboarding workflow Add typed setup, bounded plans, safe collector preview and resume, and ordinary predict/recommend configs using local systems data. Track AIC-1963. Signed-off-by: Yiming Liu --- README.md | 1 + docs/self-service-support.md | 117 ++++ python/aisimulate/src/aisimulate/main.py | 9 + .../src/aisimulate/support/__init__.py | 8 + .../aisimulate/src/aisimulate/support/cli.py | 315 +++++++++ .../aisimulate/src/aisimulate/support/fpm.py | 145 +++++ .../aisimulate/src/aisimulate/support/plan.py | 300 +++++++++ .../src/aisimulate/support/schema.py | 129 ++++ .../aisimulate/tests/unit/test_support_cli.py | 427 ++++++++++++ .../tests/unit/test_support_plan.py | 610 ++++++++++++++++++ 10 files changed, 2061 insertions(+) create mode 100644 docs/self-service-support.md create mode 100644 python/aisimulate/src/aisimulate/support/__init__.py create mode 100644 python/aisimulate/src/aisimulate/support/cli.py create mode 100644 python/aisimulate/src/aisimulate/support/fpm.py create mode 100644 python/aisimulate/src/aisimulate/support/plan.py create mode 100644 python/aisimulate/src/aisimulate/support/schema.py create mode 100644 python/aisimulate/tests/unit/test_support_cli.py create mode 100644 python/aisimulate/tests/unit/test_support_plan.py diff --git a/README.md b/README.md index d50bded9bd..6ae78d0f2d 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,7 @@ Use the focused SDK documentation instead of treating CLI internals as public APIs: - [Estimator/FPE Python and Rust SDK](docs/core-api.md) +- [Guided FPM onboarding and local data setup](docs/self-service-support.md) - [FPM collection-to-prediction workflow](python/aisimulate/docs/fpm/end-to-end-workflow.md) - [Replay SDK and artifact contract](crates/core/src/replay/README.md) - [Sweeper SDK](docs/sweeper/overview.md) diff --git a/docs/self-service-support.md b/docs/self-service-support.md new file mode 100644 index 0000000000..f911fc54e3 --- /dev/null +++ b/docs/self-service-support.md @@ -0,0 +1,117 @@ + + +# Set up FPM collection for a model and GPU allocation + +`aisimulate support` creates a small onboarding request, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. + +Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model integration, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not implement an Inkling model class, run preflight checks, provision GPU resources, or establish measured accuracy. Those steps belong to the broader integration project. + +## Create the request + +Use an environment installed from this checkout; see [development setup](../DEVELOPMENT.md). Guided setup requires a terminal and starts only when explicitly requested: + +```bash +aisimulate support init --interactive --output support-request.yaml +``` + +Enter the actual model identifier or checkpoint path, pinned model revision, dense/MoE kind, pinned vLLM version, GPU system, allocation, and interconnect. Then choose a TP size and pilot workload. Supplied options skip their prompts. Enter accepts displayed defaults; invalid values can be corrected; Ctrl-C or end-of-input cancels without saving. Existing files require `--overwrite`. + +Both guided and scripted setup use onboarding. `--profile onboarding` is an optional spelling of the same behavior. For automation, supply the identity flags directly: + +```bash +aisimulate support init \ + --model /models/your-pinned-checkpoint \ + --model-revision YOUR_IMMUTABLE_REVISION \ + --model-kind dense \ + --framework-version YOUR_PINNED_VLLM_VERSION \ + --gpu h200_sxm --gpu-count 4 --interconnect nvswitch \ + --tensor-parallel 2 \ + --output support-request.yaml +``` + +Replace the model and runtime placeholders with the actual inputs. The example does not identify an Inkling checkpoint or claim that four H200s can run your model. Framework support currently selects vLLM. Optional tokenizer, chat-template, and AISimulate revisions are recorded only when supplied. + +The default pilot uses 1,024 input tokens, 128 output tokens, concurrency 1, four requests, a 16,384-token context limit, TTFT target 1,000 ms, and TPOT target 100 ms. Scripted setup defaults to TP1 unless `--tensor-parallel` is supplied. These are planning defaults, not measured model capacity or latency. Change them with the corresponding flags shown by `aisimulate support init --help`. + +One node is the default; GPUs per node then equals `--gpu-count`. For multiple nodes, supply both `--node-count` and `--gpus-per-node`; their product must equal the total allocation. Each pure-TP worker must fit on one node. Advanced flags include `--request-count`, `--max-candidates`, `--objective`, `--seed`, and `--sm`. + +## Plan, preview, and explicitly execute + +```bash +aisimulate support plan \ + --config support-request.yaml --output-dir ./aisimulate-support + +aisimulate support collect-fpm \ + --config ./aisimulate-support/request.yaml \ + --output-dir ./aisimulate-support +``` + +The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. The second prints the collector command without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. On an existing plan, `--overwrite` can repair missing generated files for the identical request; it rejects changed inputs and preserves existing collected data. + +The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. + +Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` includes the underlying collector command for reviewing or extending it with the collector's deployment flags. + +After those prerequisites are ready, explicitly launch collection from that environment: + +```bash +aisimulate support collect-fpm \ + --config ./aisimulate-support/request.yaml \ + --output-dir ./aisimulate-support --execute +``` + +Execution requires a matching saved plan. The request records model and runtime revisions; this setup does not download a pinned checkpoint or verify the installed runtime against them. Keep the actual checkpoint and runtime consistent with the request before collecting or predicting. + +For a diagnostic run, add `--execute --smoke`; `--limit N` also requires `--smoke`. Diagnostic smoke and limited runs do not publish formal FPM data. Existing campaign data, raw artifacts, or checkpoints require explicit `--resume` and a readable matching collector checkpoint; otherwise choose a new output directory. A custom `--checkpoint-dir`, if needed, must remain inside the plan's `fpm-checkpoint/` directory. Selecting an empty checkpoint directory does not allow reuse of existing campaign artifacts. Smoke and formal campaigns have separate checkpoints and artifact directories, so an existing smoke run does not prevent the first formal run, or vice versa. The collector verifies the resumed checkpoint's frozen-plan identity. + +The collector narrows initial prefill sampling with the pilot's input-token and concurrency bounds. Decode uses the collector's existing profile; a four-request synthetic pilot does not imply four timing samples or a short decode campaign. Inspect the generated command and collector plan before committing GPU time. Successful formal collection publishes the FPM Parquet file and metadata pair into the plan's local systems data directory; diagnostic success alone does not provide that pair. + +## Run the generated ordinary configurations + +After model integration and formal data collection are complete: + +```bash +aisimulate predict \ + --config ./aisimulate-support/predict/pilot.yaml \ + --output-dir ./aisimulate-support/predict-results/pilot + +aisimulate recommend \ + --config ./aisimulate-support/recommend/pilot.yaml \ + --output-dir ./aisimulate-support/recommend-results/pilot +``` + +With two candidates, run every recommendation command in `commands.json` so both replica counts are evaluated. This executes the emitted ordinary command vectors: + +```bash +python3 - <<'PY' +import json +import shlex +import subprocess +from pathlib import Path + +commands = json.loads(Path("./aisimulate-support/commands.json").read_text()) +statuses = [] +for command in commands["recommend"]: + result = subprocess.run(command, check=False) + statuses.append(result.returncode) + print(f"exit {result.returncode}: {shlex.join(command)}", flush=True) +raise SystemExit(1 if any(statuses) else 0) +PY +``` + +The loop reports each command's exit status and attempts all commands, including when the pilot finds no feasible candidate. It exits 1 after all attempts if any command returned a nonzero status, otherwise 0. Successful outputs remain usable even when the loop exits 1; inspect each result before comparing candidates. + +Run either the individual recommendation command or the command-vector loop against fresh result directories. The loop also handles a one-candidate plan. Results remain separate under `recommend-results/pilot` and, when present, `recommend-results/replicas-N`; compare their objective and latency results for the same workload. This plan does not produce a combined ranking or search additional TP sizes or scheduler settings. + +The generated configurations select `engine.workers.aggregated.timing.forward_model: fpm` and set `engine.systems_path` to the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. + +Ordinary `predict` and `recommend` commands retain their existing behavior and defaults. Their generated configs can be loaded and edited through the public configuration schema. Missing model integration or data may still prevent execution; a successful simulation is not an accuracy result. Compare its output with an independent run of the same model, runtime, topology, and workload to assess accuracy. + +## Hand off a new model architecture + +Give an engineer the saved request, the pinned model configuration, and the plan. Follow [How to Add a New Model](../python/aisimulate/docs/add_a_new_model.md) to integrate the architecture and model class, including its operation graph, memory accounting, and KV-cache behavior. FPM supplies measured whole-forward timings; it does not replace the model structure needed by simulation. + +The model guide also describes legacy per-operation silicon profiling. Collecting those per-operation timings is not mandatory for this FPM workflow. Integrate the model structure first, collect the matching whole-forward FPM data through the existing collector, and then verify prediction and recommendation on the target deployment. diff --git a/python/aisimulate/src/aisimulate/main.py b/python/aisimulate/src/aisimulate/main.py index b7da8a8520..d3c7641097 100644 --- a/python/aisimulate/src/aisimulate/main.py +++ b/python/aisimulate/src/aisimulate/main.py @@ -39,6 +39,7 @@ write_requests, ) from .stack import StackResolutionError, resolve_runner_factory +from .support.cli import add_support_parser, run_support_command from .sweeper.provider import AdapterReplaySpec from .sweeper.replay import ReplayOutputRequirements @@ -77,6 +78,7 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="pace prediction against the real wall clock instead of virtual time", ) + add_support_parser(subparsers) return parser @@ -301,6 +303,13 @@ def _recommend(args: argparse.Namespace, raw: dict[str, Any], factory) -> int: def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(list(sys.argv[1:] if argv is None else argv)) + if args.command == "support": + try: + return run_support_command(args) + except (ValidationError, ValueError, OSError) as exc: + parser.error(str(exc)) + except KeyboardInterrupt: + return 130 # Stack resolution deliberately precedes opening the configuration file. try: factory = resolve_runner_factory(args.stack) diff --git a/python/aisimulate/src/aisimulate/support/__init__.py b/python/aisimulate/src/aisimulate/support/__init__.py new file mode 100644 index 0000000000..e5bce66e62 --- /dev/null +++ b/python/aisimulate/src/aisimulate/support/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Request and planning helpers for local FPM onboarding.""" + +from .schema import SupportRequest + +__all__ = ["SupportRequest"] diff --git a/python/aisimulate/src/aisimulate/support/cli.py b/python/aisimulate/src/aisimulate/support/cli.py new file mode 100644 index 0000000000..0cb56b974f --- /dev/null +++ b/python/aisimulate/src/aisimulate/support/cli.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Terminal setup and local planning for FPM onboarding.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import sys +import tempfile +from pathlib import Path +from typing import Any, get_args + +import yaml +from pydantic import ValidationError + +from .fpm import run_fpm +from .plan import create_plan, request_id +from .schema import SearchProfile, SloSpec, SupportIdentity, SupportRequest, WorkloadSpec + + +def add_support_parser(subparsers: Any) -> None: + support = subparsers.add_parser( + "support", + help="Set up a model and GPU allocation for local FPM collection.", + description="Create an onboarding request, generate a plan, and preview or execute local FPM collection.", + ) + actions = support.add_subparsers(dest="support_action", required=True) + init = actions.add_parser( + "init", + help="Create an onboarding request with prompts or scripted options.", + description=( + "Use --interactive for terminal prompts. Scripted setup requires --model, --model-revision, " + "--model-kind, --framework-version, --gpu, --gpu-count, and --interconnect. " + "One node and a small TP1 synthetic pilot are defaults; review the saved scope before collection." + ), + ) + init.add_argument("--interactive", action="store_true", help="Guide setup with terminal prompts.") + init.add_argument("--profile", choices=("onboarding",), help="Optional alias; all setup uses onboarding.") + init.add_argument("--model") + init.add_argument("--model-revision", help="Pinned model revision; moving labels such as main are unsupported.") + init.add_argument("--model-kind", choices=("dense", "moe")) + init.add_argument("--framework", choices=("vllm",), help="Collection runtime (default: vllm).") + init.add_argument("--framework-version", help="Pinned vLLM version in the collection environment.") + init.add_argument("--gpu", help="GPU system name, for example h200_sxm.") + init.add_argument("--gpu-count", type=int, help="Total GPUs available.") + init.add_argument("--node-count", type=int, help="Number of nodes (default: 1).") + init.add_argument("--gpus-per-node", type=int, help="GPUs available per node; defaults to gpu-count on one node.") + init.add_argument("--interconnect", help="Interconnect, for example nvswitch, pcie, or none.") + init.add_argument("--sm", type=int, help="GPU SM version override.") + init.add_argument("--tokenizer-revision") + init.add_argument("--chat-template-revision") + init.add_argument("--aisimulate-revision", help="Optional pinned AISimulate source revision.") + init.add_argument("--tensor-parallel", type=int, help="GPUs per pure-TP worker (default: 1).") + init.add_argument("--input-tokens", type=int, help="Input tokens per request (default: 1024).") + init.add_argument("--output-tokens", type=int, help="Output tokens per request (default: 128).") + init.add_argument("--concurrency", type=int, help="Concurrent requests (default: 1).") + init.add_argument("--request-count", type=int, help="Synthetic request count (default: 4).") + init.add_argument("--context-length", type=int, help="Pilot context limit in tokens (default: 16384).") + init.add_argument("--ttft-ms", type=float, help="Target time to first token in ms (default: 1000).") + init.add_argument("--tpot-ms", type=float, help="Target time per output token in ms (default: 100).") + init.add_argument("--max-candidates", type=int, help="One worker, or also the largest replica count (1 or 2).") + init.add_argument( + "--objective", + choices=get_args(SearchProfile.model_fields["objective"].annotation), + help="Recommendation objective (default: throughput).", + ) + init.add_argument("--seed", type=int, help="Recommendation search seed (default: 42).") + init.add_argument("--output", default="support-request.yaml") + init.add_argument("--overwrite", action="store_true") + + plan = actions.add_parser("plan", help="Write ordinary predict/recommend configs and local collector commands.") + plan.add_argument("-c", "--config", required=True) + plan.add_argument("--output-dir", default="./aisimulate-support") + plan.add_argument("--overwrite", action="store_true") + plan.add_argument("--format", choices=("table", "json"), default="table") + + collect = actions.add_parser("collect-fpm", help="Preview the local collector command, or run with --execute.") + collect.add_argument("-c", "--config", required=True) + collect.add_argument("--output-dir", default="./aisimulate-support") + collect.add_argument( + "--execute", action="store_true", help="Execute collection using a matching saved support plan." + ) + collect.add_argument( + "--smoke", action="store_true", help="Diagnostic smoke collection; does not publish formal data." + ) + collect.add_argument( + "--limit", type=int, help="Limit diagnostic cases; requires --smoke, no formal data publication." + ) + collect.add_argument("--resume", action="store_true", help="Resume the existing collector checkpoint.") + collect.add_argument("--checkpoint-dir", help="Checkpoint directory inside the plan's fpm-checkpoint directory.") + + +def _values(args: argparse.Namespace, model: Any) -> dict[str, Any]: + return {name: getattr(args, name) for name in model.model_fields if getattr(args, name, None) is not None} + + +def _request_from_args(args: argparse.Namespace) -> SupportRequest: + identity = _values(args, SupportIdentity) + if args.gpus_per_node is None and identity.get("node_count", 1) == 1: + identity["gpus_per_node"] = args.gpu_count + return SupportRequest.model_validate( + { + "identity": identity, + "workload": {**_values(args, WorkloadSpec), "slo": _values(args, SloSpec)}, + "search": _values(args, SearchProfile), + } + ) + + +def _request_target(path: str | Path, *, overwrite: bool) -> Path: + target = Path(path).expanduser().absolute() + if target.exists() and not target.is_file(): + raise ValueError(f"support request {target} must be a file path") + if (target.exists() or target.is_symlink()) and not overwrite: + raise ValueError(f"support request {target} exists; pass --overwrite to replace it") + parent = target.parent + while not parent.exists() and not parent.is_symlink(): + parent = parent.parent + if not parent.is_dir() or not os.access(parent, os.W_OK): + raise ValueError(f"support request parent {parent} must be a writable directory") + return target + + +def _write_request(request: SupportRequest, path: str | Path, *, overwrite: bool) -> Path: + target = _request_target(path, overwrite=overwrite) + payload = yaml.safe_dump(request.model_dump(mode="json", exclude_none=True), sort_keys=False) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=target.parent, delete=False) as stream: + temporary = Path(stream.name) + stream.write(payload) + if overwrite: + os.replace(temporary, target) + else: + os.link(temporary, target) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return target + + +def _print(value: dict[str, Any], output_format: str = "table") -> None: + if output_format == "json": + print(json.dumps(value, sort_keys=True)) + return + for key, item in value.items(): + rendered = json.dumps(item, sort_keys=True) if isinstance(item, (dict, list)) else str(item) + print(f"{key}: {rendered}") + + +_PROMPTS = { + "model": ("Model name or path", str), + "model_revision": ("Pinned model revision (not main/latest)", str), + "model_kind": ("Model kind (dense/moe)", str), + "framework_version": ("Pinned vLLM version", str), + "gpu": ("GPU system name (for example h200_sxm)", str), + "gpu_count": ("Total GPUs available", int), + "interconnect": ("GPU interconnect (for example nvswitch, pcie, or none)", str), + "tensor_parallel": ("GPUs per pure-TP worker", int), + "input_tokens": ("Input tokens per request", int), + "output_tokens": ("Output tokens per request", int), + "concurrency": ("Concurrent requests", int), + "context_length": ("Pilot context limit in tokens", int), + "ttft_ms": ("Target time to first token (ms)", float), + "tpot_ms": ("Target time per output token (ms)", float), +} +_CORRECTION_PROMPTS = { + **_PROMPTS, + "framework": ("Runtime (vllm)", str), + "tokenizer_revision": ("Pinned tokenizer revision", str), + "chat_template_revision": ("Pinned chat-template revision", str), + "aisimulate_revision": ("Pinned AISimulate revision", str), + "node_count": ("Number of nodes", int), + "gpus_per_node": ("GPUs per node", int), + "sm": ("GPU SM version", int), + "request_count": ("Number of synthetic requests", int), + "max_candidates": ("Maximum candidates (1 or 2)", int), + "objective": ("Recommendation objective", str), + "seed": ("Recommendation search seed", int), +} + + +def _prompt(args: argparse.Namespace, name: str) -> None: + label, convert = _CORRECTION_PROMPTS[name] + default = getattr(args, name) + if default is None: + for model in (SupportIdentity, WorkloadSpec, SloSpec, SearchProfile): + field = model.model_fields.get(name) + if field is not None and not field.is_required(): + default = field.default + if name == "gpu_count": + default = 1 + while True: + answer = input(f"{label}{f' [{default}]' if default is not None else ''}: ").strip() + if not answer and default is None: + print("A value is required.") + continue + try: + value = convert(answer) if answer else default + except ValueError: + print(f"Enter a valid {'integer' if convert is int else 'number'}.") + continue + setattr(args, name, value) + return + + +def _guided_request(args: argparse.Namespace) -> SupportRequest: + if not sys.stdin.isatty(): + raise ValueError( + "--interactive requires a terminal; for automation supply options (aisimulate support init --help)" + ) + print("FPM onboarding: a pure-TP worker and a small synthetic workload on vLLM.") + print("Enter accepts a displayed default. Ctrl-C cancels without saving. Supplied options skip their prompts.") + for name in _PROMPTS: + if getattr(args, name) is None: + _prompt(args, name) + while True: + try: + return _request_from_args(args) + except ValidationError as exc: + error = exc.errors(include_url=False)[0] + name = error["loc"][-1] if error["loc"] else None + print(f"{name or 'Request'}: {error['msg']}") + while name not in _CORRECTION_PROMPTS: + name = ( + input("Option to correct (for example --tensor-parallel): ") + .strip() + .removeprefix("--") + .replace("-", "_") + ) + if name not in _CORRECTION_PROMPTS: + print("Choose one of: " + ", ".join(key.replace("_", "-") for key in _CORRECTION_PROMPTS)) + _prompt(args, name) + + +def _init(args: argparse.Namespace) -> int: + _request_target(args.output, overwrite=args.overwrite) + try: + request = _guided_request(args) if args.interactive else _request_from_args(args) + path = _write_request(request, args.output, overwrite=args.overwrite) + except (EOFError, KeyboardInterrupt): + print("Setup cancelled.", file=sys.stderr) + return 130 + workload = request.workload + print( + f"Scope: {request.identity.model_kind} model, vLLM {request.identity.framework_version}, " + f"{request.identity.gpu_count} {request.identity.gpu} GPU(s) on {request.identity.node_count} node(s); " + f"TP{request.search.tensor_parallel}, {workload.input_tokens}/{workload.output_tokens} tokens, " + f"concurrency {workload.concurrency}, {workload.request_count} requests, " + f"up to {request.search.max_candidates} candidate(s)." + ) + print( + "Request saved. Model integration, runtime compatibility and FPM data are unchecked; " + "accuracy is not assessed. Setup has not launched GPU work." + ) + _print( + { + "request_id": request_id(request), + "request": str(path), + "next": shlex.join(["aisimulate", "support", "plan", "--config", str(path)]), + } + ) + return 0 + + +def _plan(args: argparse.Namespace) -> int: + request = SupportRequest.from_yaml(args.config) + root = Path(args.output_dir).expanduser().resolve() + plan = create_plan(request, root, overwrite=args.overwrite) + _print( + { + "request_id": plan["request_id"], + "candidate_count": plan["search"]["candidate_count"], + "plan": str(root / "support-plan.json"), + "commands": plan["outputs"]["commands"], + "prerequisites": "unchecked", + "accuracy": "not assessed", + "next": shlex.join( + [ + "aisimulate", + "support", + "collect-fpm", + "--config", + plan["outputs"]["request"], + "--output-dir", + str(root), + ] + ), + }, + args.format, + ) + return 0 + + +def run_support_command(args: argparse.Namespace) -> int: + if args.support_action == "init": + return _init(args) + if args.support_action == "plan": + return _plan(args) + if args.support_action == "collect-fpm": + return run_fpm( + SupportRequest.from_yaml(args.config), + output_dir=args.output_dir, + execute=args.execute, + smoke=args.smoke, + limit=args.limit, + resume=args.resume, + checkpoint_dir=args.checkpoint_dir, + ) + raise AssertionError(f"unhandled support action {args.support_action!r}") diff --git a/python/aisimulate/src/aisimulate/support/fpm.py b/python/aisimulate/src/aisimulate/support/fpm.py new file mode 100644 index 0000000000..f09634310f --- /dev/null +++ b/python/aisimulate/src/aisimulate/support/fpm.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Preview or explicitly invoke the existing packaged FPM collector.""" + +from __future__ import annotations + +import json +import shlex +from pathlib import Path + +from .schema import SupportRequest + + +def fpm_cli_args( + request: SupportRequest, + *, + output_dir: str | Path, + plan_only: bool, + smoke: bool = False, + limit: int | None = None, + resume: bool = False, + checkpoint_dir: str | Path | None = None, +) -> list[str]: + if limit is not None and (not smoke or type(limit) is not int or limit < 1): + raise ValueError("limit must be a positive cell count and requires smoke=True") + root = Path(output_dir).expanduser().resolve() + checkpoint = Path(checkpoint_dir).expanduser().resolve() if checkpoint_dir else root / "fpm-checkpoint" + if checkpoint != root / "fpm-checkpoint" and root / "fpm-checkpoint" not in checkpoint.parents: + raise ValueError("checkpoint_dir must stay within the plan's fpm-checkpoint directory") + command = [ + "python3", + "-m", + "collector.fpm_forward", + "--backend", + request.identity.framework, + "--model-path", + request.identity.model, + "--gpu", + request.identity.gpu, + "--fpm-max-gpus", + str(request.search.tensor_parallel), + "--fpm-gpu-counts", + str(request.search.tensor_parallel), + "--fpm-parallel-presets", + "pure_tp" if request.identity.model_kind == "moe" else "tp", + "--fpm-max-prefill-isl", + str(max(2, request.workload.input_tokens * request.workload.concurrency)), + "--fpm-max-prefill-batch-size", + str(request.workload.concurrency), + "--checkpoint-dir", + str(checkpoint), + "--fpm-artifact-root", + str(root / "fpm-artifacts"), + "--fpm-database-root", + str(root / "systems/data"), + ] + if request.identity.sm is not None: + command.extend(("--sm", str(request.identity.sm))) + if plan_only: + command.append("--plan-only") + if smoke: + command.append("--smoke") + if limit is not None: + command.extend(("--limit", str(limit))) + if resume: + command.append("--resume") + return command + + +def _check_campaign_outputs(root: Path, *, smoke: bool, resume: bool, checkpoint_dir: str | Path | None) -> None: + checkpoint_name = "fpm_forward_smoke.json" if smoke else "fpm_forward.json" + checkpoint_root = root / "fpm-checkpoint" + artifact_paths = (root / "fpm-artifacts").glob("*/smoke/*" if smoke else "*/*") + occupied = ( + any(smoke or path.name != "smoke" for path in artifact_paths) + or any(checkpoint_root.rglob(checkpoint_name)) + or (not smoke and any((root / "systems/data").glob("*"))) + ) + if not occupied: + return + if not resume: + raise ValueError( + "existing campaign outputs require --resume with a matching checkpoint; " + "choose a new output directory otherwise" + ) + selected = Path(checkpoint_dir).expanduser().resolve() if checkpoint_dir else checkpoint_root + checkpoint = selected / checkpoint_name + try: + if checkpoint.is_symlink(): + raise ValueError("symlinked checkpoint") + payload = json.loads(checkpoint.read_text(encoding="utf-8")) + if ( + not isinstance(payload, dict) + or not isinstance(payload.get("schema"), str) + or not payload["schema"] + or not isinstance(payload.get("plan_sha256"), str) + or len(payload["plan_sha256"]) != 64 + or not isinstance(payload.get("cells"), dict) + ): + raise ValueError("invalid checkpoint document") + except (OSError, ValueError) as exc: + raise ValueError( + f"existing campaign outputs require a readable matching checkpoint at {checkpoint}; " + "choose a new output directory otherwise" + ) from exc + # The collector remains responsible for the checkpoint schema and frozen-plan identity. + + +def run_fpm( + request: SupportRequest, + *, + output_dir: str | Path, + execute: bool = False, + smoke: bool = False, + limit: int | None = None, + resume: bool = False, + checkpoint_dir: str | Path | None = None, +) -> int: + """Preview without side effects; execution requires the matching saved plan.""" + + from .plan import check_plan, plan_lock + + root = Path(output_dir).expanduser().resolve() + command = fpm_cli_args( + request, + output_dir=root, + plan_only=not execute, + smoke=smoke, + limit=limit, + resume=resume, + checkpoint_dir=checkpoint_dir, + ) + if not execute: + if resume or (root.exists() and any(root.iterdir())): + check_plan(request, root) + print(shlex.join(command)) + return 0 + check_plan(request, root) + with plan_lock(root): + check_plan(request, root) + _check_campaign_outputs(root, smoke=smoke, resume=resume, checkpoint_dir=checkpoint_dir) + from collector.fpm_forward.cli import main as fpm_main + + return fpm_main(command[3:]) diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py new file mode 100644 index 0000000000..9d55daff03 --- /dev/null +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -0,0 +1,300 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create ordinary prediction/recommendation inputs without resolving a model.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import yaml + +from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig + +from .fpm import fpm_cli_args +from .schema import SupportRequest + +_LOCK_NAME = ".support.lock" + + +def request_id(request: SupportRequest) -> str: + """Bind the entire declared request, including workload and search settings.""" + + payload = json.dumps(request.model_dump(mode="json", exclude_none=True), sort_keys=True, separators=(",", ":")) + return "onboarding-" + hashlib.sha256(payload.encode()).hexdigest() + + +def _parallelism(request: SupportRequest, replicas: int) -> dict[str, int]: + return { + "replicas": replicas, + "tensor": request.search.tensor_parallel, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": request.search.tensor_parallel if request.identity.model_kind == "moe" else 1, + "moe_expert": 1, + } + + +def _configs( + request: SupportRequest, root: Path +) -> tuple[dict[str, Any], dict[str, dict[str, Any]], list[dict[str, int]]]: + max_replicas = request.identity.node_count * (request.identity.gpus_per_node // request.search.tensor_parallel) + replicas = list(dict.fromkeys((1, max_replicas)))[: request.search.max_candidates] + presets = [_parallelism(request, count) for count in replicas] + workload = request.workload + common = { + "traffic": { + "source": { + "type": "synthetic", + "input_tokens": workload.input_tokens, + "output_tokens": workload.output_tokens, + }, + "load": {"type": "concurrency", "concurrency": workload.concurrency}, + "stop": {"requests": workload.request_count}, + }, + "evaluation": {"sla": {"ttft_ms": workload.slo.ttft_ms, "itl_ms": workload.slo.tpot_ms}}, + } + engine = { + "mode": "aggregated", + "model": request.identity.model, + "hardware": request.identity.gpu, + "backend": request.identity.framework, + "backend_version": request.identity.framework_version, + "systems_path": str(root / "systems"), + "context_length": request.search.context_length, + } + # Recommendation otherwise expands scheduler defaults into extra domains. + worker = { + "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + "timing": {"type": "default", "forward_model": "fpm"}, + } + prediction = { + **common, + "engine": {**engine, "workers": {"aggregated": {**worker, "parallelism": presets[0]}}}, + } + CorePredictionConfig.model_validate(prediction) + recommendations = {} + for preset in presets: + recommendation = { + **common, + "engine": {**engine, "workers": {"aggregated": {**worker, "parallelism": {"preset": [preset]}}}}, + "optimization": { + "target": request.search.objective, + "constraints": {"max_candidate_gpus": request.identity.gpu_count}, + }, + "optimizer": {"algorithm": "random", "max_trials": 1, "parallelism": 1, "seed": request.search.seed}, + } + CoreRecommendationConfig.model_validate(recommendation) + name = "pilot" if preset["replicas"] == 1 else f"replicas-{preset['replicas']}" + recommendations[name] = recommendation + return prediction, recommendations, presets + + +def _commands(request: SupportRequest, root: Path, recommendation_names: list[str]) -> dict[str, Any]: + return { + "fpm_plan_local": fpm_cli_args(request, output_dir=root, plan_only=True), + "fpm_run_local": fpm_cli_args(request, output_dir=root, plan_only=False), + "predict": [ + [ + "aisimulate", + "predict", + "--config", + str(root / "predict/pilot.yaml"), + "--output-dir", + str(root / "predict-results/pilot"), + "--format", + "json", + ] + ], + "recommend": [ + [ + "aisimulate", + "recommend", + "--config", + str(root / f"recommend/{name}.yaml"), + "--output-dir", + str(root / f"recommend-results/{name}"), + "--format", + "json", + ] + for name in recommendation_names + ], + } + + +def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any], dict[Path, bytes]]: + source_spec = Path(__file__).resolve().parents[2] / "aiconfigurator_core/systems" / f"{request.identity.gpu}.yaml" + if not source_spec.is_file(): + raise ValueError( + f"GPU {request.identity.gpu!r} has no packaged system specification; " + "new GPU architectures require integration" + ) + prediction, recommendations, presets = _configs(request, root) + commands = _commands(request, root, list(recommendations)) + plan = { + "schema_version": "aisimulate-support-plan/v1", + "request_id": request_id(request), + "search": { + "candidate_count": len(presets), + "max_candidates": request.search.max_candidates, + "candidates": [ + {"parallelism": preset, "total_gpus": preset["replicas"] * preset["tensor"]} for preset in presets + ], + "detail": ( + "A single selected worker is planned; recommendation does not compare alternative configurations." + if len(presets) == 1 + else "Run one independent one-trial recommendation for the selected worker and one for the maximum " + "same-worker replicas that fit within each node. Compare their separate results; no combined ranking " + "or additional topology search is generated." + ), + "baseline_rule": "Selected single worker; model parallelism legality and memory fit remain unchecked.", + }, + "fpm": { + "status": "planned", + "worker_gpus": request.search.tensor_parallel, + "plan_command": commands["fpm_plan_local"], + "sampling": ( + "Prefill is bounded by max(2, input_tokens * concurrency) and concurrency. Decode sampling and " + "runtime context auto-fitting use the collector's existing profile. The synthetic request count " + "does not bound timing samples. Smoke and limited runs do not publish formal FPM data." + ), + }, + "prerequisites": [ + { + "id": "model_integration", + "status": "not_checked", + "detail": ( + "Add or verify the registered model class, architecture mapping, memory/cache accounting, " + "and chosen parallelism using docs/add_a_new_model.md. Per-operation silicon data is not required." + ), + }, + { + "id": "runtime", + "status": "not_checked", + "detail": ( + "Prepare and verify the declared model/tokenizer revisions, vLLM version, visible GPUs, " + "model access, and the packaged Dynamo/Kubernetes/Generator collector runtime. Local invocation " + "uses that existing runtime. The collector does not apply revision/version declarations; " + "use a pinned local model snapshot where needed." + ), + }, + { + "id": "fpm_data", + "status": "not_checked", + "detail": "Collect matching prefill/decode timings into the local systems tree before prediction or " + "recommendation. Creating a plan establishes neither timing coverage nor memory fit.", + }, + ], + "accuracy": { + "status": "not_assessed", + "detail": "No measured accuracy or E2E validation is claimed by this plan.", + }, + "outputs": { + "request": str(root / "request.yaml"), + "prediction_configs": [str(root / "predict/pilot.yaml")], + "recommendation_configs": [str(root / f"recommend/{name}.yaml") for name in recommendations], + "recommendation_results": [str(root / f"recommend-results/{name}") for name in recommendations], + "commands": str(root / "commands.json"), + "systems_root": str(root / "systems"), + }, + } + yaml_documents = { + Path("request.yaml"): request.model_dump(mode="json", exclude_none=True), + Path("predict/pilot.yaml"): prediction, + **{Path(f"recommend/{name}.yaml"): config for name, config in recommendations.items()}, + } + documents = {path: yaml.safe_dump(data, sort_keys=False).encode() for path, data in yaml_documents.items()} + documents[Path("systems") / source_spec.name] = source_spec.read_bytes() + documents[Path("commands.json")] = (json.dumps(commands, indent=2, sort_keys=True) + "\n").encode() + documents[Path("support-plan.json")] = (json.dumps(plan, indent=2, sort_keys=True) + "\n").encode() + return plan, documents + + +@contextmanager +def plan_lock(root: Path) -> Iterator[None]: + """Reject overlapping support operations instead of racing plan writes.""" + + lock = root / _LOCK_NAME + try: + handle = lock.open("x", encoding="utf-8") + except FileExistsError as exc: + raise ValueError(f"another support operation holds {lock}") from exc + try: + with handle: + yield + finally: + lock.unlink() + + +def _check_paths(root: Path) -> None: + for relative in ( + "request.yaml", + "support-plan.json", + "commands.json", + "predict", + "predict/pilot.yaml", + "recommend", + "recommend/pilot.yaml", + "systems", + "systems/data", + "fpm-checkpoint", + "fpm-artifacts", + ): + if (root / relative).is_symlink(): + raise ValueError(f"refusing symlinked plan output {root / relative}") + + +def check_plan(request: SupportRequest, root: Path) -> None: + """Verify saved identity before collecting or reusing any local output.""" + + _check_paths(root) + try: + prior = json.loads((root / "support-plan.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"a readable support plan is required in {root}; run support plan first") from exc + if not isinstance(prior, dict) or prior.get("request_id") != request_id(request): + raise ValueError(f"refusing to mix a different request identity in {root}; choose a new output directory") + try: + saved = SupportRequest.from_yaml(root / "request.yaml") + except ValueError as exc: + raise ValueError(f"saved request in {root} was modified or cannot be read") from exc + if saved != request: + raise ValueError(f"saved request identity in {root} differs from the requested plan") + + +def create_plan(request: SupportRequest, output_dir: str | Path, *, overwrite: bool = False) -> dict[str, Any]: + """Write a new plan, or repair missing files for an identical saved request.""" + + root = Path(output_dir).expanduser().resolve() + plan, documents = _plan_documents(request, root) + root.mkdir(parents=True, exist_ok=True) + with plan_lock(root): + _check_paths(root) + nonempty = any(path.name != _LOCK_NAME for path in root.iterdir()) + if nonempty: + if not overwrite: + raise ValueError(f"output directory {root} is nonempty; use overwrite only for the same request") + check_plan(request, root) + # Inspect every generated file before writing any file. Existing data, + # checkpoints, results, and edited inputs are never replaced or deleted. + pending: dict[Path, bytes] = {} + for relative, content in documents.items(): + destination = root / relative + if destination.is_symlink(): + raise ValueError(f"refusing symlinked plan output {destination}") + if not destination.exists(): + pending[relative] = content + elif destination.read_bytes() != content: + raise ValueError(f"generated plan input {destination} was modified; choose a new output directory") + (root / "systems/data").mkdir(parents=True, exist_ok=True) + for relative, content in pending.items(): + destination = root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("xb") as handle: + handle.write(content) + return plan diff --git a/python/aisimulate/src/aisimulate/support/schema.py b/python/aisimulate/src/aisimulate/support/schema.py new file mode 100644 index 0000000000..60c73e368c --- /dev/null +++ b/python/aisimulate/src/aisimulate/support/schema.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Declared onboarding inputs; validity does not establish simulation readiness.""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from aisimulate.config.common import PositiveFiniteFloat, PositiveStrictInt, StrictModel, load_yaml + + +class SupportIdentity(StrictModel): + model: str + model_revision: str + model_kind: Literal["dense", "moe"] + framework: Literal["vllm"] = "vllm" + framework_version: str + gpu: str + gpu_count: PositiveStrictInt + node_count: PositiveStrictInt = 1 + gpus_per_node: PositiveStrictInt + interconnect: str + sm: PositiveStrictInt | None = None + tokenizer_revision: str | None = None + chat_template_revision: str | None = None + aisimulate_revision: str | None = None + + @field_validator( + "model", + "model_revision", + "framework_version", + "gpu", + "interconnect", + "tokenizer_revision", + "chat_template_revision", + "aisimulate_revision", + ) + @classmethod + def _nonempty(cls, value: str | None) -> str | None: + if value is None: + return None + value = value.strip() + if not value or "\x00" in value: + raise ValueError("identity values must be nonempty and contain no NUL characters") + return value + + @field_validator( + "model_revision", "framework_version", "tokenizer_revision", "chat_template_revision", "aisimulate_revision" + ) + @classmethod + def _pinned_revision(cls, value: str | None) -> str | None: + if value is not None and value.lower() in {"main", "master", "latest", "head"}: + raise ValueError("declare a pinned revision or version rather than a mutable default") + return value + + @field_validator("gpu") + @classmethod + def _system_name(cls, value: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", value): + raise ValueError("gpu must be a packaged system name, not a path") + return value + + @model_validator(mode="after") + def _allocation(self) -> SupportIdentity: + if self.node_count * self.gpus_per_node != self.gpu_count: + raise ValueError("node_count * gpus_per_node must equal gpu_count") + return self + + +class SloSpec(StrictModel): + ttft_ms: PositiveFiniteFloat = 1000.0 + tpot_ms: PositiveFiniteFloat = 100.0 + + +class WorkloadSpec(StrictModel): + """One fixed synthetic shape; request_count is not a timing sample count.""" + + input_tokens: PositiveStrictInt = 1024 + output_tokens: PositiveStrictInt = 128 + concurrency: PositiveStrictInt = 1 + request_count: PositiveStrictInt = 4 + slo: SloSpec = Field(default_factory=SloSpec) + + @model_validator(mode="after") + def _request_count(self) -> WorkloadSpec: + if self.request_count < self.concurrency: + raise ValueError("request_count must be at least concurrency") + return self + + +class SearchProfile(StrictModel): + tensor_parallel: PositiveStrictInt = 1 + context_length: PositiveStrictInt = 16384 + max_candidates: int = Field(default=1, strict=True, ge=1, le=2) + objective: Literal[ + "throughput", + "throughput_per_gpu", + "throughput_per_user", + "goodput", + "goodput_per_gpu", + "ttft", + "e2e_latency", + "pareto", + ] = "throughput" + seed: int = Field(default=42, strict=True, ge=0) + + +class SupportRequest(StrictModel): + schema_version: Literal["aisimulate-support-request/v1"] = "aisimulate-support-request/v1" + identity: SupportIdentity + workload: WorkloadSpec = Field(default_factory=WorkloadSpec) + search: SearchProfile = Field(default_factory=SearchProfile) + + @model_validator(mode="after") + def _shape(self) -> SupportRequest: + if self.search.tensor_parallel > self.identity.gpus_per_node: + raise ValueError("search.tensor_parallel must fit within gpus_per_node") + if self.workload.input_tokens + self.workload.output_tokens > self.search.context_length: + raise ValueError("search.context_length must cover the input and output tokens") + return self + + @classmethod + def from_yaml(cls, path: str | Path) -> SupportRequest: + return cls.model_validate(load_yaml(path)) diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py new file mode 100644 index 0000000000..fde9f707e2 --- /dev/null +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -0,0 +1,427 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Public onboarding CLI behavior without a model download or GPU collection.""" + +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import aisimulate.main as cli +from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig +from aisimulate.support.schema import SupportRequest + +pytestmark = pytest.mark.unit + +_REQUIRED = { + "model": "example/unintegrated-model", + "model_revision": "revision-123", + "model_kind": "dense", + "framework_version": "0.24.0", + "gpu": "h200_sxm", + "gpu_count": 4, + "interconnect": "nvswitch", +} +_PILOT = { + "tensor_parallel": 2, + "input_tokens": 1024, + "output_tokens": 128, + "concurrency": 1, + "context_length": 16384, + "ttft_ms": 1000, + "tpot_ms": 100, +} + + +def _init_args(output: Path, *, full: bool = False, **changes) -> list[str]: + options = {**_REQUIRED, **(_PILOT if full else {}), **changes} + return ["support", "init", "--output", str(output)] + [ + part + for name, value in options.items() + if value is not None + for part in ("--" + name.replace("_", "-"), str(value)) + ] + + +def _terminal(monkeypatch, answers=()) -> list[str]: + prompts = [] + remaining = iter(answers) + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + + def respond(prompt: str) -> str: + prompts.append(prompt) + try: + answer = next(remaining) + except StopIteration: + pytest.fail(f"unexpected prompt: {prompt}") + if isinstance(answer, BaseException): + raise answer + return answer + + monkeypatch.setattr("builtins.input", respond) + return prompts + + +def test_guided_and_scripted_setup_produce_the_same_request(tmp_path, monkeypatch, capsys) -> None: + guided = tmp_path / "guided request.yaml" + scripted = tmp_path / "scripted.yaml" + prompts = _terminal( + monkeypatch, + ["example/unintegrated-model", "revision-123", "dense", "0.24.0", "h200_sxm", "4", "nvswitch", "2"] + [""] * 6, + ) + + assert cli.main(["support", "init", "--interactive", "--output", str(guided)]) == 0 + assert cli.main(_init_args(scripted, tensor_parallel=2)) == 0 + + assert SupportRequest.from_yaml(guided) == SupportRequest.from_yaml(scripted) + assert any("Input tokens per request [1024]" in prompt for prompt in prompts) + assert any("Target time to first token (ms) [1000.0]" in prompt for prompt in prompts) + request = SupportRequest.from_yaml(guided) + assert request.workload.request_count == 4 + assert request.identity.gpus_per_node == 4 + assert request.identity.aisimulate_revision is None + assert request.identity.tokenizer_revision is None + output = capsys.readouterr().out + assert "accuracy is not assessed" in output + assert "are unchecked" in output + + +def test_supplied_options_skip_prompts_and_onboarding_alias_is_optional(tmp_path, monkeypatch) -> None: + prompts = _terminal(monkeypatch) + guided = tmp_path / "guided.yaml" + scripted = tmp_path / "scripted.yaml" + options = {"request_count": 8, "seed": 123, "max_candidates": 2, "objective": "goodput"} + + assert cli.main(_init_args(guided, full=True, **options) + ["--interactive", "--profile", "onboarding"]) == 0 + assert cli.main(_init_args(scripted, full=True, **options)) == 0 + + assert prompts == [] + assert SupportRequest.from_yaml(guided) == SupportRequest.from_yaml(scripted) + + +def test_guided_setup_recovers_numeric_input_and_shared_field_validation(tmp_path, monkeypatch, capsys) -> None: + output = tmp_path / "request.yaml" + prompts = _terminal(monkeypatch, ["many", "4", "revision-123"]) + + assert cli.main(_init_args(output, full=True, model_revision="main", gpu_count=None) + ["--interactive"]) == 0 + + request = SupportRequest.from_yaml(output) + assert request.identity.gpu_count == 4 + assert request.identity.model_revision == "revision-123" + assert len(prompts) == 3 + transcript = capsys.readouterr().out + assert "Enter a valid integer" in transcript + assert "declare a pinned revision" in transcript + + +@pytest.mark.parametrize( + ("changes", "answers", "expected"), + [ + ({"tensor_parallel": 8}, ["unknown-option", "--tensor-parallel", "2"], "tensor_parallel"), + ({"context_length": 100}, ["context-length", "4096"], "context_length"), + ({"concurrency": 8}, ["request-count", "8"], "request_count"), + ({"node_count": 2, "gpus_per_node": 4}, ["gpu-count", "8"], "gpu_count"), + ], +) +def test_guided_setup_recovers_cross_field_validation( + tmp_path, monkeypatch, capsys, changes, answers, expected +) -> None: + output = tmp_path / "request.yaml" + prompts = _terminal(monkeypatch, answers) + + assert cli.main(_init_args(output, full=True, **changes) + ["--interactive"]) == 0 + + SupportRequest.from_yaml(output) + assert any("Option to correct" in prompt for prompt in prompts) + assert expected in capsys.readouterr().out + + +@pytest.mark.parametrize( + "changes", + [ + {"model": None}, + {"model_revision": "latest"}, + {"gpu_count": 0}, + {"ttft_ms": "nan"}, + {"request_count": 0}, + {"context_length": 100}, + {"tensor_parallel": 8}, + {"max_candidates": 3}, + {"objective": "invented"}, + {"node_count": 2}, + ], +) +def test_scripted_setup_uses_shared_validation_and_never_writes_invalid_requests(tmp_path, capsys, changes) -> None: + output = tmp_path / "new" / "request.yaml" + + with pytest.raises(SystemExit) as error: + cli.main(_init_args(output, **changes)) + + assert error.value.code == 2 + assert not output.parent.exists() + assert "error:" in capsys.readouterr().err + + +@pytest.mark.parametrize("interruption", [EOFError, KeyboardInterrupt]) +@pytest.mark.parametrize("existing", [False, True]) +def test_cancelled_setup_preserves_files_and_creates_no_partial_request( + tmp_path, monkeypatch, capsys, interruption, existing +) -> None: + output = tmp_path / "new" / "request.yaml" + if existing: + output.parent.mkdir() + output.write_text("existing contents\n") + _terminal(monkeypatch, ["example/unintegrated-model", interruption()]) + + assert cli.main(["support", "init", "--interactive", "--output", str(output), "--overwrite"]) == 130 + + if existing: + assert output.read_text() == "existing contents\n" + assert list(output.parent.iterdir()) == [output] + else: + assert not output.parent.exists() + assert "Setup cancelled" in capsys.readouterr().err + + +def test_interactive_requires_a_terminal_without_reading_input(tmp_path, monkeypatch, capsys) -> None: + _terminal(monkeypatch) + monkeypatch.setattr(sys.stdin, "isatty", lambda: False) + output = tmp_path / "request.yaml" + + with pytest.raises(SystemExit) as error: + cli.main(["support", "init", "--interactive", "--output", str(output)]) + + assert error.value.code == 2 + assert "requires a terminal" in capsys.readouterr().err + assert not output.exists() + + +@pytest.mark.parametrize("target_kind", ["existing_file", "directory", "parent_file"]) +def test_invalid_output_is_rejected_before_prompting(tmp_path, monkeypatch, target_kind) -> None: + _terminal(monkeypatch) + target = tmp_path / "request.yaml" + if target_kind == "existing_file": + target.write_text("keep me") + elif target_kind == "directory": + target.mkdir() + else: + parent = tmp_path / "file" + parent.write_text("keep me") + target = parent / "request.yaml" + + with pytest.raises(SystemExit) as error: + cli.main(["support", "init", "--interactive", "--output", str(target)]) + + assert error.value.code == 2 + + +def test_output_install_does_not_overwrite_a_file_created_during_setup(tmp_path, monkeypatch) -> None: + output = tmp_path / "request.yaml" + real_link = os.link + + def competing_writer(source, destination) -> None: + Path(destination).write_text("concurrent writer\n") + real_link(source, destination) + + monkeypatch.setattr(os, "link", competing_writer) + + with pytest.raises(SystemExit) as error: + cli.main(_init_args(output)) + + assert error.value.code == 2 + assert output.read_text() == "concurrent writer\n" + assert list(tmp_path.iterdir()) == [output] + + +def test_explicit_overwrite_replaces_a_symlink_without_modifying_its_target(tmp_path) -> None: + existing = tmp_path / "existing.yaml" + existing.write_text("keep me\n") + output = tmp_path / "request.yaml" + output.symlink_to(existing) + + assert cli.main(_init_args(output) + ["--overwrite"]) == 0 + + assert not output.is_symlink() + SupportRequest.from_yaml(output) + assert existing.read_text() == "keep me\n" + + +def test_init_next_command_quotes_the_request_path(tmp_path, capsys) -> None: + output = tmp_path / "request 'with spaces'; $(unused).yaml" + + assert cli.main(_init_args(output)) == 0 + + next_command = next( + line.removeprefix("next: ") for line in capsys.readouterr().out.splitlines() if line.startswith("next: ") + ) + assert shlex.split(next_command) == ["aisimulate", "support", "plan", "--config", str(output)] + + +def test_next_command_handles_a_relative_filename_starting_with_a_dash(tmp_path, monkeypatch, capsys) -> None: + monkeypatch.chdir(tmp_path) + args = _init_args(Path("unused.yaml")) + args[2:4] = ["--output=-request.yaml"] + + assert cli.main(args) == 0 + + next_command = next( + line.removeprefix("next: ") for line in capsys.readouterr().out.splitlines() if line.startswith("next: ") + ) + parsed = cli.build_parser().parse_args(shlex.split(next_command)[1:]) + assert Path(parsed.config) == tmp_path / "-request.yaml" + + +@pytest.mark.parametrize("output_dir", ["-plan", "plan 'quoted'; $(unused)"]) +def test_printed_plan_next_command_runs_in_a_shell(tmp_path, monkeypatch, capsys, output_dir) -> None: + monkeypatch.chdir(tmp_path) + request = tmp_path / "request.yaml" + assert cli.main(_init_args(request)) == 0 + capsys.readouterr() + assert cli.main(["support", "plan", "-c", str(request), f"--output-dir={output_dir}", "--format", "json"]) == 0 + summary = json.loads(capsys.readouterr().out) + + result = subprocess.run( + ["/bin/sh", "-c", summary["next"]], + cwd=tmp_path, + env={**os.environ, "PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", "")}, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert Path(summary["plan"]) == tmp_path / output_dir / "support-plan.json" + assert "collector.fpm_forward" in result.stdout + assert "--plan-only" in result.stdout + + +@pytest.mark.parametrize("model_kind", ["dense", "moe"]) +def test_init_plan_and_preview_use_real_public_configs_without_launching_collection( + tmp_path, monkeypatch, capsys, model_kind +) -> None: + def unexpected_launch(*args, **kwargs): + pytest.fail("setup, plan, and preview must not launch a collector") + + monkeypatch.setattr(subprocess, "run", unexpected_launch) + monkeypatch.setitem(sys.modules, "collector.fpm_forward.cli", SimpleNamespace(main=unexpected_launch)) + monkeypatch.setattr(cli, "resolve_runner_factory", unexpected_launch) + request_path = tmp_path / "request 'quoted'.yaml" + output = tmp_path / "plan 'quoted'" + assert cli.main(_init_args(request_path, model_kind=model_kind, tensor_parallel=2)) == 0 + capsys.readouterr() + + assert ( + cli.main(["support", "plan", "--config", str(request_path), "--output-dir", str(output), "--format", "json"]) + == 0 + ) + summary = json.loads(capsys.readouterr().out) + plan = json.loads((output / "support-plan.json").read_text()) + assert summary["request_id"] == plan["request_id"] + assert summary["candidate_count"] == 1 + prediction = CorePredictionConfig.from_yaml(plan["outputs"]["prediction_configs"][0]) + recommendation = CoreRecommendationConfig.from_yaml(plan["outputs"]["recommendation_configs"][0]) + assert prediction.engine.workers.aggregated.timing.forward_model == "fpm" + assert recommendation.engine.workers.aggregated.timing.forward_model == "fpm" + assert prediction.engine.systems_path == plan["outputs"]["systems_root"] + assert recommendation.engine.systems_path == prediction.engine.systems_path + assert prediction.traffic.source.input_tokens == 1024 + + next_command = shlex.split(summary["next"]) + assert next_command[:3] == ["aisimulate", "support", "collect-fpm"] + assert cli.main(next_command[1:]) == 0 + preview = capsys.readouterr().out + assert "collector.fpm_forward" in preview + assert "--fpm-max-gpus 2 --fpm-gpu-counts 2" in preview + assert "--fpm-parallel-presets " + ("pure_tp" if model_kind == "moe" else "tp") in preview + + +def test_explicit_execute_forwards_diagnostic_options_and_exit_status(tmp_path, monkeypatch) -> None: + calls = [] + + def collector_main(argv): + calls.append(argv) + return 7 + + monkeypatch.setitem(sys.modules, "collector.fpm_forward.cli", SimpleNamespace(main=collector_main)) + request_path = tmp_path / "request.yaml" + output = tmp_path / "plan" + assert cli.main(_init_args(request_path, tensor_parallel=2)) == 0 + assert cli.main(["support", "plan", "-c", str(request_path), "--output-dir", str(output)]) == 0 + + assert ( + cli.main( + [ + "support", + "collect-fpm", + "-c", + str(request_path), + "--output-dir", + str(output), + "--execute", + "--smoke", + "--limit", + "1", + "--resume", + ] + ) + == 7 + ) + + assert len(calls) == 1 + assert "--plan-only" not in calls[0] + assert "--smoke" in calls[0] + assert "--resume" in calls[0] + assert calls[0][calls[0].index("--limit") + 1] == "1" + + +@pytest.mark.parametrize("contents", ["[one, two]\n", "identity: [\n", "identity: {}\n"]) +def test_bad_request_yaml_is_reported_without_a_traceback(tmp_path, capsys, contents) -> None: + request = tmp_path / "bad.yaml" + request.write_text(contents) + + with pytest.raises(SystemExit) as error: + cli.main(["support", "plan", "--config", str(request), "--output-dir", str(tmp_path / "plan")]) + + assert error.value.code == 2 + assert "Traceback" not in capsys.readouterr().err + assert not (tmp_path / "plan").exists() + + +@pytest.mark.parametrize("command", ["predict", "recommend"]) +def test_ordinary_cli_options_and_numerical_defaults_are_preserved(command) -> None: + args = cli.build_parser().parse_args([command, "--config", "ordinary.yaml"]) + assert args.command == command + assert args.stack == "engine" + assert args.output_dir == "./aisimulate-output" + assert args.overrides == [] + assert args.overwrite is False + assert args.format == "table" + config = CorePredictionConfig.model_validate( + {"engine": {"model": "example/model", "hardware": "h200_sxm", "workers": {"aggregated": {}}}} + ) + assert config.engine.workers.aggregated.timing.forward_model == "op_level" + assert config.traffic.load.concurrency == 10 + assert config.traffic.stop.requests == 100 + + +def test_installed_module_scripted_init_works_without_a_terminal(tmp_path) -> None: + output = tmp_path / "request.yaml" + result = subprocess.run( + [sys.executable, "-m", "aisimulate.main", *_init_args(output)], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert SupportRequest.from_yaml(output).identity.model == _REQUIRED["model"] + assert "next:" in result.stdout diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py new file mode 100644 index 0000000000..7b5807f829 --- /dev/null +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -0,0 +1,610 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The onboarding plan stays usable before model integration or GPU collection.""" + +from __future__ import annotations + +import builtins +import json +import os +import shlex +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig +from aisimulate.support.fpm import run_fpm +from aisimulate.support.plan import create_plan, request_id +from aisimulate.support.schema import SupportRequest + +pytestmark = pytest.mark.unit + + +class _OfflineRunner: + def run(self, spec): + from aisimulate.sweeper.replay import ReplayReport + + return ReplayReport(metrics={"output_throughput_tok_s": 10.0, "gpu_hours": 1.0}) + + def close(self): + pass + + +class _OfflineRunnerFactory: + def capabilities(self): + from aisimulate.sweeper.replay import RunnerCapabilities + + return RunnerCapabilities(supported_backend_topologies=(("vllm", "agg"),)) + + def create(self, worker_id): + return _OfflineRunner() + + +def _request(**updates) -> SupportRequest: + payload = { + "identity": { + "model": "unregistered/Example Model", + "model_revision": "checkpoint-2026-09-14", + "model_kind": "moe", + "framework_version": "0.25.1", + "gpu": "h200_sxm", + "gpu_count": 12, + "node_count": 2, + "gpus_per_node": 6, + "interconnect": "NVLink", + }, + "search": {"tensor_parallel": 4}, + } + for section, values in updates.items(): + payload.setdefault(section, {}).update(values) + return SupportRequest.model_validate(payload) + + +@pytest.mark.parametrize( + "updates", + [ + {"identity": {"model_kind": "auto"}}, + {"identity": {"model_revision": "main"}}, + {"identity": {"framework_version": " "}}, + {"identity": {"gpu": "../h200_sxm"}}, + {"identity": {"gpu_count": 13}}, + {"search": {"tensor_parallel": 8}}, + {"search": {"tensor_parallel": True}}, + {"search": {"max_candidates": 3}}, + {"search": {"context_length": 1151}}, + {"workload": {"input_tokens": 0}}, + {"workload": {"request_count": 0}}, + {"workload": {"concurrency": 5}}, + {"workload": {"slo": {"ttft_ms": float("inf"), "tpot_ms": 1.0}}}, + {"workload": {"slo": {"ttft_ms": 1.0, "tpot_ms": True}}}, + ], +) +def test_request_rejects_invalid_allocation_workload_and_identity(updates): + with pytest.raises(ValidationError): + _request(**updates) + + +def test_plan_does_not_resolve_unknown_model_and_reloads_public_configs(tmp_path, monkeypatch): + real_import = builtins.__import__ + + def no_model_or_collector_import(name, *args, **kwargs): + assert not name.startswith(("collector", "huggingface_hub", "transformers")), name + assert ".sdk.models" not in name, name + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_model_or_collector_import) + request = _request() + root = tmp_path / "plan with 'quotes'" + plan = create_plan(request, root) + prediction = CorePredictionConfig.from_yaml(root / "predict/pilot.yaml") + recommendation = CoreRecommendationConfig.from_yaml(root / "recommend/pilot.yaml") + assert SupportRequest.from_yaml(root / "request.yaml") == request + assert json.loads((root / "support-plan.json").read_text()) == plan + assert prediction.engine.systems_path == str(root / "systems") + assert recommendation.engine.systems_path == prediction.engine.systems_path + assert prediction.engine.model == "unregistered/Example Model" + worker = prediction.engine.workers.aggregated + assert worker.timing.type == "default" + assert worker.timing.forward_model == "fpm" + assert worker.parallelism.model_dump() == { + "replicas": 1, + "tensor": 4, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 4, + "moe_expert": 1, + } + assert recommendation.engine.workers.aggregated.timing.forward_model == "fpm" + assert recommendation.optimizer.max_trials == 1 + assert recommendation.optimizer.parallelism == 1 + assert plan["search"]["candidate_count"] == 1 + assert "single" in plan["search"]["detail"] + assert {item["status"] for item in plan["prerequisites"]} == {"not_checked"} + assert plan["accuracy"]["status"] == "not_assessed" + assert (root / "systems/h200_sxm.yaml").is_file() + assert not list((root / "systems/data").iterdir()) + assert not (root / "evidence.yaml").exists() + + +@pytest.mark.parametrize("model_kind,preset,moe_tensor", [("dense", "tp", 1), ("moe", "pure_tp", 4)]) +def test_plan_collects_one_chosen_worker_and_bounds_recommendation(tmp_path, model_kind, preset, moe_tensor): + from aisimulate.recommend import recommendation_to_sweeper + + request = _request(identity={"model_kind": model_kind}, search={"max_candidates": 2}) + plan = create_plan(request, tmp_path) + configs = [CoreRecommendationConfig.from_yaml(path) for path in plan["outputs"]["recommendation_configs"]] + candidates = [candidate for config in configs for candidate in config.engine.workers.aggregated.parallelism.preset] + assert [(c.replicas, c.tensor, c.moe_tensor) for c in candidates] == [(1, 4, moe_tensor), (2, 4, moe_tensor)] + assert all(config.optimizer.max_trials == 1 for config in configs) + assert plan["search"]["candidate_count"] == 2 + for config in configs: + search = recommendation_to_sweeper(config).search_space + assert search.agg_max_num_batched_tokens == [8192] + assert search.agg_max_num_seqs == [256] + commands = json.loads((tmp_path / "commands.json").read_text()) + command = commands["fpm_run_local"] + assert command[command.index("--fpm-gpu-counts") + 1] == "4" + assert command[command.index("--fpm-max-gpus") + 1] == "4" + assert command[command.index("--fpm-parallel-presets") + 1] == preset + assert command[command.index("--fpm-max-prefill-isl") + 1] == "1024" + assert command[command.index("--fpm-max-prefill-batch-size") + 1] == "1" + assert "--plan-only" not in command + assert "--plan-only" in commands["fpm_plan_local"] + + +def test_one_worker_allocation_keeps_one_pilot_even_with_two_candidate_limit(tmp_path): + request = _request(identity={"node_count": 1, "gpus_per_node": 4, "gpu_count": 4}, search={"max_candidates": 2}) + plan = create_plan(request, tmp_path) + commands = json.loads((tmp_path / "commands.json").read_text()) + + assert plan["search"]["candidate_count"] == 1 + assert plan["outputs"]["recommendation_configs"] == [str(tmp_path / "recommend/pilot.yaml")] + assert len(commands["recommend"]) == 1 + assert "single" in plan["search"]["detail"] + + +@pytest.mark.parametrize("seed", [0, 7, 42]) +@pytest.mark.parametrize("model_kind,moe_tensor", [("dense", 1), ("moe", 4)]) +def test_emitted_recommendations_evaluate_both_replica_choices(tmp_path, monkeypatch, seed, model_kind, moe_tensor): + from aisimulate.main import build_parser + from aisimulate.recommend import run_recommendation + from aisimulate.sweeper.parallel_enum import ParallelShape, ReplicaParallelConfig + + # Supply only offline model/runtime/data legality. Both replica choices are + # legal for every command; the real config lowering, sampler and budget run. + legal = [ + ReplicaParallelConfig(shape=ParallelShape(tp=4, pp=1, dp=1, moe_tp=moe_tensor, moe_ep=1), replicas=count) + for count in (1, 2) + ] + monkeypatch.setattr("aisimulate.sweeper.search_space.parallel_configs_for", lambda *args, **kwargs: legal) + request = _request(identity={"model_kind": model_kind}, search={"max_candidates": 2, "seed": seed}) + plan = create_plan(request, tmp_path) + commands = json.loads((tmp_path / "commands.json").read_text())["recommend"] + evaluated = [] + results = [] + configs = [] + outputs = [] + for command in commands: + args = build_parser().parse_args(command[1:]) + config = CoreRecommendationConfig.from_yaml(args.config) + result = run_recommendation(config, stack="engine", runner_factory=_OfflineRunnerFactory(), show_progress=False) + evaluated.extend(candidate.config["replicas"] for candidate in result.candidates) + results.append(result) + configs.append(args.config) + outputs.append(args.output_dir) + assert evaluated == [1, 2] + assert configs == plan["outputs"]["recommendation_configs"] + assert outputs == plan["outputs"]["recommendation_results"] + assert len(set(outputs)) == 2 + assert all(result.counts.evaluated == 1 and result.counts.cache_hits == 0 for result in results) + + +@pytest.mark.parametrize( + "max_candidates,missing_pilot_samples,statuses", + [(1, False, [0]), (2, False, [0, 0]), (2, True, [1, 0])], +) +def test_documented_recommendation_loop_attempts_every_candidate( + tmp_path, max_candidates, missing_pilot_samples, statuses +): + guide = Path(__file__).resolve().parents[4] / "docs/self-service-support.md" + snippet = guide.read_text().split("python3 - <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] + workdir = tmp_path / "work with 'quotes'; $(literal)" + workdir.mkdir() + request = _request( + identity={"model_kind": "dense", "gpu_count": 8, "node_count": 1, "gpus_per_node": 8}, + search={"max_candidates": max_candidates, "objective": "ttft"}, + ) + plan = create_plan(request, workdir / "aisimulate-support") + commands = json.loads(Path(plan["outputs"]["commands"]).read_text())["recommend"] + evaluated = tmp_path / "evaluated.txt" + wrapper = tmp_path / "bin/aisimulate" + wrapper.parent.mkdir() + # Substitute only offline legality and replay metrics. The snippet executes + # real subprocesses, public CLI dispatch, sampling, scoring and output export. + wrapper.write_text( + f"#!{sys.executable}\n" + f""" +import os +import sys +from pathlib import Path + +import aisimulate.main as cli +import aisimulate.sweeper.search_space as search_space +from aisimulate.sweeper.parallel_enum import ParallelShape, ReplicaParallelConfig +from aisimulate.sweeper.replay import ReplayReport, RunnerCapabilities + +legal = [ + ReplicaParallelConfig(shape=ParallelShape(tp=4, pp=1, dp=1, moe_tp=1, moe_ep=1), replicas=count) + for count in (1, 2) +] +search_space.parallel_configs_for = lambda *args, **kwargs: legal + +class Runner: + def run(self, spec): + replicas = spec.backend_deployment.num_workers + with Path(os.environ["SUPPORT_TEST_EVALUATED"]).open("a") as stream: + stream.write(str(replicas) + "\\n") + samples = 0.0 if {missing_pilot_samples!r} and replicas == 1 else 4.0 + return ReplayReport(metrics={{"mean_ttft_ms": 10.0, "num_ttft_samples": samples}}) + + def close(self): + pass + +class Factory: + def capabilities(self): + return RunnerCapabilities(supported_backend_topologies=(("vllm", "agg"),)) + + def create(self, worker_id): + return Runner() + +cli.resolve_runner_factory = lambda stack: Factory() +if __name__ == "__main__": + raise SystemExit(cli.main(sys.argv[1:])) +""" + ) + wrapper.chmod(0o755) + result = subprocess.run( + [sys.executable, "-c", snippet], + cwd=workdir, + env={ + **os.environ, + "PATH": str(wrapper.parent) + os.pathsep + os.environ.get("PATH", ""), + "SUPPORT_TEST_EVALUATED": str(evaluated), + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert result.returncode == (1 if any(statuses) else 0), result.stderr + assert evaluated.read_text().splitlines() == [str(index) for index in range(1, max_candidates + 1)] + for index, (output, status) in enumerate(zip(plan["outputs"]["recommendation_results"], statuses, strict=True), 1): + root = Path(output) + payload = json.loads((root / "recommendation.json").read_text()) + assert payload["counts"]["evaluated"] == 1 + assert payload["counts"]["feasible"] == (0 if status else 1) + assert payload["counts"]["infeasible"] == (1 if status else 0) + assert payload["candidates"][0]["config"]["replicas"] == index + exported = root / "recommendations/0001.yaml" + if status: + assert "no qualifying samples" in payload["candidates"][0]["reason"] + assert not exported.exists() + else: + prediction = CorePredictionConfig.from_yaml(exported) + assert prediction.engine.workers.aggregated.parallelism.replicas == index + assert prediction.engine.systems_path == str(workdir / "aisimulate-support/systems") + assert [line for line in result.stdout.splitlines() if line.startswith("exit ")] == [ + f"exit {status}: {shlex.join(command)}" for command, status in zip(commands, statuses, strict=True) + ] + + +def test_prefill_bounds_follow_concurrent_workload_and_collector_minimum(tmp_path): + request = _request(workload={"input_tokens": 10, "concurrency": 3, "request_count": 4}) + command = create_plan(request, tmp_path)["fpm"]["plan_command"] + assert command[command.index("--fpm-max-prefill-isl") + 1] == "30" + assert command[command.index("--fpm-max-prefill-batch-size") + 1] == "3" + small = _request(workload={"input_tokens": 1}) + command = create_plan(small, tmp_path / "small")["fpm"]["plan_command"] + assert command[command.index("--fpm-max-prefill-isl") + 1] == "2" + + +@pytest.mark.parametrize( + "updates", + [ + {"search": {"tensor_parallel": 2}}, + {"search": {"max_candidates": 2}}, + {"search": {"seed": 7}}, + {"workload": {"input_tokens": 64}}, + {"identity": {"model_revision": "other-checkpoint"}}, + {"identity": {"framework_version": "0.26.0"}}, + {"identity": {"tokenizer_revision": "tokenizer-v2"}}, + ], +) +def test_overwrite_rejects_different_requests_without_touching_data(tmp_path, updates): + request = _request() + create_plan(request, tmp_path) + data = tmp_path / "systems/data/collected.parquet" + data.write_bytes(b"timings") + before = {str(p.relative_to(tmp_path)): p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()} + changed = _request(**updates) + assert request_id(changed) != request_id(request) + with pytest.raises(ValueError, match="identity|different request|mix"): + create_plan(changed, tmp_path, overwrite=True) + after = {str(p.relative_to(tmp_path)): p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()} + assert before == after + + +def test_same_request_overwrite_preserves_timings_and_rejects_modified_inputs(tmp_path): + request = _request() + plan = create_plan(request, tmp_path) + data = tmp_path / "systems/data/collected.parquet" + data.write_bytes(b"timings") + with pytest.raises(ValueError, match="nonempty|already exists|overwrite"): + create_plan(request, tmp_path) + assert create_plan(request, tmp_path, overwrite=True) == plan + assert data.read_bytes() == b"timings" + (tmp_path / "request.yaml").write_text("modified: true\n") + with pytest.raises(ValueError, match="modified|request|identity"): + create_plan(request, tmp_path, overwrite=True) + assert data.read_bytes() == b"timings" + + +def test_new_gpu_is_rejected_before_creating_outputs(tmp_path): + root = tmp_path / "new-plan" + with pytest.raises(ValueError, match="packaged.*system|system specification"): + create_plan(_request(identity={"gpu": "unreleased_gpu"}), root) + assert not root.exists() + + +def test_preview_is_shell_safe_and_does_not_create_outputs_or_import_collector(tmp_path, monkeypatch, capsys): + request = _request() + root = tmp_path / "plan with 'quotes'" + real_import = builtins.__import__ + + def no_collector_import(name, *args, **kwargs): + assert not name.startswith("collector"), name + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_collector_import) + assert run_fpm(request, output_dir=root) == 0 + command = shlex.split(capsys.readouterr().out) + assert command[1:3] == ["-m", "collector.fpm_forward"] + assert command[command.index("--model-path") + 1] == request.identity.model + assert command[command.index("--fpm-database-root") + 1] == str(root / "systems/data") + assert "--plan-only" in command + assert not root.exists() + + +def test_execute_delegates_only_for_matching_plan(tmp_path, monkeypatch): + from collector.fpm_forward import cli + + calls = [] + monkeypatch.setattr(cli, "main", lambda argv: calls.append(argv) or 7) + request = _request() + with pytest.raises(ValueError, match="plan"): + run_fpm(request, execute=True, output_dir=tmp_path) + assert not calls + create_plan(request, tmp_path) + assert run_fpm(request, execute=True, output_dir=tmp_path, smoke=True, limit=1) == 7 + assert calls[0][calls[0].index("--limit") + 1] == "1" + assert "--smoke" in calls[0] + assert "--plan-only" not in calls[0] + with pytest.raises(ValueError, match="identity|different request|mix"): + run_fpm(_request(search={"tensor_parallel": 2}), execute=True, resume=True, output_dir=tmp_path) + assert len(calls) == 1 + + +def _seed_campaign(root, *, smoke=False, checkpoint_dir=None): + sha = "a" * 64 + artifact = root / "fpm-artifacts" / sha[:16] + if smoke: + artifact /= "smoke" + timing = artifact / "cells/cell-prefill/raw/existing-timing.json" + timing.parent.mkdir(parents=True) + timing.write_bytes(b"previous measured data") + checkpoint = (checkpoint_dir or root / "fpm-checkpoint") / ( + "fpm_forward_smoke.json" if smoke else "fpm_forward.json" + ) + checkpoint.parent.mkdir(parents=True, exist_ok=True) + checkpoint.write_text(json.dumps({"schema": "aic-fpm-collector-checkpoint-v3", "plan_sha256": sha, "cells": {}})) + if not smoke: + (root / "systems/data/collected.parquet").write_bytes(b"formal timings") + return checkpoint, timing + + +def _file_contents(root): + return {str(path.relative_to(root)): path.read_bytes() for path in root.rglob("*") if path.is_file()} + + +def _reject_collector_import(monkeypatch): + real_import = builtins.__import__ + + def no_collector_import(name, *args, **kwargs): + assert not name.startswith("collector"), "occupied campaigns must be rejected before collector import" + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_collector_import) + + +@pytest.mark.parametrize("smoke", [False, True]) +def test_repeated_execute_requires_resume_without_touching_campaign(tmp_path, monkeypatch, smoke): + create_plan(_request(), tmp_path) + _seed_campaign(tmp_path, smoke=smoke) + before = _file_contents(tmp_path) + _reject_collector_import(monkeypatch) + + with pytest.raises(ValueError, match="resume.*new output|new output.*resume"): + run_fpm(SupportRequest.from_yaml(tmp_path / "request.yaml"), output_dir=tmp_path, execute=True, smoke=smoke) + + assert _file_contents(tmp_path) == before + assert not (tmp_path / ".support.lock").exists() + + +@pytest.mark.parametrize( + "smoke,relative", + [ + (False, "systems/data/collected.parquet"), + (False, "fpm-checkpoint/custom/fpm_forward.json"), + (True, "fpm-checkpoint/custom/fpm_forward_smoke.json"), + (False, "fpm-artifacts/aaaaaaaaaaaaaaaa/cells/raw.json"), + (True, "fpm-artifacts/aaaaaaaaaaaaaaaa/smoke/cells/raw.json"), + ], +) +def test_partial_campaign_output_alone_prevents_restart(tmp_path, monkeypatch, smoke, relative): + request = _request() + create_plan(request, tmp_path) + sentinel = tmp_path / relative + sentinel.parent.mkdir(parents=True, exist_ok=True) + sentinel.write_bytes(b"interrupted campaign") + before = _file_contents(tmp_path) + _reject_collector_import(monkeypatch) + + with pytest.raises(ValueError, match="resume"): + run_fpm(request, output_dir=tmp_path, execute=True, smoke=smoke) + + assert _file_contents(tmp_path) == before + + +@pytest.mark.parametrize("smoke", [False, True]) +@pytest.mark.parametrize("checkpoint_state", ["missing", "invalid_json", "empty_object", "new_directory"]) +def test_resume_occupied_campaign_requires_usable_selected_checkpoint(tmp_path, monkeypatch, smoke, checkpoint_state): + request = _request() + create_plan(request, tmp_path) + checkpoint, _ = _seed_campaign(tmp_path, smoke=smoke) + selected_dir = checkpoint.parent + if checkpoint_state == "missing": + checkpoint.unlink() + elif checkpoint_state == "new_directory": + selected_dir /= "new-campaign" + else: + checkpoint.write_text("invalid JSON" if checkpoint_state == "invalid_json" else "{}") + before = _file_contents(tmp_path) + _reject_collector_import(monkeypatch) + + with pytest.raises(ValueError, match="checkpoint"): + run_fpm(request, output_dir=tmp_path, execute=True, smoke=smoke, resume=True, checkpoint_dir=selected_dir) + + assert _file_contents(tmp_path) == before + assert not (tmp_path / ".support.lock").exists() + + +@pytest.mark.parametrize("smoke", [False, True]) +def test_matching_explicit_resume_delegates_and_preserves_campaign(tmp_path, monkeypatch, smoke): + from collector.fpm_forward import cli + + request = _request() + create_plan(request, tmp_path) + checkpoint, _ = _seed_campaign(tmp_path, smoke=smoke, checkpoint_dir=tmp_path / "fpm-checkpoint/custom") + before = _file_contents(tmp_path) + calls = [] + monkeypatch.setattr(cli, "main", lambda argv: calls.append(argv) or 7) + + assert ( + run_fpm(request, output_dir=tmp_path, execute=True, smoke=smoke, resume=True, checkpoint_dir=checkpoint.parent) + == 7 + ) + + assert "--resume" in calls[0] + assert calls[0][calls[0].index("--checkpoint-dir") + 1] == str(checkpoint.parent) + assert _file_contents(tmp_path) == before + + +@pytest.mark.parametrize("smoke", [False, True]) +def test_fresh_smoke_and_formal_campaigns_are_independent(tmp_path, monkeypatch, smoke): + from collector.fpm_forward import cli + + request = _request() + create_plan(request, tmp_path) + _seed_campaign(tmp_path, smoke=not smoke) + before = _file_contents(tmp_path) + calls = [] + monkeypatch.setattr(cli, "main", lambda argv: calls.append(argv) or 7) + + assert run_fpm(request, output_dir=tmp_path, execute=True, smoke=smoke) == 7 + + assert len(calls) == 1 + assert _file_contents(tmp_path) == before + + +def test_execute_checks_for_campaign_outputs_while_holding_lock(tmp_path, monkeypatch): + from aisimulate.support import plan + + request = _request() + create_plan(request, tmp_path) + real_lock = plan.plan_lock + + @contextmanager + def raced_lock(root): + with real_lock(root): + _seed_campaign(root) + yield + + monkeypatch.setattr(plan, "plan_lock", raced_lock) + _reject_collector_import(monkeypatch) + + with pytest.raises(ValueError, match="resume"): + run_fpm(request, output_dir=tmp_path, execute=True) + + assert (tmp_path / "systems/data/collected.parquet").read_bytes() == b"formal timings" + assert not (tmp_path / ".support.lock").exists() + + +@pytest.mark.parametrize("smoke,limit", [(False, 1), (True, 0), (True, -1), (True, True)]) +def test_invalid_limited_collection_fails_before_preview_or_execution(tmp_path, smoke, limit): + with pytest.raises(ValueError, match="limit"): + run_fpm(_request(), output_dir=tmp_path, smoke=smoke, limit=limit) + + +def test_plan_rejects_symlinked_output_without_touching_target(tmp_path): + external = tmp_path / "external" + external.mkdir() + data = external / "collected.parquet" + data.write_bytes(b"timings") + root = tmp_path / "plan" + create_plan(_request(), root) + (root / "systems/data").rmdir() + (root / "systems/data").symlink_to(external, target_is_directory=True) + with pytest.raises(ValueError, match="symlink"): + create_plan(_request(), root, overwrite=True) + assert data.read_bytes() == b"timings" + + +@pytest.mark.parametrize("relative", ["../shared", "fpm-checkpoint/../../shared"]) +def test_checkpoint_override_cannot_escape_request_directory(tmp_path, relative): + with pytest.raises(ValueError, match="checkpoint_dir"): + run_fpm(_request(), output_dir=tmp_path, checkpoint_dir=tmp_path / relative) + + +def test_concurrent_plan_operation_preserves_existing_files(tmp_path): + request = _request() + create_plan(request, tmp_path) + (tmp_path / ".support.lock").write_text("busy") + before = (tmp_path / "support-plan.json").read_bytes() + with pytest.raises(ValueError, match="another support operation"): + create_plan(request, tmp_path, overwrite=True) + assert (tmp_path / "support-plan.json").read_bytes() == before + assert (tmp_path / ".support.lock").read_text() == "busy" + + +def test_racing_writer_cannot_be_overwritten_or_reported_as_success(tmp_path, monkeypatch): + original_open = Path.open + destination = tmp_path / "predict/pilot.yaml" + + def racing_open(path, mode="r", *args, **kwargs): + if path == destination and mode == "xb": + with original_open(path, "wb") as handle: + handle.write(b"another writer's config") + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", racing_open) + with pytest.raises(FileExistsError): + create_plan(_request(), tmp_path) + assert destination.read_bytes() == b"another writer's config" + assert not (tmp_path / "support-plan.json").exists() + assert not (tmp_path / ".support.lock").exists() From ecb6ee5ad76847e7784ef1a72c7d2ef0dafaedde Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 13:12:11 +0800 Subject: [PATCH 03/27] fix: guard guided FPM collection and recover interrupted plans Keep generated collection commands behind support guards, bind validated deployment options to collector resume identity, release plan locks on process exit, classify execution failures, and construct trusted recommendation commands in the guide. Refs: AIC-1963 Signed-off-by: Yiming Liu --- docs/self-service-support.md | 29 ++- .../aisimulate/collector/fpm_forward/cli.py | 2 +- .../aisimulate/src/aisimulate/support/cli.py | 10 +- .../aisimulate/src/aisimulate/support/fpm.py | 20 +- .../aisimulate/src/aisimulate/support/plan.py | 32 +++- .../src/aisimulate/support/schema.py | 39 +++- .../aisimulate/tests/unit/test_support_cli.py | 177 ++++++++++++++++++ .../tests/unit/test_support_plan.py | 121 ++++++++++-- 8 files changed, 396 insertions(+), 34 deletions(-) diff --git a/docs/self-service-support.md b/docs/self-service-support.md index f911fc54e3..8671e585f6 100644 --- a/docs/self-service-support.md +++ b/docs/self-service-support.md @@ -53,7 +53,7 @@ The first command saves the request, `support-plan.json`, `commands.json`, `pred The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. -Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` includes the underlying collector command for reviewing or extending it with the collector's deployment flags. +Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate support collect-fpm --execute` command for collection, alongside a read-only collector planning command. After those prerequisites are ready, explicitly launch collection from that environment: @@ -65,8 +65,12 @@ aisimulate support collect-fpm \ Execution requires a matching saved plan. The request records model and runtime revisions; this setup does not download a pinned checkpoint or verify the installed runtime against them. Keep the actual checkpoint and runtime consistent with the request before collecting or predicting. +Set deployment options directly on `support collect-fpm`: `--dynamo-version VERSION`, `--image IMAGE`, `--namespace NAME`, `--model-cache NAME[:MOUNT[:SUBPATH]]`, `--transport nvlink|ib|efa`, and `--image-pull-secret NAME`. The mount, when supplied, is an absolute container path. Prefer an immutable image digest. Supply the same options when previewing, executing, and resuming; deployment settings are part of the collector's frozen-plan identity, so changed settings require a new output directory. Arbitrary collector arguments and engine overrides are not accepted by this command. + For a diagnostic run, add `--execute --smoke`; `--limit N` also requires `--smoke`. Diagnostic smoke and limited runs do not publish formal FPM data. Existing campaign data, raw artifacts, or checkpoints require explicit `--resume` and a readable matching collector checkpoint; otherwise choose a new output directory. A custom `--checkpoint-dir`, if needed, must remain inside the plan's `fpm-checkpoint/` directory. Selecting an empty checkpoint directory does not allow reuse of existing campaign artifacts. Smoke and formal campaigns have separate checkpoints and artifact directories, so an existing smoke run does not prevent the first formal run, or vice versa. The collector verifies the resumed checkpoint's frozen-plan identity. +Planning and collection reject concurrent support operations. The persistent `.support.lock` file uses an OS advisory lock; ownership is released when the process exits, including after an abrupt termination. Leave the file in place. Request validation, saved support-plan checks, and collector input resolution exit 2. Failures after collector execution starts, including a frozen checkpoint identity mismatch, exit 1 with a concise message. Interruption exits 130. + The collector narrows initial prefill sampling with the pilot's input-token and concurrency bounds. Decode uses the collector's existing profile; a four-request synthetic pilot does not imply four timing samples or a short decode campaign. Inspect the generated command and collector plan before committing GPU time. Successful formal collection publishes the FPM Parquet file and metadata pair into the plan's local systems data directory; diagnostic success alone does not provide that pair. ## Run the generated ordinary configurations @@ -83,18 +87,31 @@ aisimulate recommend \ --output-dir ./aisimulate-support/recommend-results/pilot ``` -With two candidates, run every recommendation command in `commands.json` so both replica counts are evaluated. This executes the emitted ordinary command vectors: +With two candidates, run both replica counts. This loop derives the candidate names from the validated saved request and constructs fixed `aisimulate recommend` commands: ```bash python3 - <<'PY' -import json import shlex import subprocess from pathlib import Path -commands = json.loads(Path("./aisimulate-support/commands.json").read_text()) +from aisimulate.support.plan import check_plan +from aisimulate.support.schema import SupportRequest + +root = Path("./aisimulate-support").resolve() +request = SupportRequest.from_yaml(root / "request.yaml") +check_plan(request, root) +max_replicas = request.identity.node_count * (request.identity.gpus_per_node // request.search.tensor_parallel) +replicas = list(dict.fromkeys((1, max_replicas)))[:request.search.max_candidates] statuses = [] -for command in commands["recommend"]: +for count in replicas: + name = "pilot" if count == 1 else f"replicas-{count}" + command = [ + "aisimulate", "recommend", + "--config", str(root / f"recommend/{name}.yaml"), + "--output-dir", str(root / f"recommend-results/{name}"), + "--format", "json", + ] result = subprocess.run(command, check=False) statuses.append(result.returncode) print(f"exit {result.returncode}: {shlex.join(command)}", flush=True) @@ -104,7 +121,7 @@ PY The loop reports each command's exit status and attempts all commands, including when the pilot finds no feasible candidate. It exits 1 after all attempts if any command returned a nonzero status, otherwise 0. Successful outputs remain usable even when the loop exits 1; inspect each result before comparing candidates. -Run either the individual recommendation command or the command-vector loop against fresh result directories. The loop also handles a one-candidate plan. Results remain separate under `recommend-results/pilot` and, when present, `recommend-results/replicas-N`; compare their objective and latency results for the same workload. This plan does not produce a combined ranking or search additional TP sizes or scheduler settings. +Run either the individual recommendation command or the loop against fresh result directories. The loop also handles a one-candidate plan and does not execute entries from `commands.json`. Results remain separate under `recommend-results/pilot` and, when present, `recommend-results/replicas-N`; compare their objective and latency results for the same workload. This plan does not produce a combined ranking or search additional TP sizes or scheduler settings. The generated configurations select `engine.workers.aggregated.timing.forward_model: fpm` and set `engine.systems_path` to the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. diff --git a/python/aisimulate/collector/fpm_forward/cli.py b/python/aisimulate/collector/fpm_forward/cli.py index fd674867f2..7096f59335 100644 --- a/python/aisimulate/collector/fpm_forward/cli.py +++ b/python/aisimulate/collector/fpm_forward/cli.py @@ -16,7 +16,7 @@ from .config import add_fpm_arguments, add_fpm_generator_arguments from .entry import resolve_inputs, resolve_run_inputs, run_resolved -_INPUT_ERRORS = (FileNotFoundError, RuntimeError, TypeError, ValueError) +_INPUT_ERRORS = (OSError, RuntimeError, TypeError, ValueError) def _parser() -> argparse.ArgumentParser: diff --git a/python/aisimulate/src/aisimulate/support/cli.py b/python/aisimulate/src/aisimulate/support/cli.py index 0cb56b974f..0653c68c59 100644 --- a/python/aisimulate/src/aisimulate/support/cli.py +++ b/python/aisimulate/src/aisimulate/support/cli.py @@ -19,7 +19,7 @@ from .fpm import run_fpm from .plan import create_plan, request_id -from .schema import SearchProfile, SloSpec, SupportIdentity, SupportRequest, WorkloadSpec +from .schema import FPMDeployment, SearchProfile, SloSpec, SupportIdentity, SupportRequest, WorkloadSpec def add_support_parser(subparsers: Any) -> None: @@ -92,6 +92,13 @@ def add_support_parser(subparsers: Any) -> None: ) collect.add_argument("--resume", action="store_true", help="Resume the existing collector checkpoint.") collect.add_argument("--checkpoint-dir", help="Checkpoint directory inside the plan's fpm-checkpoint directory.") + deployment = collect.add_argument_group("Collector deployment") + deployment.add_argument("--dynamo-version", help="Target Dynamo release used to resolve collector templates.") + deployment.add_argument("--image", help="Collector container image; prefer an immutable digest.") + deployment.add_argument("--namespace", help="Kubernetes namespace for collector resources.") + deployment.add_argument("--model-cache", metavar="NAME[:MOUNT[:SUBPATH]]", help="Model-cache PVC and mount.") + deployment.add_argument("--transport", choices=("nvlink", "ib", "efa")) + deployment.add_argument("--image-pull-secret", help="Kubernetes secret for pulling the collector image.") def _values(args: argparse.Namespace, model: Any) -> dict[str, Any]: @@ -311,5 +318,6 @@ def run_support_command(args: argparse.Namespace) -> int: limit=args.limit, resume=args.resume, checkpoint_dir=args.checkpoint_dir, + deployment=FPMDeployment(**_values(args, FPMDeployment)), ) raise AssertionError(f"unhandled support action {args.support_action!r}") diff --git a/python/aisimulate/src/aisimulate/support/fpm.py b/python/aisimulate/src/aisimulate/support/fpm.py index f09634310f..cd38c77833 100644 --- a/python/aisimulate/src/aisimulate/support/fpm.py +++ b/python/aisimulate/src/aisimulate/support/fpm.py @@ -7,9 +7,10 @@ import json import shlex +import sys from pathlib import Path -from .schema import SupportRequest +from .schema import FPMDeployment, SupportRequest def fpm_cli_args( @@ -21,6 +22,7 @@ def fpm_cli_args( limit: int | None = None, resume: bool = False, checkpoint_dir: str | Path | None = None, + deployment: FPMDeployment | None = None, ) -> list[str]: if limit is not None and (not smoke or type(limit) is not int or limit < 1): raise ValueError("limit must be a positive cell count and requires smoke=True") @@ -57,6 +59,12 @@ def fpm_cli_args( ] if request.identity.sm is not None: command.extend(("--sm", str(request.identity.sm))) + if deployment is not None: + for name, value in deployment.model_dump(exclude_none=True).items(): + if name == "image": + command.extend(("--generator-set", f"K8sConfig.k8s_image={json.dumps(value)}")) + else: + command.extend(("--" + name.replace("_", "-"), value)) if plan_only: command.append("--plan-only") if smoke: @@ -116,6 +124,7 @@ def run_fpm( limit: int | None = None, resume: bool = False, checkpoint_dir: str | Path | None = None, + deployment: FPMDeployment | None = None, ) -> int: """Preview without side effects; execution requires the matching saved plan.""" @@ -130,6 +139,7 @@ def run_fpm( limit=limit, resume=resume, checkpoint_dir=checkpoint_dir, + deployment=deployment, ) if not execute: if resume or (root.exists() and any(root.iterdir())): @@ -142,4 +152,10 @@ def run_fpm( _check_campaign_outputs(root, smoke=smoke, resume=resume, checkpoint_dir=checkpoint_dir) from collector.fpm_forward.cli import main as fpm_main - return fpm_main(command[3:]) + # The collector reports input/plan failures through argparse before + # entering run_resolved; only execution failures escape this call. + try: + return fpm_main(command[3:]) + except (RuntimeError, ValueError, OSError) as exc: + print(f"aisimulate support collect-fpm failed: {exc}", file=sys.stderr) + return 1 diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index 9d55daff03..d7c29a6d40 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -5,8 +5,11 @@ from __future__ import annotations +import fcntl import hashlib import json +import os +import stat from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -98,7 +101,16 @@ def _configs( def _commands(request: SupportRequest, root: Path, recommendation_names: list[str]) -> dict[str, Any]: return { "fpm_plan_local": fpm_cli_args(request, output_dir=root, plan_only=True), - "fpm_run_local": fpm_cli_args(request, output_dir=root, plan_only=False), + "fpm_run_local": [ + "aisimulate", + "support", + "collect-fpm", + "--config", + str(root / "request.yaml"), + "--output-dir", + str(root), + "--execute", + ], "predict": [ [ "aisimulate", @@ -217,18 +229,20 @@ def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any] @contextmanager def plan_lock(root: Path) -> Iterator[None]: - """Reject overlapping support operations instead of racing plan writes.""" + """Keep one stable lock inode; the OS releases ownership on process exit.""" lock = root / _LOCK_NAME + descriptor = os.open(lock, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) try: - handle = lock.open("x", encoding="utf-8") - except FileExistsError as exc: - raise ValueError(f"another support operation holds {lock}") from exc - try: - with handle: - yield + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise ValueError(f"support lock must be a regular file: {lock}") + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise ValueError(f"another support operation holds {lock}") from exc + yield finally: - lock.unlink() + os.close(descriptor) def _check_paths(root: Path) -> None: diff --git a/python/aisimulate/src/aisimulate/support/schema.py b/python/aisimulate/src/aisimulate/support/schema.py index 60c73e368c..603f6356c1 100644 --- a/python/aisimulate/src/aisimulate/support/schema.py +++ b/python/aisimulate/src/aisimulate/support/schema.py @@ -6,9 +6,10 @@ from __future__ import annotations import re -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Literal +from packaging.version import Version from pydantic import Field, field_validator, model_validator from aisimulate.config.common import PositiveFiniteFloat, PositiveStrictInt, StrictModel, load_yaml @@ -110,6 +111,42 @@ class SearchProfile(StrictModel): seed: int = Field(default=42, strict=True, ge=0) +_DNS_LABEL = r"[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?" +_DNS_SUBDOMAIN = rf"{_DNS_LABEL}(?:\.{_DNS_LABEL})*" + + +class FPMDeployment(StrictModel): + """Deployment-only inputs included in the collector's frozen-plan identity.""" + + dynamo_version: str | None = None + image: str | None = Field(default=None, pattern=r"^[^\s\x00]+$") + namespace: str | None = Field(default=None, pattern=rf"^{_DNS_LABEL}$") + model_cache: str | None = None + transport: Literal["nvlink", "ib", "efa"] | None = None + image_pull_secret: str | None = Field(default=None, pattern=rf"^{_DNS_SUBDOMAIN}$", max_length=253) + + @field_validator("dynamo_version") + @classmethod + def _release_version(cls, value: str | None) -> str | None: + if value is not None: + Version(value) + return value + + @field_validator("model_cache") + @classmethod + def _model_cache(cls, value: str | None) -> str | None: + if value is None: + return None + parts = value.split(":") + if len(parts) > 3 or len(parts[0]) > 253 or not re.fullmatch(_DNS_SUBDOMAIN, parts[0]): + raise ValueError("model_cache must be NAME[:MOUNT[:SUBPATH]] with a valid PVC name") + if len(parts) > 1 and parts[1] and not PurePosixPath(parts[1]).is_absolute(): + raise ValueError("model_cache MOUNT must be an absolute container path") + if "\x00" in value or "\n" in value or "\r" in value: + raise ValueError("model_cache must contain no NUL characters or newlines") + return value + + class SupportRequest(StrictModel): schema_version: Literal["aisimulate-support-request/v1"] = "aisimulate-support-request/v1" identity: SupportIdentity diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py index fde9f707e2..4f93847a37 100644 --- a/python/aisimulate/tests/unit/test_support_cli.py +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -383,6 +383,183 @@ def collector_main(argv): assert calls[0][calls[0].index("--limit") + 1] == "1" +def _local_collection_command(tmp_path): + model = tmp_path / "model with spaces" + model.mkdir() + (model / "config.json").write_text( + json.dumps( + { + "architectures": ["LlamaForCausalLM"], + "model_type": "llama", + "num_hidden_layers": 2, + "hidden_size": 128, + "intermediate_size": 256, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "vocab_size": 512, + "max_position_embeddings": 16384, + "torch_dtype": "bfloat16", + } + ) + ) + request_path = tmp_path / "request.yaml" + root = tmp_path / "plan" + assert cli.main(_init_args(request_path, model=model)) == 0 + assert cli.main(["support", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 + return ["support", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] + + +def test_deployment_options_reach_frozen_collector_plan(tmp_path, monkeypatch): + from collector.fpm_forward import runner + + calls = [] + monkeypatch.setattr(runner, "run_collection", lambda plan, **kwargs: calls.append((plan, kwargs)) or []) + command = _local_collection_command(tmp_path) + command += [ + "--dynamo-version", + "1.2.0", + "--image", + "registry.example/fpm@sha256:" + "a" * 64, + "--namespace", + "pilot-collection", + "--model-cache", + "model-cache:/models:checkpoint", + "--transport", + "nvlink", + "--image-pull-secret", + "registry-secret", + ] + + assert cli.main(command) == 0 + + [(plan, kwargs)] = calls + assert kwargs["generator_overrides"] == { + "generator_dynamo_version": "1.2.0", + "K8sConfig": { + "k8s_image": "registry.example/fpm@sha256:" + "a" * 64, + "k8s_namespace": "pilot-collection", + "k8s_pvc_name": "model-cache", + "k8s_pvc_mount_path": "/models", + "k8s_model_path_in_pvc": "checkpoint", + "transport": "nvlink", + "k8s_image_pull_secret": "registry-secret", + }, + } + assert plan.cells + assert kwargs["artifact_root"] == str(tmp_path / "plan/fpm-artifacts") + assert kwargs["database_root"] == str(tmp_path / "plan/systems/data") + + +@pytest.mark.parametrize( + "option,value", + [ + ("--dynamo-version", "latest"), + ("--image", "bad image"), + ("--image", ""), + ("--namespace", "invalid namespace"), + ("--model-cache", ":/models"), + ("--model-cache", "cache:a:b:c"), + ("--model-cache", "cache:relative"), + ("--transport", "pcie"), + ("--image-pull-secret", "invalid secret"), + ("--generator-set", "params.agg.tp=8"), + ("--generator-config", "unrestricted.yaml"), + ], +) +def test_deployment_options_reject_invalid_or_unrestricted_inputs(tmp_path, capsys, option, value): + command = _local_collection_command(tmp_path) + command.remove("--execute") + + with pytest.raises(SystemExit) as error: + cli.main([*command, option, value]) + + assert error.value.code == 2 + assert "Traceback" not in capsys.readouterr().err + assert not (tmp_path / "plan/fpm-checkpoint").exists() + assert not (tmp_path / "plan/fpm-artifacts").exists() + + +@pytest.mark.parametrize( + "option,first,changed", + [ + ("--dynamo-version", "1.2.0", "1.3.0"), + ("--image", "registry.example/fpm:first", "registry.example/fpm:second"), + ("--namespace", "first", "second"), + ("--model-cache", "cache:/models:first", "cache:/models:second"), + ("--transport", "nvlink", "ib"), + ("--image-pull-secret", "first", "second"), + ], +) +def test_deployment_resume_requires_the_same_frozen_identity(tmp_path, monkeypatch, capsys, option, first, changed): + from collector.fpm_forward import planner, runner + + command = [*_local_collection_command(tmp_path), option, first] + checkpoint = tmp_path / "plan/fpm-checkpoint/fpm_forward.json" + plans = [] + actual_run = runner.run_collection + + def checkpoint_only(plan, **kwargs): + # No silicon data is produced: exercise planning and checkpoint guards + # with a synthetic empty campaign checkpoint in place of GPU execution. + plans.append(plan) + checkpoint.parent.mkdir(exist_ok=True) + checkpoint.write_text(json.dumps({"schema": runner.CHECKPOINT_SCHEMA, "plan_sha256": plan.sha256, "cells": {}})) + return [] + + monkeypatch.setattr(planner, "_git_revision", lambda: "test-source-revision") + monkeypatch.setattr(runner, "run_collection", checkpoint_only) + assert cli.main(command) == 0 + saved = checkpoint.read_bytes() + with pytest.raises(SystemExit) as error: + cli.main(command) + assert error.value.code == 2 + assert len(plans) == 1 + assert cli.main([*command, "--resume"]) == 0 + assert plans[0].sha256 == plans[1].sha256 + assert checkpoint.read_bytes() == saved + + # The real collector must reject a changed identity before GPU execution. + monkeypatch.setattr(runner, "run_collection", actual_run) + monkeypatch.setattr(runner, "KubernetesCellRunner", lambda *args, **kwargs: pytest.fail("must not launch GPU work")) + capsys.readouterr() + assert cli.main([*command, "--resume", option, changed]) == 1 + assert "checkpoint does not match the current frozen plan" in capsys.readouterr().err + assert checkpoint.read_bytes() == saved + + +@pytest.mark.parametrize("stage", ["input", "execution"]) +@pytest.mark.parametrize("error_type", [RuntimeError, ValueError, OSError, KeyboardInterrupt]) +def test_collector_failures_keep_public_cli_exit_codes(tmp_path, monkeypatch, capsys, stage, error_type): + from collector.fpm_forward import cli as collector_cli + + request_path = tmp_path / "request.yaml" + root = tmp_path / "plan" + assert cli.main(_init_args(request_path)) == 0 + assert cli.main(["support", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 + capsys.readouterr() + + def fail(*args, **kwargs): + raise error_type("test collection failure") + + monkeypatch.setattr(collector_cli, "build_collection_case_plan", lambda **kwargs: SimpleNamespace(model_path=None)) + monkeypatch.setattr(collector_cli, "resolve_run_inputs", fail if stage == "input" else lambda *args: None) + monkeypatch.setattr(collector_cli, "run_resolved", fail) + command = ["support", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] + + if error_type is KeyboardInterrupt: + assert cli.main(command) == 130 + elif stage == "input": + with pytest.raises(SystemExit) as error: + cli.main(command) + assert error.value.code == 2 + else: + assert cli.main(command) == 1 + stderr = capsys.readouterr().err + assert "Traceback" not in stderr + if stage == "execution" and error_type is not KeyboardInterrupt: + assert "aisimulate support collect-fpm failed: test collection failure" in stderr + + @pytest.mark.parametrize("contents", ["[one, two]\n", "identity: [\n", "identity: {}\n"]) def test_bad_request_yaml_is_reported_without_a_traceback(tmp_path, capsys, contents) -> None: request = tmp_path / "bad.yaml" diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py index 7b5807f829..040cd0b3ce 100644 --- a/python/aisimulate/tests/unit/test_support_plan.py +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -6,8 +6,10 @@ from __future__ import annotations import builtins +import fcntl import json import os +import select import shlex import subprocess import sys @@ -19,7 +21,7 @@ from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig from aisimulate.support.fpm import run_fpm -from aisimulate.support.plan import create_plan, request_id +from aisimulate.support.plan import create_plan, plan_lock, request_id from aisimulate.support.schema import SupportRequest pytestmark = pytest.mark.unit @@ -147,14 +149,14 @@ def test_plan_collects_one_chosen_worker_and_bounds_recommendation(tmp_path, mod assert search.agg_max_num_batched_tokens == [8192] assert search.agg_max_num_seqs == [256] commands = json.loads((tmp_path / "commands.json").read_text()) - command = commands["fpm_run_local"] + command = commands["fpm_plan_local"] assert command[command.index("--fpm-gpu-counts") + 1] == "4" assert command[command.index("--fpm-max-gpus") + 1] == "4" assert command[command.index("--fpm-parallel-presets") + 1] == preset assert command[command.index("--fpm-max-prefill-isl") + 1] == "1024" assert command[command.index("--fpm-max-prefill-batch-size") + 1] == "1" - assert "--plan-only" not in command - assert "--plan-only" in commands["fpm_plan_local"] + assert "--plan-only" in command + assert "--execute" in commands["fpm_run_local"] def test_one_worker_allocation_keeps_one_pilot_even_with_two_candidate_limit(tmp_path): @@ -208,8 +210,9 @@ def test_emitted_recommendations_evaluate_both_replica_choices(tmp_path, monkeyp "max_candidates,missing_pilot_samples,statuses", [(1, False, [0]), (2, False, [0, 0]), (2, True, [1, 0])], ) +@pytest.mark.parametrize("tampered_commands", [False, True]) def test_documented_recommendation_loop_attempts_every_candidate( - tmp_path, max_candidates, missing_pilot_samples, statuses + tmp_path, max_candidates, missing_pilot_samples, statuses, tampered_commands ): guide = Path(__file__).resolve().parents[4] / "docs/self-service-support.md" snippet = guide.read_text().split("python3 - <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] @@ -221,6 +224,20 @@ def test_documented_recommendation_loop_attempts_every_candidate( ) plan = create_plan(request, workdir / "aisimulate-support") commands = json.loads(Path(plan["outputs"]["commands"]).read_text())["recommend"] + injected_marker = tmp_path / "injected-command-ran" + if tampered_commands: + command_file = Path(plan["outputs"]["commands"]) + payload = json.loads(command_file.read_text()) + payload["recommend"].insert( + 0, + [ + sys.executable, + "-c", + "import sys; from pathlib import Path; Path(sys.argv[1]).write_text('unexpected execution')", + str(injected_marker), + ], + ) + command_file.write_text(json.dumps(payload)) evaluated = tmp_path / "evaluated.txt" wrapper = tmp_path / "bin/aisimulate" wrapper.parent.mkdir() @@ -282,6 +299,7 @@ def create(self, worker_id): check=False, ) + assert not injected_marker.exists() assert result.returncode == (1 if any(statuses) else 0), result.stderr assert evaluated.read_text().splitlines() == [str(index) for index in range(1, max_candidates + 1)] for index, (output, status) in enumerate(zip(plan["outputs"]["recommendation_results"], statuses, strict=True), 1): @@ -432,6 +450,24 @@ def no_collector_import(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", no_collector_import) +def test_published_execution_command_rejects_an_occupied_campaign(tmp_path, monkeypatch, capsys): + from aisimulate.main import main + + create_plan(_request(), tmp_path) + _seed_campaign(tmp_path) + before = _file_contents(tmp_path) + command = json.loads((tmp_path / "commands.json").read_text())["fpm_run_local"] + assert command[:3] == ["aisimulate", "support", "collect-fpm"] + _reject_collector_import(monkeypatch) + + with pytest.raises(SystemExit) as error: + main(command[1:]) + + assert error.value.code == 2 + assert "require --resume" in capsys.readouterr().err + assert _file_contents(tmp_path) == before + + @pytest.mark.parametrize("smoke", [False, True]) def test_repeated_execute_requires_resume_without_touching_campaign(tmp_path, monkeypatch, smoke): create_plan(_request(), tmp_path) @@ -443,7 +479,8 @@ def test_repeated_execute_requires_resume_without_touching_campaign(tmp_path, mo run_fpm(SupportRequest.from_yaml(tmp_path / "request.yaml"), output_dir=tmp_path, execute=True, smoke=smoke) assert _file_contents(tmp_path) == before - assert not (tmp_path / ".support.lock").exists() + with plan_lock(tmp_path): + pass @pytest.mark.parametrize( @@ -491,7 +528,8 @@ def test_resume_occupied_campaign_requires_usable_selected_checkpoint(tmp_path, run_fpm(request, output_dir=tmp_path, execute=True, smoke=smoke, resume=True, checkpoint_dir=selected_dir) assert _file_contents(tmp_path) == before - assert not (tmp_path / ".support.lock").exists() + with plan_lock(tmp_path): + pass @pytest.mark.parametrize("smoke", [False, True]) @@ -552,7 +590,8 @@ def raced_lock(root): run_fpm(request, output_dir=tmp_path, execute=True) assert (tmp_path / "systems/data/collected.parquet").read_bytes() == b"formal timings" - assert not (tmp_path / ".support.lock").exists() + with plan_lock(tmp_path): + pass @pytest.mark.parametrize("smoke,limit", [(False, 1), (True, 0), (True, -1), (True, True)]) @@ -581,15 +620,68 @@ def test_checkpoint_override_cannot_escape_request_directory(tmp_path, relative) run_fpm(_request(), output_dir=tmp_path, checkpoint_dir=tmp_path / relative) -def test_concurrent_plan_operation_preserves_existing_files(tmp_path): +@pytest.mark.parametrize("kill_owner", [False, True]) +def test_plan_lock_recovers_after_owner_exit_and_preserves_existing_files(tmp_path, kill_owner): request = _request() create_plan(request, tmp_path) - (tmp_path / ".support.lock").write_text("busy") before = (tmp_path / "support-plan.json").read_bytes() - with pytest.raises(ValueError, match="another support operation"): - create_plan(request, tmp_path, overwrite=True) + process = subprocess.Popen( + [ + sys.executable, + "-c", + "import sys; from pathlib import Path; from aisimulate.support.plan import plan_lock; " + "lock = plan_lock(Path(sys.argv[1])); lock.__enter__(); " + "print('locked', flush=True); sys.stdin.read(); lock.__exit__(None, None, None)", + str(tmp_path), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + assert select.select([process.stdout], [], [], 10)[0], "lock owner did not become ready" + assert process.stdout.readline().strip() == "locked" + with pytest.raises(ValueError, match="another support operation"): + create_plan(request, tmp_path, overwrite=True) + if kill_owner: + process.kill() + process.communicate(timeout=10) + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) + + (tmp_path / "predict/pilot.yaml").unlink() + create_plan(request, tmp_path, overwrite=True) + assert (tmp_path / "predict/pilot.yaml").is_file() assert (tmp_path / "support-plan.json").read_bytes() == before - assert (tmp_path / ".support.lock").read_text() == "busy" + + +def test_plan_lock_does_not_unlink_the_inode_used_by_a_waiting_process(tmp_path): + create_plan(_request(), tmp_path) + with (tmp_path / ".support.lock").open("r") as waiting: + with plan_lock(tmp_path): + pass + fcntl.flock(waiting, fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(ValueError, match="another support operation"): + create_plan(_request(), tmp_path, overwrite=True) + + +@pytest.mark.parametrize("existing_target", [False, True]) +def test_plan_lock_does_not_follow_a_symlink(tmp_path, existing_target): + root = tmp_path / "plan" + root.mkdir() + external = tmp_path / "external" + if existing_target: + external.write_text("preserve this") + (root / ".support.lock").symlink_to(external) + + with pytest.raises(OSError): + create_plan(_request(), root) + + assert not (root / "request.yaml").exists() + assert external.read_text() == "preserve this" if existing_target else not external.exists() def test_racing_writer_cannot_be_overwritten_or_reported_as_success(tmp_path, monkeypatch): @@ -607,4 +699,5 @@ def racing_open(path, mode="r", *args, **kwargs): create_plan(_request(), tmp_path) assert destination.read_bytes() == b"another writer's config" assert not (tmp_path / "support-plan.json").exists() - assert not (tmp_path / ".support.lock").exists() + with plan_lock(tmp_path): + pass From b4af59d609c6daa9c509c458b8ff90a2ea9cc185 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 13:31:46 +0800 Subject: [PATCH 04/27] fix: report unexpected FPM collection failures cleanly Signed-off-by: Yiming Liu --- .../aisimulate/src/aisimulate/support/fpm.py | 2 +- .../aisimulate/tests/unit/test_support_cli.py | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/python/aisimulate/src/aisimulate/support/fpm.py b/python/aisimulate/src/aisimulate/support/fpm.py index cd38c77833..c443421fb2 100644 --- a/python/aisimulate/src/aisimulate/support/fpm.py +++ b/python/aisimulate/src/aisimulate/support/fpm.py @@ -156,6 +156,6 @@ def run_fpm( # entering run_resolved; only execution failures escape this call. try: return fpm_main(command[3:]) - except (RuntimeError, ValueError, OSError) as exc: + except Exception as exc: print(f"aisimulate support collect-fpm failed: {exc}", file=sys.stderr) return 1 diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py index 4f93847a37..bc60f64d95 100644 --- a/python/aisimulate/tests/unit/test_support_cli.py +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -560,6 +560,68 @@ def fail(*args, **kwargs): assert "aisimulate support collect-fpm failed: test collection failure" in stderr +def test_malformed_resumed_checkpoint_fails_cleanly_without_launching_collection(tmp_path, capsys): + from collector.fpm_forward.runner import CHECKPOINT_SCHEMA + + command = _local_collection_command(tmp_path) + capsys.readouterr() + assert cli.main([argument for argument in command if argument != "--execute"]) == 0 + preview = shlex.split(capsys.readouterr().out) + launch_marker = tmp_path / "cluster-command-launched" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + kubectl = bin_dir / "kubectl" + kubectl.write_text('#!/bin/sh\n: > "$FPM_TEST_LAUNCH_MARKER"\nexit 99\n') + kubectl.chmod(0o755) + environment = { + **os.environ, + "PATH": str(bin_dir) + os.pathsep + os.environ.get("PATH", ""), + "FPM_KUBECTL": "kubectl", + "FPM_TEST_LAUNCH_MARKER": str(launch_marker), + } + planned = subprocess.run( + [sys.executable, *preview[1:]], + cwd=tmp_path, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + assert planned.returncode == 0, planned.stderr + frozen = json.loads(planned.stdout) + root = tmp_path / "plan" + checkpoint = root / "fpm-checkpoint/fpm_forward.json" + checkpoint.parent.mkdir() + checkpoint.write_text( + json.dumps( + { + "schema": CHECKPOINT_SCHEMA, + "plan_sha256": frozen["sha256"], + "cells": {frozen["cells"][0]["cell_id"]: []}, + } + ) + ) + original = checkpoint.read_bytes() + + resumed = subprocess.run( + [sys.executable, "-m", "aisimulate", *command, "--resume"], + cwd=tmp_path, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + + assert resumed.returncode == 1, resumed.stderr + assert "Traceback" not in resumed.stderr + assert "aisimulate support collect-fpm failed:" in resumed.stderr + assert checkpoint.read_bytes() == original + assert not launch_marker.exists() + assert not list((root / "fpm-artifacts").rglob("*.yaml")) + + @pytest.mark.parametrize("contents", ["[one, two]\n", "identity: [\n", "identity: {}\n"]) def test_bad_request_yaml_is_reported_without_a_traceback(tmp_path, capsys, contents) -> None: request = tmp_path / "bad.yaml" From d6650fd10c2245da6cc07ce0b18513abb23100ff Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 16:58:24 +0800 Subject: [PATCH 05/27] test: assert local FPM timing for worker overrides Signed-off-by: Yiming Liu --- python/aisimulate/tests/unit/test_systems_path.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/aisimulate/tests/unit/test_systems_path.py b/python/aisimulate/tests/unit/test_systems_path.py index 223956c615..fd67bdf5d7 100644 --- a/python/aisimulate/tests/unit/test_systems_path.py +++ b/python/aisimulate/tests/unit/test_systems_path.py @@ -223,7 +223,14 @@ def test_worker_hardware_override_resolves_in_local_systems_root(local_profiles, if not explicit_version: raw["engine"].pop("backend_version") - assert _predict(raw).metrics["completed_requests"] == 1 + report = _predict(raw) + + assert report.metrics["completed_requests"] == 1 + # The first token needs a 20 ms prefill and a 20 ms decode; the next needs only decode. + assert report.metrics["mean_ttft_ms"] == pytest.approx(40.0) + assert report.metrics["mean_itl_ms"] == pytest.approx(20.0) + deployment = prediction_to_replay_spec(CorePredictionConfig.model_validate(raw)).backend_deployment + assert deployment.backend_version == _VERSION def test_default_prediction_still_uses_bundled_op_level_data_after_a_local_request(local_profiles): From 29a19ae9ec51b4159c473d039991f4c9c0c70603 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 17:35:59 +0800 Subject: [PATCH 06/27] fix: preserve installed-wheel imports in recommendation tests Support-matrix imports and the database fixture leaked the checkout source path into spawned recommendation workers, which then failed to find the native runtime in wheel-based CI. Restrict standalone path bootstrapping to script execution, restore fixture import state, and cover both support-matrix imports in isolated processes. Signed-off-by: Yiming Liu --- .../sdk/database/test_database_helpers.py | 28 ++++++++------ .../support_matrix/test_script_imports.py | 38 +++++++++++++++++++ .../support_matrix/compare_support_matrix.py | 7 ++-- .../support_matrix/generate_support_matrix.py | 7 ++-- 4 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 python/aisimulate/tests/unit/support_matrix/test_script_imports.py diff --git a/python/aisimulate/tests/unit/sdk/database/test_database_helpers.py b/python/aisimulate/tests/unit/sdk/database/test_database_helpers.py index 5288a28d9b..378f7c70a2 100644 --- a/python/aisimulate/tests/unit/sdk/database/test_database_helpers.py +++ b/python/aisimulate/tests/unit/sdk/database/test_database_helpers.py @@ -31,6 +31,7 @@ def perf_database(): """ project_root = _find_repo_root(Path(__file__)) src_path = project_root / "src" + saved_sys_path = list(sys.path) sys.path.insert(0, str(src_path)) saved_aiconfigurator_modules = {} @@ -42,20 +43,23 @@ def perf_database(): ): saved_aiconfigurator_modules[key] = sys.modules.pop(key) - import aiconfigurator.sdk.perf_database as perf_database + try: + import aiconfigurator.sdk.perf_database as perf_database - importlib.reload(perf_database) - yield perf_database + importlib.reload(perf_database) + yield perf_database - # Drop the isolated legacy/canonical module pair, then restore both module - # graphs together so imported class references in other test modules do not - # point at a reloaded canonical implementation. - for key in list(sys.modules.keys()): - if key in {"aiconfigurator", "aiconfigurator_core"} or key.startswith( - ("aiconfigurator.", "aiconfigurator_core.") - ): - sys.modules.pop(key) - sys.modules.update(saved_aiconfigurator_modules) + finally: + sys.path[:] = saved_sys_path + # Drop the isolated legacy/canonical module pair, then restore both module + # graphs together so imported class references in other test modules do not + # point at a reloaded canonical implementation. + for key in list(sys.modules.keys()): + if key in {"aiconfigurator", "aiconfigurator_core"} or key.startswith( + ("aiconfigurator.", "aiconfigurator_core.") + ): + sys.modules.pop(key) + sys.modules.update(saved_aiconfigurator_modules) @pytest.fixture diff --git a/python/aisimulate/tests/unit/support_matrix/test_script_imports.py b/python/aisimulate/tests/unit/support_matrix/test_script_imports.py new file mode 100644 index 0000000000..a2a4d8abb3 --- /dev/null +++ b/python/aisimulate/tests/unit/support_matrix/test_script_imports.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Importing support-matrix scripts must preserve the caller's package search path.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize("script", ["compare_support_matrix", "generate_support_matrix"]) +def test_script_import_preserves_package_search_path(script): + result = subprocess.run( + [ + sys.executable, + "-c", + """ +import importlib +import sys + +original_path = list(sys.path) +importlib.import_module(sys.argv[1]) +assert sys.path == original_path, (original_path, sys.path) +""", + f"tools.support_matrix.{script}", + ], + cwd=Path(__file__).resolve().parents[3], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr diff --git a/python/aisimulate/tools/support_matrix/compare_support_matrix.py b/python/aisimulate/tools/support_matrix/compare_support_matrix.py index 6cceaf231c..585e643c8c 100644 --- a/python/aisimulate/tools/support_matrix/compare_support_matrix.py +++ b/python/aisimulate/tools/support_matrix/compare_support_matrix.py @@ -27,9 +27,10 @@ from pathlib import Path # Ensure local repo paths are importable when running as a standalone script. -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) -sys.path.insert(0, _REPO_ROOT) +if __name__ == "__main__": + _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + sys.path.insert(0, _REPO_ROOT) from tools.support_matrix.support_matrix import ( ENCODER_DATA_UNAVAILABLE_PREFIX, diff --git a/python/aisimulate/tools/support_matrix/generate_support_matrix.py b/python/aisimulate/tools/support_matrix/generate_support_matrix.py index 3327f782c4..4212025f65 100644 --- a/python/aisimulate/tools/support_matrix/generate_support_matrix.py +++ b/python/aisimulate/tools/support_matrix/generate_support_matrix.py @@ -15,9 +15,10 @@ import sys # Ensure local repo paths are importable when running as a standalone script. -_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) -sys.path.insert(0, _REPO_ROOT) +if __name__ == "__main__": + _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.insert(0, os.path.join(_REPO_ROOT, "src")) + sys.path.insert(0, _REPO_ROOT) from tools.support_matrix.support_matrix import ( VALID_STATUSES, From dffb80d93aa04d9b1e4c4fbadab4d65c70663e4e Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 19:52:57 +0800 Subject: [PATCH 07/27] feat: name model and hardware FPM onboarding explicitly Signed-off-by: Yiming Liu --- README.md | 2 +- docs/self-service-support.md | 24 ++++---- python/aisimulate/src/aisimulate/main.py | 2 +- .../aisimulate/src/aisimulate/support/cli.py | 51 ++++++++++------- .../aisimulate/src/aisimulate/support/fpm.py | 2 +- .../aisimulate/src/aisimulate/support/plan.py | 23 ++++++-- .../aisimulate/tests/unit/test_support_cli.py | 55 +++++++++++-------- .../tests/unit/test_support_plan.py | 35 ++++++++++-- 8 files changed, 129 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index d9dde012a2..341aed8904 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ Use the focused SDK documentation instead of treating CLI internals as public APIs: - [Estimator/FPE Python and Rust SDK](docs/core-api.md) -- [Guided FPM onboarding and local data setup](docs/self-service-support.md) +- [Onboard a model for FPM simulation on a target hardware platform](docs/self-service-support.md) - [FPM collection-to-prediction workflow](python/aisimulate/docs/fpm/end-to-end-workflow.md) - [Replay SDK and artifact contract](crates/core/src/replay/README.md) - [Sweeper SDK](docs/sweeper/overview.md) diff --git a/docs/self-service-support.md b/docs/self-service-support.md index 8671e585f6..04d72378c6 100644 --- a/docs/self-service-support.md +++ b/docs/self-service-support.md @@ -3,9 +3,11 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Set up FPM collection for a model and GPU allocation +# Onboard a model for FPM simulation on a target hardware platform -`aisimulate support` creates a small onboarding request, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. +`aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. + +`aisimulate support` remains a compatibility alias for existing commands and scripts. Saved request schemas, default output paths, and plan filenames remain unchanged; existing plans can be reused, and `plan --overwrite` repairs missing files without rewriting their saved commands. New next-step and collection commands use `aisimulate onboard`. Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model integration, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not implement an Inkling model class, run preflight checks, provision GPU resources, or establish measured accuracy. Those steps belong to the broader integration project. @@ -14,7 +16,7 @@ Planning works before the model has an AISimulate model class or measured FPM ti Use an environment installed from this checkout; see [development setup](../DEVELOPMENT.md). Guided setup requires a terminal and starts only when explicitly requested: ```bash -aisimulate support init --interactive --output support-request.yaml +aisimulate onboard init --interactive --output support-request.yaml ``` Enter the actual model identifier or checkpoint path, pinned model revision, dense/MoE kind, pinned vLLM version, GPU system, allocation, and interconnect. Then choose a TP size and pilot workload. Supplied options skip their prompts. Enter accepts displayed defaults; invalid values can be corrected; Ctrl-C or end-of-input cancels without saving. Existing files require `--overwrite`. @@ -22,7 +24,7 @@ Enter the actual model identifier or checkpoint path, pinned model revision, den Both guided and scripted setup use onboarding. `--profile onboarding` is an optional spelling of the same behavior. For automation, supply the identity flags directly: ```bash -aisimulate support init \ +aisimulate onboard init \ --model /models/your-pinned-checkpoint \ --model-revision YOUR_IMMUTABLE_REVISION \ --model-kind dense \ @@ -34,17 +36,17 @@ aisimulate support init \ Replace the model and runtime placeholders with the actual inputs. The example does not identify an Inkling checkpoint or claim that four H200s can run your model. Framework support currently selects vLLM. Optional tokenizer, chat-template, and AISimulate revisions are recorded only when supplied. -The default pilot uses 1,024 input tokens, 128 output tokens, concurrency 1, four requests, a 16,384-token context limit, TTFT target 1,000 ms, and TPOT target 100 ms. Scripted setup defaults to TP1 unless `--tensor-parallel` is supplied. These are planning defaults, not measured model capacity or latency. Change them with the corresponding flags shown by `aisimulate support init --help`. +The default pilot uses 1,024 input tokens, 128 output tokens, concurrency 1, four requests, a 16,384-token context limit, TTFT target 1,000 ms, and TPOT target 100 ms. Scripted setup defaults to TP1 unless `--tensor-parallel` is supplied. These are planning defaults, not measured model capacity or latency. Change them with the corresponding flags shown by `aisimulate onboard init --help`. One node is the default; GPUs per node then equals `--gpu-count`. For multiple nodes, supply both `--node-count` and `--gpus-per-node`; their product must equal the total allocation. Each pure-TP worker must fit on one node. Advanced flags include `--request-count`, `--max-candidates`, `--objective`, `--seed`, and `--sm`. ## Plan, preview, and explicitly execute ```bash -aisimulate support plan \ +aisimulate onboard plan \ --config support-request.yaml --output-dir ./aisimulate-support -aisimulate support collect-fpm \ +aisimulate onboard collect-fpm \ --config ./aisimulate-support/request.yaml \ --output-dir ./aisimulate-support ``` @@ -53,23 +55,23 @@ The first command saves the request, `support-plan.json`, `commands.json`, `pred The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. -Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate support collect-fpm --execute` command for collection, alongside a read-only collector planning command. +Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate onboard collect-fpm --execute` command for collection, alongside a read-only collector planning command. After those prerequisites are ready, explicitly launch collection from that environment: ```bash -aisimulate support collect-fpm \ +aisimulate onboard collect-fpm \ --config ./aisimulate-support/request.yaml \ --output-dir ./aisimulate-support --execute ``` Execution requires a matching saved plan. The request records model and runtime revisions; this setup does not download a pinned checkpoint or verify the installed runtime against them. Keep the actual checkpoint and runtime consistent with the request before collecting or predicting. -Set deployment options directly on `support collect-fpm`: `--dynamo-version VERSION`, `--image IMAGE`, `--namespace NAME`, `--model-cache NAME[:MOUNT[:SUBPATH]]`, `--transport nvlink|ib|efa`, and `--image-pull-secret NAME`. The mount, when supplied, is an absolute container path. Prefer an immutable image digest. Supply the same options when previewing, executing, and resuming; deployment settings are part of the collector's frozen-plan identity, so changed settings require a new output directory. Arbitrary collector arguments and engine overrides are not accepted by this command. +Set deployment options directly on `onboard collect-fpm`: `--dynamo-version VERSION`, `--image IMAGE`, `--namespace NAME`, `--model-cache NAME[:MOUNT[:SUBPATH]]`, `--transport nvlink|ib|efa`, and `--image-pull-secret NAME`. The mount, when supplied, is an absolute container path. Prefer an immutable image digest. Supply the same options when previewing, executing, and resuming; deployment settings are part of the collector's frozen-plan identity, so changed settings require a new output directory. Arbitrary collector arguments and engine overrides are not accepted by this command. For a diagnostic run, add `--execute --smoke`; `--limit N` also requires `--smoke`. Diagnostic smoke and limited runs do not publish formal FPM data. Existing campaign data, raw artifacts, or checkpoints require explicit `--resume` and a readable matching collector checkpoint; otherwise choose a new output directory. A custom `--checkpoint-dir`, if needed, must remain inside the plan's `fpm-checkpoint/` directory. Selecting an empty checkpoint directory does not allow reuse of existing campaign artifacts. Smoke and formal campaigns have separate checkpoints and artifact directories, so an existing smoke run does not prevent the first formal run, or vice versa. The collector verifies the resumed checkpoint's frozen-plan identity. -Planning and collection reject concurrent support operations. The persistent `.support.lock` file uses an OS advisory lock; ownership is released when the process exits, including after an abrupt termination. Leave the file in place. Request validation, saved support-plan checks, and collector input resolution exit 2. Failures after collector execution starts, including a frozen checkpoint identity mismatch, exit 1 with a concise message. Interruption exits 130. +Planning and collection reject concurrent onboarding operations. The persistent `.support.lock` file uses an OS advisory lock; ownership is released when the process exits, including after an abrupt termination. Leave the file in place. Request validation, saved onboarding-plan checks, and collector input resolution exit 2. Failures after collector execution starts, including a frozen checkpoint identity mismatch, exit 1 with a concise message. Interruption exits 130. The collector narrows initial prefill sampling with the pilot's input-token and concurrency bounds. Decode uses the collector's existing profile; a four-request synthetic pilot does not imply four timing samples or a short decode campaign. Inspect the generated command and collector plan before committing GPU time. Successful formal collection publishes the FPM Parquet file and metadata pair into the plan's local systems data directory; diagnostic success alone does not provide that pair. diff --git a/python/aisimulate/src/aisimulate/main.py b/python/aisimulate/src/aisimulate/main.py index 9e34e8251d..25110edb57 100644 --- a/python/aisimulate/src/aisimulate/main.py +++ b/python/aisimulate/src/aisimulate/main.py @@ -321,7 +321,7 @@ def _recommend(args: argparse.Namespace, raw: dict[str, Any], factory) -> int: def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(list(sys.argv[1:] if argv is None else argv)) - if args.command == "support": + if args.command in {"onboard", "support"}: try: return run_support_command(args) except (ValidationError, ValueError, OSError) as exc: diff --git a/python/aisimulate/src/aisimulate/support/cli.py b/python/aisimulate/src/aisimulate/support/cli.py index 0653c68c59..9dc72ed760 100644 --- a/python/aisimulate/src/aisimulate/support/cli.py +++ b/python/aisimulate/src/aisimulate/support/cli.py @@ -23,15 +23,20 @@ def add_support_parser(subparsers: Any) -> None: - support = subparsers.add_parser( - "support", - help="Set up a model and GPU allocation for local FPM collection.", - description="Create an onboarding request, generate a plan, and preview or execute local FPM collection.", + onboard = subparsers.add_parser( + "onboard", + aliases=("support",), + help="Onboard a model for FPM simulation on a target hardware platform.", + description=( + "Onboard a model for FPM simulation on a target hardware platform. " + "Declare the model and hardware, generate a plan, and preview or execute FPM collection. " + "The previous command name 'support' remains a compatibility alias." + ), ) - actions = support.add_subparsers(dest="support_action", required=True) + actions = onboard.add_subparsers(dest="support_action", required=True) init = actions.add_parser( "init", - help="Create an onboarding request with prompts or scripted options.", + help="Declare a model, target hardware, and FPM pilot workload.", description=( "Use --interactive for terminal prompts. Scripted setup requires --model, --model-revision, " "--model-kind, --framework-version, --gpu, --gpu-count, and --interconnect. " @@ -45,7 +50,7 @@ def add_support_parser(subparsers: Any) -> None: init.add_argument("--model-kind", choices=("dense", "moe")) init.add_argument("--framework", choices=("vllm",), help="Collection runtime (default: vllm).") init.add_argument("--framework-version", help="Pinned vLLM version in the collection environment.") - init.add_argument("--gpu", help="GPU system name, for example h200_sxm.") + init.add_argument("--gpu", help="Target GPU system name, for example h200_sxm.") init.add_argument("--gpu-count", type=int, help="Total GPUs available.") init.add_argument("--node-count", type=int, help="Number of nodes (default: 1).") init.add_argument("--gpus-per-node", type=int, help="GPUs available per node; defaults to gpu-count on one node.") @@ -72,17 +77,21 @@ def add_support_parser(subparsers: Any) -> None: init.add_argument("--output", default="support-request.yaml") init.add_argument("--overwrite", action="store_true") - plan = actions.add_parser("plan", help="Write ordinary predict/recommend configs and local collector commands.") + plan = actions.add_parser( + "plan", help="Plan FPM collection and write predict/recommend configs for the target hardware." + ) plan.add_argument("-c", "--config", required=True) plan.add_argument("--output-dir", default="./aisimulate-support") plan.add_argument("--overwrite", action="store_true") plan.add_argument("--format", choices=("table", "json"), default="table") - collect = actions.add_parser("collect-fpm", help="Preview the local collector command, or run with --execute.") + collect = actions.add_parser( + "collect-fpm", help="Preview FPM collection on the target hardware, or run with --execute." + ) collect.add_argument("-c", "--config", required=True) collect.add_argument("--output-dir", default="./aisimulate-support") collect.add_argument( - "--execute", action="store_true", help="Execute collection using a matching saved support plan." + "--execute", action="store_true", help="Execute collection using a matching saved onboarding plan." ) collect.add_argument( "--smoke", action="store_true", help="Diagnostic smoke collection; does not publish formal data." @@ -121,14 +130,14 @@ def _request_from_args(args: argparse.Namespace) -> SupportRequest: def _request_target(path: str | Path, *, overwrite: bool) -> Path: target = Path(path).expanduser().absolute() if target.exists() and not target.is_file(): - raise ValueError(f"support request {target} must be a file path") + raise ValueError(f"onboarding request {target} must be a file path") if (target.exists() or target.is_symlink()) and not overwrite: - raise ValueError(f"support request {target} exists; pass --overwrite to replace it") + raise ValueError(f"onboarding request {target} exists; pass --overwrite to replace it") parent = target.parent while not parent.exists() and not parent.is_symlink(): parent = parent.parent if not parent.is_dir() or not os.access(parent, os.W_OK): - raise ValueError(f"support request parent {parent} must be a writable directory") + raise ValueError(f"onboarding request parent {parent} must be a writable directory") return target @@ -165,7 +174,7 @@ def _print(value: dict[str, Any], output_format: str = "table") -> None: "model_revision": ("Pinned model revision (not main/latest)", str), "model_kind": ("Model kind (dense/moe)", str), "framework_version": ("Pinned vLLM version", str), - "gpu": ("GPU system name (for example h200_sxm)", str), + "gpu": ("Target GPU system name (for example h200_sxm)", str), "gpu_count": ("Total GPUs available", int), "interconnect": ("GPU interconnect (for example nvswitch, pcie, or none)", str), "tensor_parallel": ("GPUs per pure-TP worker", int), @@ -219,9 +228,10 @@ def _prompt(args: argparse.Namespace, name: str) -> None: def _guided_request(args: argparse.Namespace) -> SupportRequest: if not sys.stdin.isatty(): raise ValueError( - "--interactive requires a terminal; for automation supply options (aisimulate support init --help)" + "--interactive requires a terminal; for automation supply options (aisimulate onboard init --help)" ) - print("FPM onboarding: a pure-TP worker and a small synthetic workload on vLLM.") + print("Onboard a model for FPM simulation on a target hardware platform.") + print("Start with a pure-TP worker and a small synthetic workload on vLLM.") print("Enter accepts a displayed default. Ctrl-C cancels without saving. Supplied options skip their prompts.") for name in _PROMPTS: if getattr(args, name) is None: @@ -255,7 +265,8 @@ def _init(args: argparse.Namespace) -> int: return 130 workload = request.workload print( - f"Scope: {request.identity.model_kind} model, vLLM {request.identity.framework_version}, " + f"Scope: FPM simulation for {request.identity.model} ({request.identity.model_kind}), " + f"vLLM {request.identity.framework_version}, " f"{request.identity.gpu_count} {request.identity.gpu} GPU(s) on {request.identity.node_count} node(s); " f"TP{request.search.tensor_parallel}, {workload.input_tokens}/{workload.output_tokens} tokens, " f"concurrency {workload.concurrency}, {workload.request_count} requests, " @@ -269,7 +280,7 @@ def _init(args: argparse.Namespace) -> int: { "request_id": request_id(request), "request": str(path), - "next": shlex.join(["aisimulate", "support", "plan", "--config", str(path)]), + "next": shlex.join(["aisimulate", "onboard", "plan", "--config", str(path)]), } ) return 0 @@ -290,7 +301,7 @@ def _plan(args: argparse.Namespace) -> int: "next": shlex.join( [ "aisimulate", - "support", + "onboard", "collect-fpm", "--config", plan["outputs"]["request"], @@ -320,4 +331,4 @@ def run_support_command(args: argparse.Namespace) -> int: checkpoint_dir=args.checkpoint_dir, deployment=FPMDeployment(**_values(args, FPMDeployment)), ) - raise AssertionError(f"unhandled support action {args.support_action!r}") + raise AssertionError(f"unhandled onboarding action {args.support_action!r}") diff --git a/python/aisimulate/src/aisimulate/support/fpm.py b/python/aisimulate/src/aisimulate/support/fpm.py index c443421fb2..b13e4cd3ef 100644 --- a/python/aisimulate/src/aisimulate/support/fpm.py +++ b/python/aisimulate/src/aisimulate/support/fpm.py @@ -157,5 +157,5 @@ def run_fpm( try: return fpm_main(command[3:]) except Exception as exc: - print(f"aisimulate support collect-fpm failed: {exc}", file=sys.stderr) + print(f"aisimulate onboard collect-fpm failed: {exc}", file=sys.stderr) return 1 diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index d7c29a6d40..0b93466652 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -103,7 +103,7 @@ def _commands(request: SupportRequest, root: Path, recommendation_names: list[st "fpm_plan_local": fpm_cli_args(request, output_dir=root, plan_only=True), "fpm_run_local": [ "aisimulate", - "support", + "onboard", "collect-fpm", "--config", str(root / "request.yaml"), @@ -235,11 +235,11 @@ def plan_lock(root: Path) -> Iterator[None]: descriptor = os.open(lock, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) try: if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise ValueError(f"support lock must be a regular file: {lock}") + raise ValueError(f"onboarding lock must be a regular file: {lock}") try: fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError as exc: - raise ValueError(f"another support operation holds {lock}") from exc + raise ValueError(f"another onboarding operation holds {lock}") from exc yield finally: os.close(descriptor) @@ -270,7 +270,9 @@ def check_plan(request: SupportRequest, root: Path) -> None: try: prior = json.loads((root / "support-plan.json").read_text(encoding="utf-8")) except (OSError, ValueError) as exc: - raise ValueError(f"a readable support plan is required in {root}; run support plan first") from exc + raise ValueError( + f"a readable onboarding plan is required in {root}; run aisimulate onboard plan first" + ) from exc if not isinstance(prior, dict) or prior.get("request_id") != request_id(request): raise ValueError(f"refusing to mix a different request identity in {root}; choose a new output directory") try: @@ -303,7 +305,18 @@ def create_plan(request: SupportRequest, output_dir: str | Path, *, overwrite: b raise ValueError(f"refusing symlinked plan output {destination}") if not destination.exists(): pending[relative] = content - elif destination.read_bytes() != content: + else: + existing = destination.read_bytes() + if existing == content: + continue + if relative == Path("commands.json"): + # Preserve exact pre-rename commands; the support alias still + # enters the same guarded collector. Accept no other edits. + legacy_commands = json.loads(content) + legacy_commands["fpm_run_local"][1] = "support" + legacy_content = (json.dumps(legacy_commands, indent=2, sort_keys=True) + "\n").encode() + if existing == legacy_content: + continue raise ValueError(f"generated plan input {destination} was modified; choose a new output directory") (root / "systems/data").mkdir(parents=True, exist_ok=True) for relative, content in pending.items(): diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py index bc60f64d95..fc6e0d2759 100644 --- a/python/aisimulate/tests/unit/test_support_cli.py +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -43,7 +43,7 @@ def _init_args(output: Path, *, full: bool = False, **changes) -> list[str]: options = {**_REQUIRED, **(_PILOT if full else {}), **changes} - return ["support", "init", "--output", str(output)] + [ + return ["onboard", "init", "--output", str(output)] + [ part for name, value in options.items() if value is not None @@ -70,6 +70,17 @@ def respond(prompt: str) -> str: return prompts +@pytest.mark.parametrize("command", [[], ["onboard"], ["support"]]) +def test_help_explains_model_fpm_and_target_hardware_scope(capsys, command) -> None: + with pytest.raises(SystemExit) as result: + cli.main([*command, "--help"]) + + assert result.value.code == 0 + output = " ".join(capsys.readouterr().out.split()) + assert "Onboard a model for FPM simulation on a target hardware platform." in output + assert "aisimulate" in output + + def test_guided_and_scripted_setup_produce_the_same_request(tmp_path, monkeypatch, capsys) -> None: guided = tmp_path / "guided request.yaml" scripted = tmp_path / "scripted.yaml" @@ -78,7 +89,7 @@ def test_guided_and_scripted_setup_produce_the_same_request(tmp_path, monkeypatc ["example/unintegrated-model", "revision-123", "dense", "0.24.0", "h200_sxm", "4", "nvswitch", "2"] + [""] * 6, ) - assert cli.main(["support", "init", "--interactive", "--output", str(guided)]) == 0 + assert cli.main(["onboard", "init", "--interactive", "--output", str(guided)]) == 0 assert cli.main(_init_args(scripted, tensor_parallel=2)) == 0 assert SupportRequest.from_yaml(guided) == SupportRequest.from_yaml(scripted) @@ -181,7 +192,7 @@ def test_cancelled_setup_preserves_files_and_creates_no_partial_request( output.write_text("existing contents\n") _terminal(monkeypatch, ["example/unintegrated-model", interruption()]) - assert cli.main(["support", "init", "--interactive", "--output", str(output), "--overwrite"]) == 130 + assert cli.main(["onboard", "init", "--interactive", "--output", str(output), "--overwrite"]) == 130 if existing: assert output.read_text() == "existing contents\n" @@ -197,7 +208,7 @@ def test_interactive_requires_a_terminal_without_reading_input(tmp_path, monkeyp output = tmp_path / "request.yaml" with pytest.raises(SystemExit) as error: - cli.main(["support", "init", "--interactive", "--output", str(output)]) + cli.main(["onboard", "init", "--interactive", "--output", str(output)]) assert error.value.code == 2 assert "requires a terminal" in capsys.readouterr().err @@ -218,7 +229,7 @@ def test_invalid_output_is_rejected_before_prompting(tmp_path, monkeypatch, targ target = parent / "request.yaml" with pytest.raises(SystemExit) as error: - cli.main(["support", "init", "--interactive", "--output", str(target)]) + cli.main(["onboard", "init", "--interactive", "--output", str(target)]) assert error.value.code == 2 @@ -262,7 +273,7 @@ def test_init_next_command_quotes_the_request_path(tmp_path, capsys) -> None: next_command = next( line.removeprefix("next: ") for line in capsys.readouterr().out.splitlines() if line.startswith("next: ") ) - assert shlex.split(next_command) == ["aisimulate", "support", "plan", "--config", str(output)] + assert shlex.split(next_command) == ["aisimulate", "onboard", "plan", "--config", str(output)] def test_next_command_handles_a_relative_filename_starting_with_a_dash(tmp_path, monkeypatch, capsys) -> None: @@ -285,7 +296,7 @@ def test_printed_plan_next_command_runs_in_a_shell(tmp_path, monkeypatch, capsys request = tmp_path / "request.yaml" assert cli.main(_init_args(request)) == 0 capsys.readouterr() - assert cli.main(["support", "plan", "-c", str(request), f"--output-dir={output_dir}", "--format", "json"]) == 0 + assert cli.main(["onboard", "plan", "-c", str(request), f"--output-dir={output_dir}", "--format", "json"]) == 0 summary = json.loads(capsys.readouterr().out) result = subprocess.run( @@ -304,9 +315,10 @@ def test_printed_plan_next_command_runs_in_a_shell(tmp_path, monkeypatch, capsys assert "--plan-only" in result.stdout +@pytest.mark.parametrize("command", ["onboard", "support"]) @pytest.mark.parametrize("model_kind", ["dense", "moe"]) def test_init_plan_and_preview_use_real_public_configs_without_launching_collection( - tmp_path, monkeypatch, capsys, model_kind + tmp_path, monkeypatch, capsys, model_kind, command ) -> None: def unexpected_launch(*args, **kwargs): pytest.fail("setup, plan, and preview must not launch a collector") @@ -316,12 +328,11 @@ def unexpected_launch(*args, **kwargs): monkeypatch.setattr(cli, "resolve_runner_factory", unexpected_launch) request_path = tmp_path / "request 'quoted'.yaml" output = tmp_path / "plan 'quoted'" - assert cli.main(_init_args(request_path, model_kind=model_kind, tensor_parallel=2)) == 0 + assert cli.main([command, *_init_args(request_path, model_kind=model_kind, tensor_parallel=2)[1:]]) == 0 capsys.readouterr() assert ( - cli.main(["support", "plan", "--config", str(request_path), "--output-dir", str(output), "--format", "json"]) - == 0 + cli.main([command, "plan", "--config", str(request_path), "--output-dir", str(output), "--format", "json"]) == 0 ) summary = json.loads(capsys.readouterr().out) plan = json.loads((output / "support-plan.json").read_text()) @@ -336,8 +347,8 @@ def unexpected_launch(*args, **kwargs): assert prediction.traffic.source.input_tokens == 1024 next_command = shlex.split(summary["next"]) - assert next_command[:3] == ["aisimulate", "support", "collect-fpm"] - assert cli.main(next_command[1:]) == 0 + assert next_command[:3] == ["aisimulate", "onboard", "collect-fpm"] + assert cli.main([command, *next_command[2:]]) == 0 preview = capsys.readouterr().out assert "collector.fpm_forward" in preview assert "--fpm-max-gpus 2 --fpm-gpu-counts 2" in preview @@ -355,12 +366,12 @@ def collector_main(argv): request_path = tmp_path / "request.yaml" output = tmp_path / "plan" assert cli.main(_init_args(request_path, tensor_parallel=2)) == 0 - assert cli.main(["support", "plan", "-c", str(request_path), "--output-dir", str(output)]) == 0 + assert cli.main(["onboard", "plan", "-c", str(request_path), "--output-dir", str(output)]) == 0 assert ( cli.main( [ - "support", + "onboard", "collect-fpm", "-c", str(request_path), @@ -405,8 +416,8 @@ def _local_collection_command(tmp_path): request_path = tmp_path / "request.yaml" root = tmp_path / "plan" assert cli.main(_init_args(request_path, model=model)) == 0 - assert cli.main(["support", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 - return ["support", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] + assert cli.main(["onboard", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 + return ["onboard", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] def test_deployment_options_reach_frozen_collector_plan(tmp_path, monkeypatch): @@ -535,7 +546,7 @@ def test_collector_failures_keep_public_cli_exit_codes(tmp_path, monkeypatch, ca request_path = tmp_path / "request.yaml" root = tmp_path / "plan" assert cli.main(_init_args(request_path)) == 0 - assert cli.main(["support", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 + assert cli.main(["onboard", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 capsys.readouterr() def fail(*args, **kwargs): @@ -544,7 +555,7 @@ def fail(*args, **kwargs): monkeypatch.setattr(collector_cli, "build_collection_case_plan", lambda **kwargs: SimpleNamespace(model_path=None)) monkeypatch.setattr(collector_cli, "resolve_run_inputs", fail if stage == "input" else lambda *args: None) monkeypatch.setattr(collector_cli, "run_resolved", fail) - command = ["support", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] + command = ["onboard", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] if error_type is KeyboardInterrupt: assert cli.main(command) == 130 @@ -557,7 +568,7 @@ def fail(*args, **kwargs): stderr = capsys.readouterr().err assert "Traceback" not in stderr if stage == "execution" and error_type is not KeyboardInterrupt: - assert "aisimulate support collect-fpm failed: test collection failure" in stderr + assert "aisimulate onboard collect-fpm failed: test collection failure" in stderr def test_malformed_resumed_checkpoint_fails_cleanly_without_launching_collection(tmp_path, capsys): @@ -616,7 +627,7 @@ def test_malformed_resumed_checkpoint_fails_cleanly_without_launching_collection assert resumed.returncode == 1, resumed.stderr assert "Traceback" not in resumed.stderr - assert "aisimulate support collect-fpm failed:" in resumed.stderr + assert "aisimulate onboard collect-fpm failed:" in resumed.stderr assert checkpoint.read_bytes() == original assert not launch_marker.exists() assert not list((root / "fpm-artifacts").rglob("*.yaml")) @@ -628,7 +639,7 @@ def test_bad_request_yaml_is_reported_without_a_traceback(tmp_path, capsys, cont request.write_text(contents) with pytest.raises(SystemExit) as error: - cli.main(["support", "plan", "--config", str(request), "--output-dir", str(tmp_path / "plan")]) + cli.main(["onboard", "plan", "--config", str(request), "--output-dir", str(tmp_path / "plan")]) assert error.value.code == 2 assert "Traceback" not in capsys.readouterr().err diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py index 040cd0b3ce..670d5798ea 100644 --- a/python/aisimulate/tests/unit/test_support_plan.py +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -373,6 +373,31 @@ def test_same_request_overwrite_preserves_timings_and_rejects_modified_inputs(tm assert data.read_bytes() == b"timings" +@pytest.mark.parametrize("modified_command", [False, True]) +def test_pre_onboard_plan_repairs_only_exact_legacy_commands(tmp_path, modified_command): + request = _request() + plan = create_plan(request, tmp_path) + command_file = tmp_path / "commands.json" + commands = json.loads(command_file.read_text()) + commands["fpm_run_local"][1] = "support" + if modified_command: + commands["fpm_run_local"].remove("--execute") + command_file.write_text(json.dumps(commands, indent=2, sort_keys=True) + "\n") + _seed_campaign(tmp_path) + original = _file_contents(tmp_path) + prediction = tmp_path / "predict/pilot.yaml" + prediction.unlink() + before_repair = _file_contents(tmp_path) + + if modified_command: + with pytest.raises(ValueError, match="generated plan input .*commands.json was modified"): + create_plan(request, tmp_path, overwrite=True) + assert _file_contents(tmp_path) == before_repair + else: + assert create_plan(request, tmp_path, overwrite=True) == plan + assert _file_contents(tmp_path) == original + + def test_new_gpu_is_rejected_before_creating_outputs(tmp_path): root = tmp_path / "new-plan" with pytest.raises(ValueError, match="packaged.*system|system specification"): @@ -450,14 +475,16 @@ def no_collector_import(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", no_collector_import) -def test_published_execution_command_rejects_an_occupied_campaign(tmp_path, monkeypatch, capsys): +@pytest.mark.parametrize("command_name", ["onboard", "support"]) +def test_published_execution_command_rejects_an_occupied_campaign(tmp_path, monkeypatch, capsys, command_name): from aisimulate.main import main create_plan(_request(), tmp_path) _seed_campaign(tmp_path) before = _file_contents(tmp_path) command = json.loads((tmp_path / "commands.json").read_text())["fpm_run_local"] - assert command[:3] == ["aisimulate", "support", "collect-fpm"] + assert command[:3] == ["aisimulate", "onboard", "collect-fpm"] + command[1] = command_name _reject_collector_import(monkeypatch) with pytest.raises(SystemExit) as error: @@ -642,7 +669,7 @@ def test_plan_lock_recovers_after_owner_exit_and_preserves_existing_files(tmp_pa try: assert select.select([process.stdout], [], [], 10)[0], "lock owner did not become ready" assert process.stdout.readline().strip() == "locked" - with pytest.raises(ValueError, match="another support operation"): + with pytest.raises(ValueError, match="another onboarding operation"): create_plan(request, tmp_path, overwrite=True) if kill_owner: process.kill() @@ -664,7 +691,7 @@ def test_plan_lock_does_not_unlink_the_inode_used_by_a_waiting_process(tmp_path) with plan_lock(tmp_path): pass fcntl.flock(waiting, fcntl.LOCK_EX | fcntl.LOCK_NB) - with pytest.raises(ValueError, match="another support operation"): + with pytest.raises(ValueError, match="another onboarding operation"): create_plan(_request(), tmp_path, overwrite=True) From 24486a9e5a8d6a96b7e807da86ea936467da7ec1 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 23:25:07 +0800 Subject: [PATCH 08/27] fix: use onboard as the sole model onboarding command Signed-off-by: Yiming Liu --- docs/self-service-support.md | 2 -- python/aisimulate/src/aisimulate/main.py | 2 +- .../aisimulate/src/aisimulate/support/cli.py | 4 +-- .../aisimulate/src/aisimulate/support/plan.py | 13 +------ .../aisimulate/tests/unit/test_support_cli.py | 35 +++++++++++++++---- .../tests/unit/test_support_plan.py | 30 +++++++--------- 6 files changed, 45 insertions(+), 41 deletions(-) diff --git a/docs/self-service-support.md b/docs/self-service-support.md index 04d72378c6..1db4ed65f1 100644 --- a/docs/self-service-support.md +++ b/docs/self-service-support.md @@ -7,8 +7,6 @@ SPDX-License-Identifier: Apache-2.0 `aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. -`aisimulate support` remains a compatibility alias for existing commands and scripts. Saved request schemas, default output paths, and plan filenames remain unchanged; existing plans can be reused, and `plan --overwrite` repairs missing files without rewriting their saved commands. New next-step and collection commands use `aisimulate onboard`. - Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model integration, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not implement an Inkling model class, run preflight checks, provision GPU resources, or establish measured accuracy. Those steps belong to the broader integration project. ## Create the request diff --git a/python/aisimulate/src/aisimulate/main.py b/python/aisimulate/src/aisimulate/main.py index 25110edb57..b5fd0cd8cc 100644 --- a/python/aisimulate/src/aisimulate/main.py +++ b/python/aisimulate/src/aisimulate/main.py @@ -321,7 +321,7 @@ def _recommend(args: argparse.Namespace, raw: dict[str, Any], factory) -> int: def main(argv: Sequence[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(list(sys.argv[1:] if argv is None else argv)) - if args.command in {"onboard", "support"}: + if args.command == "onboard": try: return run_support_command(args) except (ValidationError, ValueError, OSError) as exc: diff --git a/python/aisimulate/src/aisimulate/support/cli.py b/python/aisimulate/src/aisimulate/support/cli.py index 9dc72ed760..e5a06ce1dc 100644 --- a/python/aisimulate/src/aisimulate/support/cli.py +++ b/python/aisimulate/src/aisimulate/support/cli.py @@ -25,12 +25,10 @@ def add_support_parser(subparsers: Any) -> None: onboard = subparsers.add_parser( "onboard", - aliases=("support",), help="Onboard a model for FPM simulation on a target hardware platform.", description=( "Onboard a model for FPM simulation on a target hardware platform. " - "Declare the model and hardware, generate a plan, and preview or execute FPM collection. " - "The previous command name 'support' remains a compatibility alias." + "Declare the model and hardware, generate a plan, and preview or execute FPM collection." ), ) actions = onboard.add_subparsers(dest="support_action", required=True) diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index 0b93466652..26ee789a15 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -305,18 +305,7 @@ def create_plan(request: SupportRequest, output_dir: str | Path, *, overwrite: b raise ValueError(f"refusing symlinked plan output {destination}") if not destination.exists(): pending[relative] = content - else: - existing = destination.read_bytes() - if existing == content: - continue - if relative == Path("commands.json"): - # Preserve exact pre-rename commands; the support alias still - # enters the same guarded collector. Accept no other edits. - legacy_commands = json.loads(content) - legacy_commands["fpm_run_local"][1] = "support" - legacy_content = (json.dumps(legacy_commands, indent=2, sort_keys=True) + "\n").encode() - if existing == legacy_content: - continue + elif destination.read_bytes() != content: raise ValueError(f"generated plan input {destination} was modified; choose a new output directory") (root / "systems/data").mkdir(parents=True, exist_ok=True) for relative, content in pending.items(): diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py index fc6e0d2759..e94bd28fa5 100644 --- a/python/aisimulate/tests/unit/test_support_cli.py +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -70,7 +70,7 @@ def respond(prompt: str) -> str: return prompts -@pytest.mark.parametrize("command", [[], ["onboard"], ["support"]]) +@pytest.mark.parametrize("command", [[], ["onboard"]]) def test_help_explains_model_fpm_and_target_hardware_scope(capsys, command) -> None: with pytest.raises(SystemExit) as result: cli.main([*command, "--help"]) @@ -81,6 +81,29 @@ def test_help_explains_model_fpm_and_target_hardware_scope(capsys, command) -> N assert "aisimulate" in output +@pytest.mark.parametrize("action", ["init", "plan", "collect-fpm"]) +def test_retired_support_command_is_rejected_before_dispatch(tmp_path, monkeypatch, capsys, action) -> None: + def unexpected_dispatch(*args, **kwargs): + pytest.fail("the retired command must be rejected before setup or collector dispatch") + + monkeypatch.setattr(cli, "run_support_command", unexpected_dispatch) + monkeypatch.setattr(cli, "resolve_runner_factory", unexpected_dispatch) + request = tmp_path / "request.yaml" + if action == "init": + args = _init_args(request)[1:] + else: + args = [action, "--config", str(request), "--output-dir", str(tmp_path / "plan")] + if action == "collect-fpm": + args.append("--execute") + + with pytest.raises(SystemExit) as result: + cli.main(["support", *args]) + + assert result.value.code == 2 + assert "invalid choice: 'support'" in capsys.readouterr().err + assert not list(tmp_path.iterdir()) + + def test_guided_and_scripted_setup_produce_the_same_request(tmp_path, monkeypatch, capsys) -> None: guided = tmp_path / "guided request.yaml" scripted = tmp_path / "scripted.yaml" @@ -315,10 +338,9 @@ def test_printed_plan_next_command_runs_in_a_shell(tmp_path, monkeypatch, capsys assert "--plan-only" in result.stdout -@pytest.mark.parametrize("command", ["onboard", "support"]) @pytest.mark.parametrize("model_kind", ["dense", "moe"]) def test_init_plan_and_preview_use_real_public_configs_without_launching_collection( - tmp_path, monkeypatch, capsys, model_kind, command + tmp_path, monkeypatch, capsys, model_kind ) -> None: def unexpected_launch(*args, **kwargs): pytest.fail("setup, plan, and preview must not launch a collector") @@ -328,11 +350,12 @@ def unexpected_launch(*args, **kwargs): monkeypatch.setattr(cli, "resolve_runner_factory", unexpected_launch) request_path = tmp_path / "request 'quoted'.yaml" output = tmp_path / "plan 'quoted'" - assert cli.main([command, *_init_args(request_path, model_kind=model_kind, tensor_parallel=2)[1:]]) == 0 + assert cli.main(_init_args(request_path, model_kind=model_kind, tensor_parallel=2)) == 0 capsys.readouterr() assert ( - cli.main([command, "plan", "--config", str(request_path), "--output-dir", str(output), "--format", "json"]) == 0 + cli.main(["onboard", "plan", "--config", str(request_path), "--output-dir", str(output), "--format", "json"]) + == 0 ) summary = json.loads(capsys.readouterr().out) plan = json.loads((output / "support-plan.json").read_text()) @@ -348,7 +371,7 @@ def unexpected_launch(*args, **kwargs): next_command = shlex.split(summary["next"]) assert next_command[:3] == ["aisimulate", "onboard", "collect-fpm"] - assert cli.main([command, *next_command[2:]]) == 0 + assert cli.main(next_command[1:]) == 0 preview = capsys.readouterr().out assert "collector.fpm_forward" in preview assert "--fpm-max-gpus 2 --fpm-gpu-counts 2" in preview diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py index 670d5798ea..19f97148ca 100644 --- a/python/aisimulate/tests/unit/test_support_plan.py +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -373,29 +373,27 @@ def test_same_request_overwrite_preserves_timings_and_rejects_modified_inputs(tm assert data.read_bytes() == b"timings" -@pytest.mark.parametrize("modified_command", [False, True]) -def test_pre_onboard_plan_repairs_only_exact_legacy_commands(tmp_path, modified_command): +@pytest.mark.parametrize("modification", ["retired_command", "missing_execute", "formatting"]) +def test_overwrite_rejects_modified_commands_before_repairing_missing_files(tmp_path, modification): request = _request() - plan = create_plan(request, tmp_path) + create_plan(request, tmp_path) command_file = tmp_path / "commands.json" commands = json.loads(command_file.read_text()) - commands["fpm_run_local"][1] = "support" - if modified_command: + if modification == "retired_command": + commands["fpm_run_local"][1] = "support" + elif modification == "missing_execute": commands["fpm_run_local"].remove("--execute") - command_file.write_text(json.dumps(commands, indent=2, sort_keys=True) + "\n") + command_file.write_text( + json.dumps(commands, indent=4 if modification == "formatting" else 2, sort_keys=True) + "\n" + ) _seed_campaign(tmp_path) - original = _file_contents(tmp_path) prediction = tmp_path / "predict/pilot.yaml" prediction.unlink() before_repair = _file_contents(tmp_path) - if modified_command: - with pytest.raises(ValueError, match="generated plan input .*commands.json was modified"): - create_plan(request, tmp_path, overwrite=True) - assert _file_contents(tmp_path) == before_repair - else: - assert create_plan(request, tmp_path, overwrite=True) == plan - assert _file_contents(tmp_path) == original + with pytest.raises(ValueError, match="generated plan input .*commands.json was modified"): + create_plan(request, tmp_path, overwrite=True) + assert _file_contents(tmp_path) == before_repair def test_new_gpu_is_rejected_before_creating_outputs(tmp_path): @@ -475,8 +473,7 @@ def no_collector_import(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", no_collector_import) -@pytest.mark.parametrize("command_name", ["onboard", "support"]) -def test_published_execution_command_rejects_an_occupied_campaign(tmp_path, monkeypatch, capsys, command_name): +def test_published_execution_command_rejects_an_occupied_campaign(tmp_path, monkeypatch, capsys): from aisimulate.main import main create_plan(_request(), tmp_path) @@ -484,7 +481,6 @@ def test_published_execution_command_rejects_an_occupied_campaign(tmp_path, monk before = _file_contents(tmp_path) command = json.loads((tmp_path / "commands.json").read_text())["fpm_run_local"] assert command[:3] == ["aisimulate", "onboard", "collect-fpm"] - command[1] = command_name _reject_collector_import(monkeypatch) with pytest.raises(SystemExit) as error: From 396af2df7c6ed5419019e20240d8fafb0ac17e99 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 16 Sep 2026 23:48:32 +0800 Subject: [PATCH 09/27] fix: reject nested symlinks in onboarding collection outputs Signed-off-by: Yiming Liu --- .../aisimulate/src/aisimulate/support/plan.py | 7 +++ .../aisimulate/tests/unit/test_support_cli.py | 42 ++++++++++++++++++ .../tests/unit/test_support_plan.py | 44 +++++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index 26ee789a15..188993b7de 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -261,6 +261,13 @@ def _check_paths(root: Path) -> None: ): if (root / relative).is_symlink(): raise ValueError(f"refusing symlinked plan output {root / relative}") + pending = [root / relative for relative in ("systems/data", "fpm-checkpoint", "fpm-artifacts")] + while pending: + path = pending.pop() + if path.is_symlink(): + raise ValueError(f"refusing symlinked plan output {path}") + if path.is_dir(): + pending.extend(path.iterdir()) def check_plan(request: SupportRequest, root: Path) -> None: diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py index e94bd28fa5..9e7c918d9b 100644 --- a/python/aisimulate/tests/unit/test_support_cli.py +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -656,6 +656,48 @@ def test_malformed_resumed_checkpoint_fails_cleanly_without_launching_collection assert not list((root / "fpm-artifacts").rglob("*.yaml")) +def test_real_collector_artifact_symlink_rejected_before_first_write(tmp_path, monkeypatch, capsys): + from collector.fpm_forward import runner + + command = _local_collection_command(tmp_path) + capsys.readouterr() + assert cli.main([argument for argument in command if argument != "--execute"]) == 0 + preview = shlex.split(capsys.readouterr().out) + planned = subprocess.run( + [sys.executable, *preview[1:]], + cwd=tmp_path, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + assert planned.returncode == 0, planned.stderr + frozen = json.loads(planned.stdout) + root = tmp_path / "plan" + external = tmp_path / "external" + external.mkdir() + (root / "fpm-artifacts").mkdir() + (root / "fpm-artifacts" / frozen["sha256"][:16]).symlink_to(external, target_is_directory=True) + reached_checkpoint = [] + + def stop_before_gpu(*args, **kwargs): + reached_checkpoint.append(True) + raise RuntimeError("collector reached checkpoint loading") + + # The real collector writes collection-plan.json before loading its + # checkpoint. Keep that path intact and stop before any GPU work. + monkeypatch.setattr(runner, "_load_checkpoint", stop_before_gpu) + + with pytest.raises(SystemExit) as error: + cli.main(command) + + assert error.value.code == 2 + assert "refusing symlinked plan output" in capsys.readouterr().err + assert not reached_checkpoint + assert not list(external.iterdir()) + assert not (root / "fpm-checkpoint").exists() + + @pytest.mark.parametrize("contents", ["[one, two]\n", "identity: [\n", "identity: {}\n"]) def test_bad_request_yaml_is_reported_without_a_traceback(tmp_path, capsys, contents) -> None: request = tmp_path / "bad.yaml" diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py index 19f97148ca..3c820f5b22 100644 --- a/python/aisimulate/tests/unit/test_support_plan.py +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -637,6 +637,50 @@ def test_plan_rejects_symlinked_output_without_touching_target(tmp_path): assert data.read_bytes() == b"timings" +@pytest.mark.parametrize( + "relative", + [ + "fpm-artifacts/aaaaaaaaaaaaaaaa/smoke/cells/output", + "fpm-checkpoint/custom/output", + "systems/data/h200_sxm/vllm/0.25.1/output", + ], +) +@pytest.mark.parametrize("target_kind", ["directory", "file", "dangling_directory", "dangling_file"]) +@pytest.mark.parametrize("operation", ["repair", "execute"]) +def test_nested_symlinks_rejected_before_repair_or_collector_import( + tmp_path, monkeypatch, relative, target_kind, operation +): + request = _request() + root = tmp_path / "plan" + create_plan(request, root) + _seed_campaign(root) + (root / "predict/pilot.yaml").unlink() + external = tmp_path / "external" + if target_kind == "directory": + external.mkdir() + elif target_kind == "file": + external.write_bytes(b"outside data") + link = root / relative + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(external, target_is_directory=target_kind.endswith("directory")) + before = _file_contents(tmp_path) + _reject_collector_import(monkeypatch) + + with pytest.raises(ValueError, match="refusing symlinked plan output"): + if operation == "repair": + create_plan(request, root, overwrite=True) + else: + run_fpm(request, output_dir=root, execute=True, resume=True) + + assert _file_contents(tmp_path) == before + assert link.is_symlink() + assert link.readlink() == external + if target_kind == "directory": + assert not list(external.iterdir()) + elif target_kind.startswith("dangling"): + assert not external.exists() + + @pytest.mark.parametrize("relative", ["../shared", "fpm-checkpoint/../../shared"]) def test_checkpoint_override_cannot_escape_request_directory(tmp_path, relative): with pytest.raises(ValueError, match="checkpoint_dir"): From 661fd9739957c319f8055d423b47ef248d37d6d7 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Thu, 17 Sep 2026 00:53:41 +0800 Subject: [PATCH 10/27] docs: rename onboarding guide to FPM self-service Signed-off-by: Yiming Liu --- README.md | 2 +- docs/{self-service-support.md => fpm-self-service.md} | 2 +- python/aisimulate/tests/unit/test_support_plan.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename docs/{self-service-support.md => fpm-self-service.md} (99%) diff --git a/README.md b/README.md index 341aed8904..ad022c242e 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ Use the focused SDK documentation instead of treating CLI internals as public APIs: - [Estimator/FPE Python and Rust SDK](docs/core-api.md) -- [Onboard a model for FPM simulation on a target hardware platform](docs/self-service-support.md) +- [FPM self-service: onboard a model on target hardware](docs/fpm-self-service.md) - [FPM collection-to-prediction workflow](python/aisimulate/docs/fpm/end-to-end-workflow.md) - [Replay SDK and artifact contract](crates/core/src/replay/README.md) - [Sweeper SDK](docs/sweeper/overview.md) diff --git a/docs/self-service-support.md b/docs/fpm-self-service.md similarity index 99% rename from docs/self-service-support.md rename to docs/fpm-self-service.md index 1db4ed65f1..1bc346a64b 100644 --- a/docs/self-service-support.md +++ b/docs/fpm-self-service.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Onboard a model for FPM simulation on a target hardware platform +# FPM self-service `aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py index 3c820f5b22..872754782f 100644 --- a/python/aisimulate/tests/unit/test_support_plan.py +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -214,7 +214,7 @@ def test_emitted_recommendations_evaluate_both_replica_choices(tmp_path, monkeyp def test_documented_recommendation_loop_attempts_every_candidate( tmp_path, max_candidates, missing_pilot_samples, statuses, tampered_commands ): - guide = Path(__file__).resolve().parents[4] / "docs/self-service-support.md" + guide = Path(__file__).resolve().parents[4] / "docs/fpm-self-service.md" snippet = guide.read_text().split("python3 - <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] workdir = tmp_path / "work with 'quotes'; $(literal)" workdir.mkdir() From f5a3ac03f0d4ace3da16cae831559d332e382846 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Thu, 17 Sep 2026 04:57:19 +0800 Subject: [PATCH 11/27] docs: align FPM onboarding with planned model decoupling Signed-off-by: Yiming Liu --- docs/fpm-self-service.md | 21 +++++++++++++------ .../aisimulate/src/aisimulate/support/plan.py | 6 ++++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index 1bc346a64b..697874c9b8 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -7,7 +7,11 @@ SPDX-License-Identifier: Apache-2.0 `aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. -Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model integration, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not implement an Inkling model class, run preflight checks, provision GPU resources, or establish measured accuracy. Those steps belong to the broader integration project. +Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model metadata and execution readiness, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not run target preflight checks, provision GPU resources, or establish measured accuracy. + +In this revision, ordinary FPM `predict` and `recommend` still construct a registered analytical model for resource accounting and SOL-based timing transfer. A separate decoupling change will add a route that uses model/resource metadata and direct interpolation of measured timings without an op-level model class. The existing registered-model/SOL route will remain available. See [Choose the model execution route](#choose-the-model-execution-route) before handing off a new architecture. + +The Inkling pilot is planned for NVIDIA GB200 with TP, DEP, and TEP configurations through the class-independent route. This guided planner currently generates pure-TP configurations only. The checkpoint, runtime, allocation, and strategy degrees must be pinned before that pilot; this guide does not establish Inkling readiness or GB200 accuracy. ## Create the request @@ -51,6 +55,8 @@ aisimulate onboard collect-fpm \ The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. The second prints the collector command without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. On an existing plan, `--overwrite` can repair missing generated files for the identical request; it rejects changed inputs and preserves existing collected data. +If a newer draft revision changes generated guidance, recreate the plan in a new output directory: repair compares generated files byte for byte and does not migrate earlier draft plans. Guidance changes do not change the saved-request identity checks used by collection. + The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate onboard collect-fpm --execute` command for collection, alongside a read-only collector planning command. @@ -75,7 +81,7 @@ The collector narrows initial prefill sampling with the pilot's input-token and ## Run the generated ordinary configurations -After model integration and formal data collection are complete: +After the selected model execution route is available and formal data collection is complete: ```bash aisimulate predict \ @@ -125,10 +131,13 @@ Run either the individual recommendation command or the loop against fresh resul The generated configurations select `engine.workers.aggregated.timing.forward_model: fpm` and set `engine.systems_path` to the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. -Ordinary `predict` and `recommend` commands retain their existing behavior and defaults. Their generated configs can be loaded and edited through the public configuration schema. Missing model integration or data may still prevent execution; a successful simulation is not an accuracy result. Compare its output with an independent run of the same model, runtime, topology, and workload to assess accuracy. +Ordinary `predict` and `recommend` commands retain their existing behavior and defaults. Their generated configs can be loaded and edited through the public configuration schema. In this revision, missing model registration or data may still prevent execution; a successful simulation is not an accuracy result. Compare its output with an independent run of the same model, runtime, topology, and workload to assess accuracy. + +## Choose the model execution route -## Hand off a new model architecture +Hand off the saved request, pinned model configuration, and plan. Both routes need a canonical checkpoint identity, effective precision and topology, correct weight and KV-cache accounting, and matching whole-forward FPM measurements. Collected timings alone do not establish memory fit. -Give an engineer the saved request, the pinned model configuration, and the plan. Follow [How to Add a New Model](../python/aisimulate/docs/add_a_new_model.md) to integrate the architecture and model class, including its operation graph, memory accounting, and KV-cache behavior. FPM supplies measured whole-forward timings; it does not replace the model structure needed by simulation. +- **Registered-model/SOL route, available in this revision:** reuse a compatible analytical class or follow [How to Add a New Model](../python/aisimulate/docs/add_a_new_model.md) when choosing to add one. Verify its operation graph, memory and cache accounting, and native FPM SOL execution. A dedicated class is an option for this route, not the intended prerequisite for every FPM onboarding. +- **Class-independent route, planned in the separate decoupling change:** resolve model and resource metadata without constructing an operation graph. Use direct measured-time interpolation, including wider two-sided KV brackets at the same batch size when both neighboring prompt curves cover the query. Missing brackets or unsupported metadata must produce explicit errors. This route is not implemented by the guided foundation; 2D interpolation remains experimental. -The model guide also describes legacy per-operation silicon profiling. Collecting those per-operation timings is not mandatory for this FPM workflow. Integrate the model structure first, collect the matching whole-forward FPM data through the existing collector, and then verify prediction and recommendation on the target deployment. +Per-operation silicon profiling described in the model guide is not required by either FPM route. The intended self-service workflow collects whole-forward timings, then verifies prediction and recommendation for the exact target deployment. Collection bootstrap can already resolve some unregistered model configurations; that does not establish that this revision's ordinary FPM prediction path can construct them. diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index 188993b7de..8f436107c8 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -181,8 +181,10 @@ def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any] "id": "model_integration", "status": "not_checked", "detail": ( - "Add or verify the registered model class, architecture mapping, memory/cache accounting, " - "and chosen parallelism using docs/add_a_new_model.md. Per-operation silicon data is not required." + "Verify pinned model metadata, memory/cache accounting, and chosen parallelism. This revision's " + "FPM predict/recommend uses registered analytical classes and SOL transfer; follow " + "python/aisimulate/docs/add_a_new_model.md for that route. Class-independent FPM with direct " + "timing interpolation is planned separately. Per-operation silicon data is not required." ), }, { From 439359650ef474736c15c579b0edece55b0df20f Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Thu, 17 Sep 2026 05:58:07 +0800 Subject: [PATCH 12/27] feat: decouple FPM workflows from analytical model classes Signed-off-by: Yiming Liu --- crates/core/src/perfmodel/config.rs | 3 +- crates/core/src/perfmodel/engine/runtime.rs | 6 + crates/core/src/perfmodel/engine/spec.rs | 3 +- crates/core/src/perfmodel/memory.rs | 89 ++- .../src/perfmodel/operators/fpm_forward.rs | 522 ++++++++++++++- .../perfmodel/perf_database/fpm_forward.rs | 38 ++ crates/core/src/perfmodel/py.rs | 75 ++- crates/core/src/python.rs | 33 +- docs/fpm-self-service.md | 70 +- .../collector/fpm_forward/capabilities.py | 75 ++- .../collector/fpm_forward/config.py | 7 + .../aisimulate/collector/fpm_forward/entry.py | 11 + .../collector/fpm_forward/memory_admission.py | 107 ++- .../collector/fpm_forward/planner.py | 132 +++- .../collector/fpm_forward/runner.py | 27 +- .../src/aiconfigurator_core/sdk/engine.py | 147 +++++ .../aiconfigurator_core/sdk/fpm_profile.py | 330 ++++++++++ .../src/aiconfigurator_core/sdk/memory.py | 88 ++- python/aisimulate/src/aisimulate/aic.py | 24 + python/aisimulate/src/aisimulate/compiler.py | 26 + .../src/aisimulate/config/engine.py | 51 ++ python/aisimulate/src/aisimulate/recommend.py | 16 +- python/aisimulate/src/aisimulate/runner.py | 8 + .../aisimulate/src/aisimulate/support/cli.py | 19 +- .../aisimulate/src/aisimulate/support/fpm.py | 38 +- .../aisimulate/src/aisimulate/support/plan.py | 96 ++- .../src/aisimulate/support/schema.py | 79 ++- .../src/aisimulate/sweeper/config.py | 4 + .../src/aisimulate/sweeper/deploy.py | 23 + .../src/aisimulate/sweeper/kv_estimate.py | 9 + .../src/aisimulate/sweeper/model_hw.py | 122 +++- .../src/aisimulate/sweeper/sample.py | 14 + .../src/aisimulate/sweeper/search_space.py | 2 + .../cross_package/test_core_public_api.py | 2 + .../collector/test_fpm_profile_collection.py | 324 ++++++++++ .../tests/unit/sdk/test_fpm_profile.py | 612 ++++++++++++++++++ tests/test_cli_config.py | 12 +- tests/test_fpm_profile_workflow.py | 440 +++++++++++++ 38 files changed, 3559 insertions(+), 125 deletions(-) create mode 100644 python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py create mode 100644 python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py create mode 100644 python/aisimulate/tests/unit/sdk/test_fpm_profile.py create mode 100644 tests/test_fpm_profile_workflow.py diff --git a/crates/core/src/perfmodel/config.rs b/crates/core/src/perfmodel/config.rs index 723b1ca756..503e62f6d1 100644 --- a/crates/core/src/perfmodel/config.rs +++ b/crates/core/src/perfmodel/config.rs @@ -80,7 +80,8 @@ pub const ENGINE_CONFIG_SCHEMA_VERSION: u32 = 1; // batch/query widths and FpmForwardOp gained verify_width. Upstream used // 14/15, already occupied here; these are positional bincode layout changes. // TokenScale was appended to remap draft query widths before op lookup. -pub const ENGINE_SPEC_SCHEMA_VERSION: u32 = 18; +// - 19: FpmForwardOp gained the SOL/direct interpolation selector. +pub const ENGINE_SPEC_SCHEMA_VERSION: u32 = 19; /// Static engine identity and setup information carried by an /// [`crate::perfmodel::engine::spec::EngineSpec`]. diff --git a/crates/core/src/perfmodel/engine/runtime.rs b/crates/core/src/perfmodel/engine/runtime.rs index 8b0fdd9a4d..41386047a1 100644 --- a/crates/core/src/perfmodel/engine/runtime.rs +++ b/crates/core/src/perfmodel/engine/runtime.rs @@ -2536,6 +2536,7 @@ mod tests { )); let fpm_op = |phase: FpmPhase| { Op::FpmForward(FpmForwardOp { + interpolation: Default::default(), name: format!("fpm_forward_{}", phase.as_str()), phase, model_path: "org/model-a".to_string(), @@ -2563,6 +2564,7 @@ mod tests { use crate::perf_database::fpm_forward::tests::default_identity; let db = PerfDatabase::load(&systems_root(), "b200_sxm", "vllm", "0.24.0").unwrap(); let fpm_op = Op::FpmForward(FpmForwardOp { + interpolation: Default::default(), name: "fpm_forward_prefill".into(), phase: FpmPhase::Prefill, model_path: "org/model-a".into(), @@ -2620,6 +2622,7 @@ mod tests { )); let fpm_op = |phase: FpmPhase| { Op::FpmForward(FpmForwardOp { + interpolation: Default::default(), name: format!("fpm_forward_{}", phase.as_str()), phase, model_path: "org/model-a".to_string(), @@ -2929,6 +2932,7 @@ mod tests { let hidden = Op::Overlap(crate::operators::OverlapOp::new( "hidden", vec![Op::FpmForward(FpmForwardOp { + interpolation: Default::default(), name: "fpm_forward_prefill".into(), phase: FpmPhase::Prefill, model_path: "org/model-a".into(), @@ -3246,6 +3250,7 @@ mod tests { "0.25.1", )); let mut op = FpmForwardOp { + interpolation: Default::default(), name: "fpm_forward_decode".into(), phase: FpmPhase::Decode, model_path: "org/model-a".into(), @@ -3291,6 +3296,7 @@ mod tests { )); let fpm_op = |phase: FpmPhase, width: u32| { Op::FpmForward(FpmForwardOp { + interpolation: Default::default(), name: format!("fpm_forward_{}", phase.as_str()), phase, model_path: "org/model-a".to_string(), diff --git a/crates/core/src/perfmodel/engine/spec.rs b/crates/core/src/perfmodel/engine/spec.rs index 442525ef45..cdf6faf21f 100644 --- a/crates/core/src/perfmodel/engine/spec.rs +++ b/crates/core/src/perfmodel/engine/spec.rs @@ -622,6 +622,7 @@ mod tests { // Recursive like Overlap/Fallback: sol_ops carries the model's // original granular list, so the round-trip must preserve nesting. crate::operators::FpmForwardOp { + interpolation: Default::default(), name: "fpm_forward_prefill".into(), phase: crate::operators::FpmPhase::Prefill, model_path: "org/model-a".into(), @@ -1166,7 +1167,7 @@ mod tests { EngineSpec::from_bincode(&bytes), Err(AicError::UnsupportedSchemaVersion { got: 17, - expected: 18, + expected: ENGINE_SPEC_SCHEMA_VERSION, .. }) )); diff --git a/crates/core/src/perfmodel/memory.rs b/crates/core/src/perfmodel/memory.rs index 491dde5dd6..0e07d1c6f1 100644 --- a/crates/core/src/perfmodel/memory.rs +++ b/crates/core/src/perfmodel/memory.rs @@ -120,6 +120,8 @@ pub enum EstimateSource { Native, /// Post-weight reservation heuristic (`naive_kv_reservation`, default 80%). NaiveFallback, + /// Explicit rank-local bounds supplied in an FPM profile. + Profile, } /// Non-KV memory components, in bytes. Maps AIC's `_get_memory_usage` dict: @@ -250,12 +252,18 @@ fn fetch_python_estimate( req: &KvCacheEstimateRequest, ) -> Result { use pyo3::prelude::*; + use pyo3::types::PyDictMethods; let engine = &req.engine; Python::with_gil(|py| -> PyResult { let engine_mod = py.import("aiconfigurator.sdk.memory")?; let kwargs = estimate_kwargs(py, req)?; + if engine.extra.contains_key("fpm_profile") { + for (name, mode) in crate::py::fpm_profile_quantization(py, engine)? { + kwargs.set_item(name, mode)?; + } + } let out = engine_mod.call_method( "estimate_kv_cache", @@ -311,6 +319,11 @@ fn estimate_kwargs<'py>( kwargs.set_item("attention_dp_size", parallel.attention_dp_size.unwrap_or(1))?; kwargs.set_item("moe_tp_size", parallel.moe_tp_size)?; kwargs.set_item("moe_ep_size", parallel.moe_ep_size)?; + kwargs.set_item("cp_size", parallel.cp_size.unwrap_or(1))?; + kwargs.set_item( + "fpm_profile", + engine.extra.get("fpm_profile").map(String::as_str), + )?; kwargs.set_item( "gemm_quant_mode", quant.weight_dtype.as_ref().map(dtype_str), @@ -364,6 +377,7 @@ fn estimate_from_dict( let source = match out.get_item("source")?.extract::()?.as_str() { "native" => EstimateSource::Native, "naive_fallback" => EstimateSource::NaiveFallback, + "profile" => EstimateSource::Profile, other => { return Err(PyValueError::new_err(format!( "estimate_kv_cache returned unknown source {other:?}" @@ -544,6 +558,63 @@ mod tests { } } + #[cfg(feature = "embed-python")] + #[test] + fn profile_memory_preserves_exact_quantization_and_rejects_wire_conflicts() { + use crate::DataType; + + pyo3::prepare_freethreaded_python(); + for (gemm_mode, weight_dtype, moe_mode, moe_dtype) in [ + ("fp8_block", DataType::Fp8Block, "nvfp4", DataType::Nvfp4), + ("int8_wo", DataType::Int8, "nvfp4", DataType::Nvfp4), + ("int4_wo", DataType::Int4, "nvfp4", DataType::Nvfp4), + ("sq", DataType::Int8, "nvfp4", DataType::Nvfp4), + ("fp8_ootb", DataType::Fp8, "nvfp4", DataType::Nvfp4), + ("nvfp4", DataType::Nvfp4, "int4_wo", DataType::Int4), + ] { + let mut req = python_forwarding_request(); + req.cuda_graph_reserved_bytes = 0; + req.gpu_memory_capacity_bytes_override = Some(1_000); + req.engine.quantization.weight_dtype = Some(weight_dtype); + req.engine.quantization.moe_dtype = Some(moe_dtype); + req.engine.quantization.activation_dtype = Some(DataType::Fp8); + req.engine.quantization.kv_cache_dtype = Some(DataType::Fp8); + req.engine.extra.insert("fpm_profile".into(), serde_json::json!({ + "schema_version": 1, + "model": "test-model", + "model_revision": "native-memory-fixture-v1", + "architecture": "UnregisteredMemoryFixture", + "context_length": 8192, + "num_experts": 0, + "provenance": "Synthetic native transport fixture; no silicon qualification", + "deployments": [{ + "system": "test-system", "backend": "vllm", "backend_version": "test-version", + "tp": 1, "pp": 1, "dp": 1, "moe_tp": 1, "moe_ep": 1, "cp": 1, + "gemm_quant_mode": gemm_mode, "moe_quant_mode": moe_mode, + "fmha_quant_mode": "fp8", "comm_quant_mode": "half", "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 100, "activations_bytes": 20, "runtime_overhead_bytes": 30, + "comm_overhead_bytes": 50, "kv_bytes_per_token": 10, "cache_layout": "linear", + "max_num_tokens": 8192, "max_batch_size": 256, + "provenance": "Declared rank-local arithmetic fixture; excludes CUDA graphs" + } + }] + }).to_string()); + let estimate = + estimate_kv_cache(req.clone()).expect("profile precision must survive the wire"); + assert_eq!(estimate.source, EstimateSource::Profile); + assert_eq!(estimate.total_kv_size_bytes, 600); + assert_eq!(estimate.total_kv_size_tokens, 60); + + req.engine.quantization.weight_dtype = Some(DataType::Bfloat16); + let error = estimate_kv_cache(req).expect_err("a different wire dtype must fail"); + assert!( + error.to_string().contains("FPM profile identity conflict"), + "{error}" + ); + } + } + #[cfg(feature = "python")] #[test] fn py_dict_forwarding_and_parsing_are_deterministic() { @@ -551,9 +622,20 @@ mod tests { use pyo3::types::{PyAnyMethods, PyDict, PyDictMethods}; pyo3::prepare_freethreaded_python(); - let req = python_forwarding_request(); + let mut req = python_forwarding_request(); + req.engine.extra.insert( + "fpm_profile".into(), + serde_json::json!({"model": "test-model"}).to_string(), + ); Python::with_gil(|py| -> PyResult<()> { let kwargs = estimate_kwargs(py, &req)?; + assert_eq!( + kwargs + .get_item("fpm_profile")? + .unwrap() + .extract::()?, + r#"{"model":"test-model"}"# + ); assert_eq!( kwargs .get_item("cuda_graph_reserved_bytes")? @@ -585,6 +667,11 @@ mod tests { .cuda_graph_reserved_bytes, req.cuda_graph_reserved_bytes ); + out.set_item("source", "profile")?; + assert_eq!( + estimate_from_dict(out.as_any())?.source, + EstimateSource::Profile + ); Ok(()) }) .expect("deterministic PyDict round-trip must succeed"); diff --git a/crates/core/src/perfmodel/operators/fpm_forward.rs b/crates/core/src/perfmodel/operators/fpm_forward.rs index 407a904142..35c7980aa4 100644 --- a/crates/core/src/perfmodel/operators/fpm_forward.rs +++ b/crates/core/src/perfmodel/operators/fpm_forward.rs @@ -22,14 +22,20 @@ //! KV tokens while retaining the normal log2 gate on batch, so small //! block-aligned coordinates around zero remain connected. //! -//! The SOL roofline anchoring the interpolation is the model's ORIGINAL -//! op-level list (carried as `sol_ops` on the wire) queried in SOL mode with -//! Python's coordinate back-mapping — see [`sol_total`]. +//! With `interpolation="sol"` (the default), the roofline anchoring transfer +//! is the model's ORIGINAL op-level list (`sol_ops` on the wire) queried in +//! SOL mode with Python's coordinate back-mapping — see [`sol_total`]. +//! `interpolation="direct"` instead uses raw linear interpolation between +//! genuine measured curves. Prefill stays at one batch with two-sided KV +//! brackets; decode stays inside its inferred capture regime. It never +//! evaluates `sol_ops`, clamps an unsupported batch, or uses fabricated KV. //! //! NOT the crate's `src/fpm/` (`ForwardPassPerfModel`) module: that is the //! online-tuning model over Dynamo ForwardPassMetrics telemetry, an unrelated //! concept that shares the "FPM" abbreviation. +use std::collections::BTreeMap; + use serde::{Deserialize, Serialize}; use crate::common::error::AicError; @@ -57,9 +63,19 @@ impl FpmPhase { } } +/// Timing interpolation policy. Existing specs retain the analytical +/// roofline; direct interpolation reads measured times without querying it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FpmInterpolation { + #[default] + Sol, + Direct, +} + /// One whole-model forward pass for a single phase. /// -/// `match_identity` is the 11-string cell identity in +/// `match_identity` is the 15-string cell identity in /// [`FPM_CELL_MATCH_COLUMNS`](crate::perf_database::fpm_forward::FPM_CELL_MATCH_COLUMNS) /// order, computed by the PYTHON producer via `_norm_identity` (None -> "", /// Enum -> `.name`) so Rust compares strings verbatim with no re-normalization @@ -90,6 +106,8 @@ pub struct FpmForwardOp { /// separate op-level draft ops). #[serde(default = "default_verify_width")] pub verify_width: u32, + #[serde(default)] + pub interpolation: FpmInterpolation, pub sol_ops: Vec, } @@ -199,6 +217,14 @@ impl FpmForwardOp { let cell = db .fpm_forward .select_cell(&self.match_identity, &self.model_path)?; + if self.interpolation == FpmInterpolation::Direct { + return Ok(cell + .direct_decode + .values() + .filter_map(|curve| curve.keys().next_back()) + .max() + .copied()); + } Ok(cell.decode_domain.as_ref().map(|domain| domain[1].1)) } @@ -232,6 +258,9 @@ impl FpmForwardOp { let cell = db .fpm_forward .select_cell(&self.match_identity, &self.model_path)?; + if self.interpolation == FpmInterpolation::Direct { + return self.resolve_direct_decode(cell, &[batch_size as f64, total_kv], true); + } let Some(domain) = cell.decode_domain else { return Err(data_err(format!( "FPM cell {:?} has no decode rows (model_path={:?}).", @@ -311,6 +340,12 @@ impl FpmForwardOp { cell: &FpmForwardCell, coords: &[f64], ) -> Result { + if self.interpolation == FpmInterpolation::Direct { + return match self.phase { + FpmPhase::Prefill => self.resolve_direct_prefill(cell, coords), + FpmPhase::Decode => self.resolve_direct_decode(cell, coords, false), + }; + } // Data-certified prefill batch clamp (mirrors Python _resolve): the // regime coordinate is the token TOTAL, which stays untouched — the // clamped query prices the same side of the capture cliff and is a @@ -442,6 +477,185 @@ impl FpmForwardOp { Ok(PerformanceResult::new(latency, Source::Silicon)) } + fn resolve_direct_prefill( + &self, + cell: &FpmForwardCell, + coords: &[f64], + ) -> Result { + let (batch, tokens, kv) = (coords[0], coords[1], coords[2]); + if !batch.is_finite() || batch.fract() != 0.0 || batch < 1.0 || batch > u32::MAX as f64 { + return Err(self.direct_coverage_err( + cell, + coords, + "batch_size must be a positive integer", + )); + } + if let Some(value) = cell + .direct_prefill + .get(&(batch as u32, kv as u32)) + .filter(|_| kv == (kv as u32) as f64) + .and_then(|curve| direct_curve_value(curve, tokens)) + { + return self.direct_result(cell, coords, value); + } + let covered: Vec<(u32, f64)> = cell + .direct_prefill + .range((batch as u32, 0)..=(batch as u32, u32::MAX)) + .filter_map(|(&(.., site_kv), curve)| { + direct_curve_value(curve, tokens).map(|value| (site_kv, value)) + }) + .collect(); + // Preserve the original raw-KV method's supported values. Only when + // its two-sided bracket is missing, widen to the nearest covered + // lower and upper KV sites at this exact batch. + let bracket = |guarded: bool| { + let admitted = |site_kv: u32| { + !guarded + || ((site_kv as f64).max(1e-12).log2() - kv.max(1e-12).log2()).abs() <= 2.0 + || (site_kv as f64 - kv).abs() <= 32.0 + }; + let lower = covered + .iter() + .rev() + .find(|&&(site_kv, _)| (site_kv as f64) < kv && admitted(site_kv)); + let upper = covered + .iter() + .find(|&&(site_kv, _)| (site_kv as f64) > kv && admitted(site_kv)); + lower.zip(upper) + }; + let Some((&(lo, lo_ms), &(hi, hi_ms))) = bracket(true).or_else(|| bracket(false)) else { + return Err(self.direct_coverage_err( + cell, + coords, + "no same-batch, two-sided KV bracket whose token curves cover this query", + )); + }; + let weight = (kv - lo as f64) / (hi as f64 - lo as f64); + let latency = lo_ms + (hi_ms - lo_ms) * weight; + self.direct_result(cell, coords, latency) + } + + fn direct_result( + &self, + cell: &FpmForwardCell, + coords: &[f64], + latency: f64, + ) -> Result { + if !latency.is_finite() || latency <= 0.0 { + return Err(self.direct_coverage_err( + cell, + coords, + &format!("interpolation produced an invalid latency ({latency})"), + )); + } + Ok(PerformanceResult::new(latency, Source::Silicon)) + } + + fn direct_coverage_err(&self, cell: &FpmForwardCell, coords: &[f64], reason: &str) -> AicError { + let axes: &[&str] = match self.phase { + FpmPhase::Prefill => &FPM_PREFILL_AXES, + FpmPhase::Decode => &FPM_DECODE_AXES, + }; + let query = axes + .iter() + .zip(coords) + .map(|(axis, value)| format!("{axis}={value}")) + .collect::>() + .join(", "); + data_err(format!( + "FPM direct {} query ({query}) is unsupported for model_path={:?}: {reason}. \ + Collect matching genuine FPM points; direct interpolation never uses SOL or extrapolation.", + self.phase.as_str(), + cell.model_path + )) + } + + fn resolve_direct_decode( + &self, + cell: &FpmForwardCell, + coords: &[f64], + baseline: bool, + ) -> Result { + let (lo_row, hi_row) = self.direct_decode_rows(cell, coords)?; + let row_value = |row: u32| { + let curve = &cell.direct_decode[&row]; + if baseline { + *curve.values().next().expect("nonempty measured curve") + } else { + direct_curve_value(curve, coords[1]).expect("row coverage checked") + } + }; + let lo_ms = row_value(lo_row); + let latency = if lo_row == hi_row { + lo_ms + } else { + let weight = (coords[0] - lo_row as f64) / (hi_row as f64 - lo_row as f64); + lo_ms + (row_value(hi_row) - lo_ms) * weight + }; + self.direct_result(cell, coords, latency) + } + + /// One shared coverage decision for decode queries and their mixed-pass + /// baseline. Neither can include a curve the other cannot evaluate. + fn direct_decode_rows( + &self, + cell: &FpmForwardCell, + coords: &[f64], + ) -> Result<(u32, u32), AicError> { + let (batch, kv) = (coords[0], coords[1]); + if !batch.is_finite() || batch.fract() != 0.0 || batch < 1.0 || batch > u32::MAX as f64 { + return Err(self.direct_coverage_err( + cell, + coords, + "batch_size must be a positive integer", + )); + } + if cell + .direct_decode + .get(&(batch as u32)) + .and_then(|curve| direct_curve_value(curve, kv)) + .is_some() + { + return Ok((batch as u32, batch as u32)); + } + let regime = |position: f64| { + cell.direct_decode_rungs + .partition_point(|&r| (r as f64) < position) + }; + let query_regime = regime(batch); + let mut covered = Vec::new(); + for (&row, curve) in &cell.direct_decode { + let distance = ((row as f64).log2() - batch.log2()).abs(); + if row as f64 != batch + && distance <= 2.0 + && regime(row as f64) == query_regime + && direct_curve_value(curve, kv).is_some() + { + covered.push((distance, row)); + } + } + covered.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); + covered.truncate(4); + let lower = covered + .iter() + .map(|&(_, row)| row) + .filter(|&row| (row as f64) < batch) + .max(); + let upper = covered + .iter() + .map(|&(_, row)| row) + .filter(|&row| (row as f64) > batch) + .min(); + if let Some(rows) = lower.zip(upper) { + return Ok(rows); + } + Err(self.direct_coverage_err( + cell, + coords, + "no two-sided covered batch bracket within one capture regime", + )) + } + /// Resolve an OFF-LATTICE decode batch by its segment bracket (mirrors /// Python `_decode_bracket`): the engine pads decode batches to capture /// rungs — every rung marked by an (x, x+1) row pair — so a query @@ -538,6 +752,22 @@ impl FpmForwardOp { } } +/// Exact lookup or raw linear interpolation strictly inside one measured +/// curve. There is deliberately no callback, boundary hold, or SOL input. +fn direct_curve_value(curve: &BTreeMap, coordinate: f64) -> Option { + if !coordinate.is_finite() || coordinate < 0.0 || coordinate > u32::MAX as f64 { + return None; + } + let (&lo, &lo_ms) = curve.range(..=(coordinate as u32)).next_back()?; + if coordinate == lo as f64 { + return Some(lo_ms); + } + let (&hi, &hi_ms) = curve + .range((std::ops::Bound::Excluded(lo), std::ops::Bound::Unbounded)) + .next()?; + Some(lo_ms + (hi_ms - lo_ms) * (coordinate - lo as f64) / (hi as f64 - lo as f64)) +} + /// FPM interpolation configs: prefill sites `(batch, kv)` own the new-token /// curve; decode sites `(batch,)` own the KV curve. Prefill adds a raw-KV /// fallback around zero to the shared `ScatteredSites` defaults. @@ -660,6 +890,7 @@ mod tests { match_identity: default_identity(4), weight_bytes: 0.0, verify_width: 1, + interpolation: FpmInterpolation::Sol, // Empty sol_ops: exact hits and in-curve lerps never call SOL. sol_ops: vec![], } @@ -674,6 +905,289 @@ mod tests { } } + #[test] + fn direct_prefill_uses_wider_two_sided_kv_brackets_without_sol() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = [ + (100, 0, 10.0), + (200, 0, 20.0), + (100, 2048, 999.0), + (100, 4096, 30.0), + (200, 4096, 40.0), + ] + .into_iter() + .map(|(tokens, kv, latency)| RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: tokens, + total_kv_read_tokens: kv, + latency_ms: latency, + ..RowSpec::default() + }) + .collect::>(); + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut spec = serde_json::to_value(op(FpmPhase::Prefill)).unwrap(); + spec["interpolation"] = serde_json::json!("direct"); + let mut direct: FpmForwardOp = serde_json::from_value(spec).unwrap(); + direct.sol_ops = vec![Op::MlaBmm(crate::operators::MlaBmmOp { + name: "unsupported_sol_sentinel".into(), + scale_factor: 1.0, + num_heads: 128, + is_pre: true, + quant_mode: crate::common::enums::GemmQuantMode::Bfloat16, + })]; + // KV=0 is outside the original two-octave/32-token admission gate. + // First evaluate the two token curves at T=150, then blend raw KV. + let result = direct.query_totals(&db, &[1.0, 150.0, 2048.0]).unwrap(); + assert_eq!(result.latency_ms, 25.0); + assert_eq!(result.source, Source::Silicon); + // The own KV curve does not cover T=150, but its exact T=100 leaf + // still takes precedence over interpolation between other KV sites. + assert_eq!( + direct + .query_totals(&db, &[1.0, 100.0, 2048.0]) + .unwrap() + .latency_ms, + 999.0 + ); + } + + #[test] + fn direct_decode_excludes_healed_fake_fallback_queries_and_baselines() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = [ + (64, 10.0, "real_kv"), + (128, 20.0, "real_kv"), + (256, 99.0, "fake_fallback"), + ] + .into_iter() + .map(|(kv, latency, seed)| RowSpec { + batch_size: 8, + total_kv_read_tokens: kv, + latency_ms: latency, + kv_seed_regime: Some(seed), + ..RowSpec::default() + }) + .collect::>(); + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut direct = op(FpmPhase::Decode); + direct.interpolation = FpmInterpolation::Direct; + // Legacy SOL keeps its in-memory fake-row healing semantics. + assert_eq!( + op(FpmPhase::Decode) + .query_totals(&db, &[8.0, 256.0]) + .unwrap() + .latency_ms, + 40.0 + ); + for kv in [192.0, 256.0] { + let error = direct.query_totals(&db, &[8.0, kv]).unwrap_err(); + assert!(error.to_string().contains("genuine"), "{error}"); + assert!(direct.query_pass_baseline(&db, 8, kv).is_err()); + } + assert_eq!(direct.decode_kv_ceiling(&db).unwrap(), Some(128)); + assert_eq!( + direct.query_totals(&db, &[8.0, 96.0]).unwrap().latency_ms, + 15.0 + ); + assert_eq!( + direct.query_pass_baseline(&db, 8, 96.0).unwrap().latency_ms, + 10.0 + ); + } + + #[test] + fn direct_decode_brackets_within_capture_regime_and_shares_baseline_coverage() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = [ + (8, 8, 1000.0), + (8, 192, 1000.0), + (9, 64, 9.0), + (9, 192, 29.0), + (16, 64, 16.0), + (16, 128, 26.0), + (17, 64, 2000.0), + (17, 192, 2000.0), + ] + .into_iter() + .map(|(batch, kv, latency)| RowSpec { + batch_size: batch, + total_kv_read_tokens: kv, + latency_ms: latency, + kv_seed_regime: Some("real_kv"), + ..RowSpec::default() + }) + .collect::>(); + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut direct = op(FpmPhase::Decode); + direct.interpolation = FpmInterpolation::Direct; + // The same padded graph owns rows 9 and 16; neighboring rows 8 and + // 17 deliberately have very different measured costs. + assert_eq!( + direct.query_totals(&db, &[12.0, 96.0]).unwrap().latency_ms, + 17.0 + ); + assert_eq!( + direct + .query_pass_baseline(&db, 12, 96.0) + .unwrap() + .latency_ms, + 12.0 + ); + for kv in [32.0, 160.0] { + // At 160 only row 9 covers the query. Direct cannot inherit the + // legacy one-sided bracket hold or fabricate a mixed baseline. + assert!(direct.query_totals(&db, &[12.0, kv]).is_err()); + assert!(direct.query_pass_baseline(&db, 12, kv).is_err()); + } + } + + #[test] + fn direct_prefill_preserves_own_curves_and_rejects_missing_batches_and_boundaries() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let mut rows = Vec::new(); + for batch in [1, 2, 4] { + for (tokens, latency) in [(100, 10.0), (200, 20.0), (300, 30.0)] { + for (kv, bump) in [(0, 0.0), (64, 4.0), (128, 8.0)] { + rows.push(RowSpec { + workload_kind: "prefill", + batch_size: batch, + total_prefill_tokens: tokens, + total_kv_read_tokens: kv, + latency_ms: latency + bump, + ..RowSpec::default() + }); + } + } + } + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut direct = op(FpmPhase::Prefill); + direct.interpolation = FpmInterpolation::Direct; + assert_eq!( + direct + .query_totals(&db, &[1.0, 100.0, 0.0]) + .unwrap() + .latency_ms, + 10.0 + ); + assert_eq!( + direct + .query_totals(&db, &[1.0, 150.0, 0.0]) + .unwrap() + .latency_ms, + 15.0 + ); + assert_eq!( + direct + .query_totals(&db, &[1.0, 150.0, 96.0]) + .unwrap() + .latency_ms, + 21.0 + ); + // This fixture certifies the legacy high-batch clamp. It remains a + // valid registered/SOL behavior but is not a measured direct point. + assert!( + op(FpmPhase::Prefill) + .query_totals(&db, &[16.0, 200.0, 0.0]) + .is_ok() + ); + for coords in [ + [3.0, 200.0, 0.0], + [16.0, 200.0, 0.0], + [1.0, 400.0, 32.0], + [1.0, 150.0, -1.0], + [1.0, 150.0, 256.0], + [1.5, 150.0, 64.0], + [1.0, f64::NAN, 64.0], + [1.0, 150.0, f64::INFINITY], + ] { + let error = direct.query_totals(&db, &coords).unwrap_err().to_string(); + assert!(error.contains("FPM direct prefill"), "{error}"); + assert!( + error.contains("Collect matching genuine FPM points"), + "{error}" + ); + } + } + + #[test] + fn omitted_interpolation_keeps_existing_sol_policy() { + let mut spec = serde_json::to_value(op(FpmPhase::Prefill)).unwrap(); + spec.as_object_mut().unwrap().remove("interpolation"); + let decoded: FpmForwardOp = serde_json::from_value(spec.clone()).unwrap(); + assert_eq!(decoded.interpolation, FpmInterpolation::Sol); + spec["interpolation"] = serde_json::json!("unknown"); + assert!(serde_json::from_value::(spec).is_err()); + } + + #[test] + fn direct_rejects_a_fake_only_cell_with_or_without_legacy_healing() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let tmp = tempfile::tempdir().unwrap(); + write_pair( + tmp.path(), + &[RowSpec { + kv_seed_regime: Some("fake_fallback"), + ..RowSpec::default() + }], + ); + for replace in [false, true] { + let mut db = db_with_pair(tmp.path()); + db.set_fpm_forward_for_test( + crate::perf_database::FpmForwardTable::new_with_replacement( + tmp.path().to_path_buf(), + "b200_sxm", + "vllm", + "0.25.1", + replace, + ), + ); + let mut direct = op(FpmPhase::Decode); + direct.interpolation = FpmInterpolation::Direct; + assert!( + op(FpmPhase::Decode) + .query_totals(&db, &[8.0, 4096.0]) + .is_ok() + ); + assert!(direct.query_totals(&db, &[8.0, 4096.0]).is_err()); + assert!(direct.query_pass_baseline(&db, 8, 4096.0).is_err()); + assert_eq!(direct.decode_kv_ceiling(&db).unwrap(), None); + } + } + + #[test] + fn direct_reports_nonfinite_interpolation_as_a_data_error() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = [(100, 1e308), (100_000_000, 1.1e308)] + .into_iter() + .map(|(tokens, latency)| RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: tokens, + total_kv_read_tokens: 0, + latency_ms: latency, + ..RowSpec::default() + }) + .collect::>(); + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut direct = op(FpmPhase::Prefill); + direct.interpolation = FpmInterpolation::Direct; + let error = direct + .query_totals(&db, &[1.0, 1_000_000.0, 0.0]) + .unwrap_err(); + assert!(error.to_string().contains("invalid latency"), "{error}"); + } + #[test] fn certified_prefill_batch_clamp_routes_to_the_ceiling() { use crate::perf_database::fpm_forward::tests::RowSpec; diff --git a/crates/core/src/perfmodel/perf_database/fpm_forward.rs b/crates/core/src/perfmodel/perf_database/fpm_forward.rs index 17d8735bfb..e5a8cd1414 100644 --- a/crates/core/src/perfmodel/perf_database/fpm_forward.rs +++ b/crates/core/src/perfmodel/perf_database/fpm_forward.rs @@ -24,6 +24,9 @@ //! `[batch][total_prefill][total_kv]`, decode `[batch][total_kv]` — plus the //! per-phase axis-aligned domain bounding box and a prebuilt //! [`SiteIndex`](super::perf_interp::SiteIndex). +//! Separate direct-timing curves omit every `fake_fallback` row, including +//! healed rows, so measured-only interpolation never inherits extrapolated +//! values or KV ceilings from the legacy SOL table. //! //! Contract notes, all mirrored from Python: //! - An ABSENT parquet is the soft "not collected" case: `cells()` errors only @@ -143,6 +146,13 @@ pub struct FpmForwardCell { pub decode_batches: Vec, pub decode_rungs: Vec, pub decode_curve_bounds: BTreeMap, + /// Direct timing curves `(batch, KV) -> (prefill tokens -> latency)`. + /// Fake-fallback rows are excluded even when the legacy table heals them. + pub direct_prefill: BTreeMap<(u32, u32), BTreeMap>, + /// Direct decode curves contain genuine measurements only; their bounds + /// do not inherit the fabricated rows' extrapolated KV ceilings. + pub direct_decode: BTreeMap>, + pub direct_decode_rungs: Vec, } /// One loaded parquet/sidecar pair: the grouped cells plus the @@ -852,12 +862,23 @@ fn load_pair( decode_batches: Vec::new(), decode_rungs: Vec::new(), decode_curve_bounds: BTreeMap::new(), + direct_prefill: BTreeMap::new(), + direct_decode: BTreeMap::new(), + direct_decode_rungs: Vec::new(), }, }); if !building.cell.cell_ids.contains(&row.cell_id) { building.cell.cell_ids.push(row.cell_id.clone()); } if row.workload_kind == "prefill" { + if !row.fake_fallback { + building + .cell + .direct_prefill + .entry((row.batch_size, row.total_kv_read_tokens)) + .or_default() + .insert(row.total_prefill_tokens, row.latency_ms); + } building.cell.prefill.insert( &[ row.batch_size, @@ -867,6 +888,14 @@ fn load_pair( row.latency_ms, ); } else { + if !row.fake_fallback { + building + .cell + .direct_decode + .entry(row.batch_size) + .or_default() + .insert(row.total_kv_read_tokens, row.latency_ms); + } building .cell .decode @@ -878,6 +907,15 @@ fn load_pair( .into_values() .map(|b| { let mut cell = b.cell; + cell.direct_decode_rungs = cell + .direct_decode + .keys() + .copied() + .filter(|b| { + b.checked_add(1) + .is_some_and(|n| cell.direct_decode.contains_key(&n)) + }) + .collect(); cell.prefill_domain = domain::<3>(&cell.prefill); cell.decode_domain = domain::<2>(&cell.decode); if cell.prefill_domain.is_some() { diff --git a/crates/core/src/perfmodel/py.rs b/crates/core/src/perfmodel/py.rs index 87b0083807..8dbdf2e562 100644 --- a/crates/core/src/perfmodel/py.rs +++ b/crates/core/src/perfmodel/py.rs @@ -1030,6 +1030,9 @@ struct EngineBuildRequest { kv_block_size: Option, systems_path: Option, forward_model: Option, + fpm_profile: Option, + fpm_interpolation: Option, + cp_size: u32, database_mode: Option, shared_layer: Option, transfer_policy: Option>, @@ -1074,6 +1077,9 @@ impl AicEngineBuilder { kv_block_size: None, systems_path: None, forward_model: None, + fpm_profile: None, + fpm_interpolation: None, + cp_size: 1, database_mode: None, shared_layer: None, transfer_policy: None, @@ -1089,6 +1095,13 @@ impl AicEngineBuilder { self } + /// Supply independent FPM identity/resource metadata and interpolation mode. + pub fn fpm_profile(mut self, profile: serde_json::Value, interpolation: &str) -> Self { + self.request.fpm_profile = Some(profile.to_string()); + self.request.fpm_interpolation = Some(interpolation.to_owned()); + self + } + /// Select the performance-database lookup mode. Unset keeps SILICON. pub fn database_mode(mut self, database_mode: DatabaseMode) -> Self { self.request.database_mode = Some(database_mode.as_str().to_owned()); @@ -1286,7 +1299,7 @@ mod builder_tests { "enable_shared_layer": true, "transfer_policy": ["xshape", "xquant"], "strict_provenance": true, - "extra": {} + "extra": {"fpm_profile": "{\"model\":\"model\"}", "fpm_interpolation": "direct"} })) .expect("deserialize engine config"); @@ -1299,6 +1312,9 @@ mod builder_tests { Some(["xshape".to_owned(), "xquant".to_owned()].as_slice()) ); assert_eq!(request.strict_provenance, Some(true)); + assert_eq!(request.fpm_profile.as_deref(), Some(r#"{"model":"model"}"#)); + assert_eq!(request.fpm_interpolation.as_deref(), Some("direct")); + assert_eq!(request.cp_size, 1); } #[test] @@ -1328,6 +1344,12 @@ fn build_engine_from_request(request: EngineBuildRequest) -> Result Result { + if request.cp_size != 1 { + return Err(AicError::InvalidEngineConfig( + "cp_size must be the integer 1; this SDK entry point does not support context parallelism" + .to_string(), + )); + } if request.database_mode.as_deref() == Some(DatabaseMode::SolFull.as_str()) { return Err(AicError::InvalidEngineConfig( "database mode SOL_FULL is a per-call diagnostic and cannot be an engine default; use SOL instead" @@ -1358,6 +1380,12 @@ fn compile_engine_from_request(request: EngineBuildRequest) -> Result Result>() }) - // PyErr → AicError inline (keeps error.rs pyo3-free). A `compile_engine` - // failure means the model cannot be built natively, so it maps to - // `UnsupportedModel` — the variant `best_available` treats as - // fallback-safe. Hard caller/config errors use `InvalidEngineConfig` (which - // is NOT fallback-safe) so they surface instead of silently degrading. - .map_err(|e| AicError::UnsupportedModel(format!("compile_engine: {e}")))?; + // Explicit FPM requests must surface invalid metadata, precision conflicts, + // and construction failures. Keep the legacy unsupported-model fallback + // only for callers that did not opt into profile/interpolation settings. + .map_err(|e| { + let message = format!("compile_engine: {e}"); + if request.fpm_profile.is_some() || request.fpm_interpolation.is_some() { + AicError::InvalidEngineConfig(message) + } else { + AicError::UnsupportedModel(message) + } + })?; Engine::from_spec_bytes(&spec_bytes, systems_root.as_path() as &Path) } @@ -1402,7 +1435,30 @@ pub(crate) fn compile_engine_to_engine( config: &EngineConfig, systems_path: Option<&str>, ) -> Result { - compile_engine_from_request(engine_build_request(config, systems_path)) + let mut request = engine_build_request(config, systems_path); + if config.extra.contains_key("fpm_profile") { + let mut modes = Python::with_gil(|py| fpm_profile_quantization(py, config)) + .map_err(|e| AicError::InvalidEngineConfig(format!("FPM profile quantization: {e}")))?; + request.gemm_quant_mode = modes.remove("gemm_quant_mode"); + request.moe_quant_mode = modes.remove("moe_quant_mode"); + request.kvcache_quant_mode = modes.remove("kvcache_quant_mode"); + request.fmha_quant_mode = modes.remove("fmha_quant_mode"); + request.comm_quant_mode = modes.remove("comm_quant_mode"); + } + compile_engine_from_request(request) +} + +/// Restore profile precision shared by the native timing and memory bridges. +/// Python checks the wire dtypes before recovering exact quantization modes. +pub(crate) fn fpm_profile_quantization( + py: Python<'_>, + config: &EngineConfig, +) -> PyResult> { + let config_json = serde_json::to_string(config) + .map_err(|e| PyValueError::new_err(format!("invalid engine config: {e}")))?; + py.import("aiconfigurator_core.sdk.fpm_profile")? + .call_method1("_quantization_from_engine_config", (config_json,))? + .extract() } fn engine_build_request(config: &EngineConfig, systems_path: Option<&str>) -> EngineBuildRequest { @@ -1436,6 +1492,9 @@ fn engine_build_request(config: &EngineConfig, systems_path: Option<&str>) -> En kv_block_size: config.kv_block_size, systems_path: systems_path.map(str::to_owned), forward_model: config.forward_model.clone(), + fpm_profile: config.extra.get("fpm_profile").cloned(), + fpm_interpolation: config.extra.get("fpm_interpolation").cloned(), + cp_size: config.parallel.cp_size.unwrap_or(1), database_mode: Some(config.database_mode.as_str().to_owned()), shared_layer: config.enable_shared_layer, transfer_policy: config.transfer_policy.clone(), diff --git a/crates/core/src/python.rs b/crates/core/src/python.rs index b1e4bd2a2a..3970dbc581 100644 --- a/crates/core/src/python.rs +++ b/crates/core/src/python.rs @@ -180,6 +180,10 @@ struct AicTimingConfig { systems_path: Option, #[serde(default)] forward_model: Option, + #[serde(default)] + fpm_profile: Option, + #[serde(default)] + fpm_interpolation: Option, } const fn one() -> u32 { @@ -295,6 +299,17 @@ impl AicTimingModel { kwargs.set_item("kv_block_size", config.kv_block_size)?; kwargs.set_item("systems_path", config.systems_path.as_deref())?; kwargs.set_item("forward_model", config.forward_model.as_deref())?; + kwargs.set_item( + "fpm_profile", + config + .fpm_profile + .as_ref() + .map(serde_json::Value::to_string), + )?; + kwargs.set_item( + "fpm_interpolation", + config.fpm_interpolation.as_deref().unwrap_or("auto"), + )?; let spec = sdk.getattr("compile_engine")?.call( ( config.model.as_str(), @@ -413,6 +428,13 @@ fn estimate_aic_num_gpu_blocks(config: &AicTimingConfig, role: &ReplayRoleConfig kwargs.set_item("fmha_quant_mode", config.fmha_dtype.as_deref())?; kwargs.set_item("kvcache_quant_mode", config.kv_cache_dtype.as_deref())?; kwargs.set_item("comm_quant_mode", config.comm_dtype.as_deref())?; + kwargs.set_item( + "fpm_profile", + config + .fpm_profile + .as_ref() + .map(serde_json::Value::to_string), + )?; kwargs.set_item( "cuda_graph_reserved_bytes", config.cuda_graph_reserved_bytes, @@ -1318,6 +1340,8 @@ mod tests { cuda_graph_reserved_bytes: 0, systems_path: None, forward_model: None, + fpm_profile: None, + fpm_interpolation: None, } } @@ -1413,10 +1437,17 @@ mod tests { "backend": "vllm", "system": "test-system", "tp": 1, - "forward_model": "fpm" + "forward_model": "fpm", + "fpm_profile": {"model": "test-model"}, + "fpm_interpolation": "direct" })) .unwrap(); assert_eq!(config.forward_model.as_deref(), Some("fpm")); + assert_eq!( + config.fpm_profile, + Some(serde_json::json!({"model": "test-model"})) + ); + assert_eq!(config.fpm_interpolation.as_deref(), Some("direct")); } #[test] diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index 697874c9b8..f2c395f6c2 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -5,13 +5,13 @@ SPDX-License-Identifier: Apache-2.0 # FPM self-service -`aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. +`aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one TP, DEP, or TEP worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's per-worker FPM support and packaged collector. PR #46 is not a dependency. -Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model metadata and execution readiness, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not run target preflight checks, provision GPU resources, or establish measured accuracy. +Planning works before the model has an AISimulate model class or measured FPM timings. With a supplied FPM profile, planning validates the declared deployment identity and estimates memory admission from your resource bounds. Runtime compatibility and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not provision GPUs or run target preflight checks. -In this revision, ordinary FPM `predict` and `recommend` still construct a registered analytical model for resource accounting and SOL-based timing transfer. A separate decoupling change will add a route that uses model/resource metadata and direct interpolation of measured timings without an op-level model class. The existing registered-model/SOL route will remain available. See [Choose the model execution route](#choose-the-model-execution-route) before handing off a new architecture. +Ordinary FPM `predict` and `recommend` accept an inline `engine.fpm_profile` with model identity and rank-local resource bounds. Direct interpolation uses measured timings without constructing an op-level model. Registered models retain SOL interpolation. See [Choose the model execution route](#choose-the-model-execution-route) for selection and coverage rules. -The Inkling pilot is planned for NVIDIA GB200 with TP, DEP, and TEP configurations through the class-independent route. This guided planner currently generates pure-TP configurations only. The checkpoint, runtime, allocation, and strategy degrees must be pinned before that pilot; this guide does not establish Inkling readiness or GB200 accuracy. +The Inkling pilot targets NVIDIA GB200 with TP, DEP, and TEP through the class-independent route. Its checkpoint, runtime, allocation, profile resource bounds, and strategy degrees must be pinned before collection. This guide does not establish Inkling readiness or GB200 accuracy. ## Create the request @@ -40,7 +40,41 @@ Replace the model and runtime placeholders with the actual inputs. The example d The default pilot uses 1,024 input tokens, 128 output tokens, concurrency 1, four requests, a 16,384-token context limit, TTFT target 1,000 ms, and TPOT target 100 ms. Scripted setup defaults to TP1 unless `--tensor-parallel` is supplied. These are planning defaults, not measured model capacity or latency. Change them with the corresponding flags shown by `aisimulate onboard init --help`. -One node is the default; GPUs per node then equals `--gpu-count`. For multiple nodes, supply both `--node-count` and `--gpus-per-node`; their product must equal the total allocation. Each pure-TP worker must fit on one node. Advanced flags include `--request-count`, `--max-candidates`, `--objective`, `--seed`, and `--sm`. +One node is the default; GPUs per node then equals `--gpu-count`. For multiple nodes, supply both `--node-count` and `--gpus-per-node`; their product must equal the total allocation. Each selected worker must fit on one node. Advanced flags include `--request-count`, `--max-candidates`, `--objective`, `--seed`, and `--sm`. + +`--tensor-parallel` always means attention TP. Use all MoE dimensions explicitly for DEP and TEP; replicas are independent workers: + +| Worker | CLI parallelism flags | `(TP, PP, attention DP, MoE TP, MoE EP, CP)` | +| --- | --- | --- | +| MoE TP4 | `--tensor-parallel 4 --moe-tensor-parallel 4` | `(4, 1, 1, 4, 1, 1)` | +| DEP8 | `--tensor-parallel 1 --attention-data-parallel 8 --moe-tensor-parallel 1 --moe-expert-parallel 8` | `(1, 1, 8, 1, 8, 1)` | +| TEP8 | `--tensor-parallel 8 --attention-data-parallel 1 --moe-tensor-parallel 1 --moe-expert-parallel 8` | `(8, 1, 1, 1, 8, 1)` | + +The first profile implementation supports vLLM decoder-only models with PP1, CP1 and linear KV storage. AFD, encoder pools, speculative decoding, nonlinear recurrent state, wide EP and EPLB need additional metadata/semantics and are rejected by this route. + +## Provide identity and resource metadata + +For a model without an analytical class, create a JSON or YAML FPM profile and pass `--fpm-profile /path/to/model-profile.yaml` to `onboard init`. The request embeds the complete profile; ordinary prediction and recommendation use the same object under `engine.fpm_profile`. The profile schema is [FpmModelProfile](../python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py). It rejects missing fields, unknown fields, conflicting identities and mutable revision placeholders. + +| Profile fields | Required meaning | +| --- | --- | +| `schema_version` | Integer `1`. | +| `model`, `model_revision`, `architecture` | Exact timing model identity, pinned checkpoint revision, and architecture identifier. An unknown architecture is valid for direct interpolation. | +| `context_length`, `num_experts`, `provenance` | Declared context limit, routed expert count (`0` for dense), and how the metadata was obtained. | +| `deployments` | One or more exact deployment records, with one precision/resource identity per hardware/runtime/full parallel tuple. | +| Deployment `system`, `backend`, `backend_version` | GPU system, `vllm`, and literal runtime version. | +| Deployment `tp`, `pp`, `dp`, `moe_tp`, `moe_ep`, `cp` | Complete topology. PP and CP default to `1`; other dimensions are explicit. | +| Deployment precision | Exact `gemm_quant_mode`, `moe_quant_mode`, `fmha_quant_mode`, `comm_quant_mode`, `kv_cache_dtype`. They must match the collected cell; FP8 FMHA and BF16 FMHA are separate identities. | +| Deployment runtime hints | `moe_backend`, `attention_backend` (default `auto`), `enable_wideep`, `enable_eplb` (currently both `false`). | +| Deployment `resources` | Conservative bounds described below, specific to this topology. | + +Each `resources` record requires integer `weights_bytes`, `activations_bytes`, `runtime_overhead_bytes`, `comm_overhead_bytes`, positive `kv_bytes_per_token`, `cache_layout: linear`, positive `max_num_tokens` and `max_batch_size`, and a `provenance` explanation. Obtain these from checkpoint tensor/storage metadata and serving-runtime accounting or conservative explicit estimates. A profile is an input declaration; AISimulate does not certify that its values describe your runtime. + +All resource bytes are **per rank**, and must bound every rank. Scheduler envelope limits are also per attention-DP rank; they are not the summed batch and token totals used by the worker's FPM timing query. Do not copy TP resource values into DEP/TEP merely because GPU counts match. Include all non-KV storage, including quantization overheads. Overhead fields exclude the separate `kv_cache.capacity.cuda_graph_reserved_bytes` reservation. Requests above the declared scheduler envelope fail explicitly. + +Automatic KV admission subtracts these non-KV bounds and CUDA graph reservation from the configured fraction of GPU memory, then divides by the declared KV bytes per token. `context_length: max` uses the profile limit; `bytes_per_token: auto` uses the exact deployment's cache geometry. These operations need a hardware specification and profile, but no FPM timing files or model graph. + +The declared checkpoint revision is preserved for review and replay. Existing FPM tables may not pin a checkpoint revision; supplying one in a profile does not prove that the historical measurements came from that exact revision. Record that limitation in `provenance` when validating existing data. ## Plan, preview, and explicitly execute @@ -53,11 +87,11 @@ aisimulate onboard collect-fpm \ --output-dir ./aisimulate-support ``` -The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. The second prints the collector command without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. On an existing plan, `--overwrite` can repair missing generated files for the identical request; it rejects changed inputs and preserves existing collected data. +The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. When supplied, the profile is also saved as `fpm-model-profile.json`, included in the collector command, and embedded in prediction/recommendation configs. The plan records the CPU resource estimate. The second command prints the collector invocation without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. On an existing plan, `--overwrite` can repair missing generated files for the identical request; it rejects changed inputs and preserves existing collected data. If a newer draft revision changes generated guidance, recreate the plan in a new output directory: repair compares generated files byte for byte and does not migrate earlier draft plans. Guidance changes do not change the saved-request identity checks used by collection. -The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. +The guided plan uses one selected parallel tuple. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Collection uses the matching `tp`, `pure_tp`, `dep`, or `tep` preset and exact worker GPU count. Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate onboard collect-fpm --execute` command for collection, alongside a read-only collector planning command. @@ -69,7 +103,11 @@ aisimulate onboard collect-fpm \ --output-dir ./aisimulate-support --execute ``` -Execution requires a matching saved plan. The request records model and runtime revisions; this setup does not download a pinned checkpoint or verify the installed runtime against them. Keep the actual checkpoint and runtime consistent with the request before collecting or predicting. +Execution requires a matching saved plan. Creating the initial plan records model and runtime revisions without downloading a pinned checkpoint or inspecting the running runtime. During profile-based collection execution, the collector checks the observed Pod's vLLM version against the profile's literal backend version before benchmarking. Keep the actual checkpoint consistent with the declared model revision. + +For a profile-based campaign, the collector validates its resolved topology and precision against the supplied profile before execution. It does not relabel an FP8 cell as BF16 or change checkpoint quantization to satisfy the profile. A mismatch reports the conflicting field and requires a matching profile/runtime or a supported collector configuration. Resource admission uses the supplied bounds without constructing an analytical model. Profile contents participate in the collector's frozen-plan identity. + +New profile-based collection currently uses checkpoint-native weight, FMHA and KV precision without consulting op-level timing tables. The requested KV dtype must match that checkpoint-native dtype. An older FPM profile can still be valid for prediction while being unsuitable for new collection: the historical MiniMax/H200 BF16 FMHA fallback cell differs from its checkpoint-native FP8 inference. Use the historical identity to query those timings and a matching native profile for new collection; the collector rejects that mismatch. Publish the new FP8 campaign into a clean, separate dataset using a new onboarding output directory. Existing cell IDs omit FMHA precision, and publication retains the first published run for a cell ID. If the historical BF16 dataset already holds that cell ID, publication skips the new FP8 run. A collection profile must select one literal runtime version for the target hardware/backend. Set deployment options directly on `onboard collect-fpm`: `--dynamo-version VERSION`, `--image IMAGE`, `--namespace NAME`, `--model-cache NAME[:MOUNT[:SUBPATH]]`, `--transport nvlink|ib|efa`, and `--image-pull-secret NAME`. The mount, when supplied, is an absolute container path. Prefer an immutable image digest. Supply the same options when previewing, executing, and resuming; deployment settings are part of the collector's frozen-plan identity, so changed settings require a new output directory. Arbitrary collector arguments and engine overrides are not accepted by this command. @@ -77,7 +115,7 @@ For a diagnostic run, add `--execute --smoke`; `--limit N` also requires `--smok Planning and collection reject concurrent onboarding operations. The persistent `.support.lock` file uses an OS advisory lock; ownership is released when the process exits, including after an abrupt termination. Leave the file in place. Request validation, saved onboarding-plan checks, and collector input resolution exit 2. Failures after collector execution starts, including a frozen checkpoint identity mismatch, exit 1 with a concise message. Interruption exits 130. -The collector narrows initial prefill sampling with the pilot's input-token and concurrency bounds. Decode uses the collector's existing profile; a four-request synthetic pilot does not imply four timing samples or a short decode campaign. Inspect the generated command and collector plan before committing GPU time. Successful formal collection publishes the FPM Parquet file and metadata pair into the plan's local systems data directory; diagnostic success alone does not provide that pair. +The collector narrows initial prefill sampling with the pilot's input-token and concurrency bounds. With an FPM profile, both the sample batch size and total new-token axis are capped by the same rank-local scheduler limits used in the generated prediction and recommendation configs. DEP does not multiply these limits by the GPU count; coverage of the worker's summed FPM queries still requires validation. Decode uses the collector's existing sampling profile within the declared resource envelope; a four-request synthetic pilot does not imply four timing samples or a short decode campaign. Inspect the generated command and collector plan before committing GPU time. Successful formal collection publishes the FPM Parquet file and metadata pair into the plan's local systems data directory; diagnostic success alone does not provide that pair. ## Run the generated ordinary configurations @@ -107,7 +145,7 @@ from aisimulate.support.schema import SupportRequest root = Path("./aisimulate-support").resolve() request = SupportRequest.from_yaml(root / "request.yaml") check_plan(request, root) -max_replicas = request.identity.node_count * (request.identity.gpus_per_node // request.search.tensor_parallel) +max_replicas = request.identity.node_count * (request.identity.gpus_per_node // request.worker_gpus) replicas = list(dict.fromkeys((1, max_replicas)))[:request.search.max_candidates] statuses = [] for count in replicas: @@ -131,13 +169,17 @@ Run either the individual recommendation command or the loop against fresh resul The generated configurations select `engine.workers.aggregated.timing.forward_model: fpm` and set `engine.systems_path` to the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. -Ordinary `predict` and `recommend` commands retain their existing behavior and defaults. Their generated configs can be loaded and edited through the public configuration schema. In this revision, missing model registration or data may still prevent execution; a successful simulation is not an accuracy result. Compare its output with an independent run of the same model, runtime, topology, and workload to assess accuracy. +Ordinary `predict` and `recommend` retain their existing defaults when no profile is supplied. Generated configs can be edited through the public schema. A recommendation with a profile enumerates only its declared deployment tuples that fit the GPU budget; it preserves the complete inline profile and interpolation choice in exported prediction configs. Default scheduler search ranges may exceed a profile's envelope, so pin or bound those domains explicitly. A successful simulation is not an accuracy result. Compare its output with an independent run of the same model, runtime, topology, and workload to assess accuracy. ## Choose the model execution route Hand off the saved request, pinned model configuration, and plan. Both routes need a canonical checkpoint identity, effective precision and topology, correct weight and KV-cache accounting, and matching whole-forward FPM measurements. Collected timings alone do not establish memory fit. -- **Registered-model/SOL route, available in this revision:** reuse a compatible analytical class or follow [How to Add a New Model](../python/aisimulate/docs/add_a_new_model.md) when choosing to add one. Verify its operation graph, memory and cache accounting, and native FPM SOL execution. A dedicated class is an option for this route, not the intended prerequisite for every FPM onboarding. -- **Class-independent route, planned in the separate decoupling change:** resolve model and resource metadata without constructing an operation graph. Use direct measured-time interpolation, including wider two-sided KV brackets at the same batch size when both neighboring prompt curves cover the query. Missing brackets or unsupported metadata must produce explicit errors. This route is not implemented by the guided foundation; 2D interpolation remains experimental. +- **Registered-model/SOL route:** reuse a compatible analytical class or follow [How to Add a New Model](../python/aisimulate/docs/add_a_new_model.md) when choosing to add one. Verify its operation graph, memory/cache accounting, and native FPM SOL execution. +- **Class-independent direct route:** supply the identity/resource profile and set worker `timing: {type: default, forward_model: fpm, fpm_interpolation: direct}`. The guided planner selects this route whenever a profile is supplied. No operation graph is constructed for resources, timing, or recommendation candidates. + +`fpm_interpolation: auto` retains SOL for a registered architecture and chooses direct for an unregistered architecture with a profile. Explicit `sol` requires a registered class; explicit `direct` requires a profile. A model-construction error does not trigger a silent change of method. The interpolation setting applies to default FPM timing only. + +Direct timing first uses an exact point or interpolation within a measured curve. Prefill interpolation stays at the same batch size, with two measured KV neighbors whose prompt curves both cover the requested token count. Wider KV bracketing removes the old distance limit only when the narrower direct bracket is unavailable. Decode interpolation respects the measured batch/capture domain. Both phases exclude synthetic `fake_fallback` rows, including healed/extrapolated values. Missing two-sided support, unmeasured batches and out-of-domain queries fail explicitly. The direct route does not apply SOL-dependent prefill batch clamping or general extrapolation; 2D interpolation remains experimental. -Per-operation silicon profiling described in the model guide is not required by either FPM route. The intended self-service workflow collects whole-forward timings, then verifies prediction and recommendation for the exact target deployment. Collection bootstrap can already resolve some unregistered model configurations; that does not establish that this revision's ordinary FPM prediction path can construct them. +Per-operation silicon profiling described in the model guide is not required by either FPM route. The workflow collects whole-forward timings, then verifies prediction and recommendation for the exact target deployment. Existing MiniMax-M2.7/H200 TP4 and GLM-5.2/B200 DEP8/TEP8 data provide functional validation targets. Coverage and interpolation error must be reported separately; these targets do not qualify Inkling or GB200. diff --git a/python/aisimulate/collector/fpm_forward/capabilities.py b/python/aisimulate/collector/fpm_forward/capabilities.py index 9beb46cd60..e060d8b99a 100644 --- a/python/aisimulate/collector/fpm_forward/capabilities.py +++ b/python/aisimulate/collector/fpm_forward/capabilities.py @@ -181,8 +181,13 @@ def resolve_model_capability( requested_kv_cache_dtypes: tuple[str, ...], model_config_path: str | None = None, database_version: str | None = None, + checkpoint_native_dtypes: bool = False, ) -> ModelCapabilityProfile: - """Resolve a conservative matrix profile from AIC code and perf data.""" + """Resolve checkpoint identity with optional legacy op-data dtype evidence. + + Profile collection supplies a literal runtime version and retains native + checkpoint precision without requiring previously collected timing data. + """ # The CLI maps an empty --fpm-kv-cache-dtypes to ("auto",); a programmatic # caller passing an empty tuple would otherwise crash below on @@ -251,34 +256,48 @@ def resolve_model_capability( f"its AIC-inferred GEMM mode {gemm!r}, got {sorted(requested_weights)}" ) + if checkpoint_native_dtypes and database_version is None: + raise ValueError("checkpoint-native FPM collection requires a literal profile backend_version") version = database_version or get_latest_database_version(system=system, backend=backend) if not version: raise ValueError(f"no AIC database version is available for system={system!r}, backend={backend!r}") - database = get_database(system, backend, version) - if database is None: - raise ValueError(f"failed to load AIC database for system={system!r}, backend={backend!r}, version={version!r}") - supported = getattr(database, "supported_quant_mode", {}) or {} context_op, generation_op = _attention_ops(source_name) + requested_kv = tuple(dict.fromkeys(_normalize_kv_dtype(value) for value in requested_kv_cache_dtypes)) + resolved_kv = tuple(dict.fromkeys(native_kv if value == "auto" else value for value in requested_kv)) + database = None + if checkpoint_native_dtypes: + if resolved_kv != (native_kv,): + raise ValueError( + "profile-based FPM collection currently requires the checkpoint-native KV dtype " + f"{native_kv!r}; got {list(resolved_kv)}" + ) + else: + database = get_database(system, backend, version) + if database is None: + raise ValueError( + f"failed to load AIC database for system={system!r}, backend={backend!r}, version={version!r}" + ) + supported = getattr(database, "supported_quant_mode", {}) or {} - _require_mode(supported, "gemm", gemm) - if is_moe: - _require_mode(supported, "moe", moe) + _require_mode(supported, "gemm", gemm) + if is_moe: + _require_mode(supported, "moe", moe) - context_modes = _supported_modes(supported, context_op) - if not context_modes: - raise ValueError(f"AIC database has no context-attention dtype evidence for template op {context_op!r}") + context_modes = _supported_modes(supported, context_op) + if not context_modes: + raise ValueError(f"AIC database has no context-attention dtype evidence for template op {context_op!r}") - generation_modes = _supported_modes(supported, generation_op) - if not generation_modes: - raise ValueError(f"AIC database has no generation-attention dtype evidence for template op {generation_op!r}") - requested_kv = tuple(dict.fromkeys(_normalize_kv_dtype(value) for value in requested_kv_cache_dtypes)) - resolved_kv = tuple(dict.fromkeys(native_kv if value == "auto" else value for value in requested_kv)) - unsupported_kv = [value for value in resolved_kv if value not in generation_modes] - if unsupported_kv: - raise ValueError( - f"AIC database does not support KV-cache dtype(s) {unsupported_kv} for {generation_op}; " - f"supported modes: {sorted(generation_modes)}" - ) + generation_modes = _supported_modes(supported, generation_op) + if not generation_modes: + raise ValueError( + f"AIC database has no generation-attention dtype evidence for template op {generation_op!r}" + ) + unsupported_kv = [value for value in resolved_kv if value not in generation_modes] + if unsupported_kv: + raise ValueError( + f"AIC database does not support KV-cache dtype(s) {unsupported_kv} for {generation_op}; " + f"supported modes: {sorted(generation_modes)}" + ) # fmha_quant_mode names the AIC data slice a template transfers # utilization from, so it is resolved PER KV DTYPE against joint @@ -293,6 +312,18 @@ def resolve_model_capability( fmha_by_kv: dict[str, str] = {} fmha_resolution_by_kv: dict[str, str] = {} for kv_name in resolved_kv: + if checkpoint_native_dtypes: + # Profile collection uses the checkpoint mode, without borrowing + # an op-table slice. This is a whole-forward identity, not a + # claim that every attention intermediate has this dtype: vLLM + # v0.25.1 (752a3a504485790a2e8491cacbb35c137339ad34), + # vllm/v1/attention/backends/mla/flashmla_sparse.py:879-894 + # selects its FP8-cache route while that route can use BF16 + # prefill intermediates. Actual backend dispatch stays in vLLM. + fmha_by_kv[kv_name] = inferred_fmha + fmha_resolution_by_kv[kv_name] = "checkpoint_native" + continue + assert database is not None joint_modes = context_fmha_supported_modes(database, context_op, common.KVCacheQuantMode[kv_name]) if inferred_fmha in joint_modes: fmha_by_kv[kv_name] = inferred_fmha diff --git a/python/aisimulate/collector/fpm_forward/config.py b/python/aisimulate/collector/fpm_forward/config.py index 3ffc02b7af..52e2158140 100644 --- a/python/aisimulate/collector/fpm_forward/config.py +++ b/python/aisimulate/collector/fpm_forward/config.py @@ -312,6 +312,12 @@ def add_fpm_arguments(parser: argparse.ArgumentParser) -> None: "FPM forward collection", "Whole-model forward-pass planning, execution, and publication.", ) + group.add_argument( + "--fpm-model-profile", + default=None, + metavar="PATH", + help="JSON or YAML FPM model profile with exact deployment identities and rank-local resource bounds.", + ) group.add_argument( "--fpm-max-gpus", type=_positive_int, @@ -515,6 +521,7 @@ def reject_fpm_arguments_without_fpm(args: argparse.Namespace) -> None: "fpm_weight_quantizations", "fpm_kv_cache_dtypes", "fpm_model_config", + "fpm_model_profile", "fpm_tp_sizes", "fpm_pp_sizes", "fpm_dp_sizes", diff --git a/python/aisimulate/collector/fpm_forward/entry.py b/python/aisimulate/collector/fpm_forward/entry.py index 96f226bd35..e494b05c75 100644 --- a/python/aisimulate/collector/fpm_forward/entry.py +++ b/python/aisimulate/collector/fpm_forward/entry.py @@ -7,10 +7,13 @@ import argparse import copy +from pathlib import Path from typing import Any import yaml +from aiconfigurator_core.sdk.fpm_profile import load_fpm_profile + from .config import FPMCollectionOptions from .planner import FPMCollectionPlan, build_collection_plan @@ -140,6 +143,13 @@ def resolve_inputs(args: argparse.Namespace, case_plan) -> ResolvedFPMInputs: options = FPMCollectionOptions.from_args(args) generator_overrides = copy.deepcopy(_load_generator_overrides(args)) + profile_path = getattr(args, "fpm_model_profile", None) + fpm_profile = None + if profile_path is not None: + payload = yaml.safe_load(Path(profile_path).expanduser().read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("--fpm-model-profile must contain a JSON or YAML mapping") + fpm_profile = load_fpm_profile(payload) plan = build_collection_plan( backend=args.backend, model_path=case_plan.model_path, @@ -149,6 +159,7 @@ def resolve_inputs(args: argparse.Namespace, case_plan) -> ResolvedFPMInputs: model_architecture=case_plan.model_architecture, has_model_cases=bool(case_plan.model_cases_paths), model_config_path=getattr(args, "fpm_model_config", None), + fpm_profile=fpm_profile, collector_config={}, generator_overrides=generator_overrides, ) diff --git a/python/aisimulate/collector/fpm_forward/memory_admission.py b/python/aisimulate/collector/fpm_forward/memory_admission.py index 89369987f3..3ee0d46a23 100644 --- a/python/aisimulate/collector/fpm_forward/memory_admission.py +++ b/python/aisimulate/collector/fpm_forward/memory_admission.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Conservative AIC memory admission for FPM parallel topologies.""" +"""Conservative memory admission for FPM parallel topologies.""" from __future__ import annotations @@ -11,6 +11,8 @@ from aiconfigurator.sdk.memory import KVCacheEstimator from aiconfigurator_core.sdk.errors import PerfDataNotAvailableError +from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile +from aiconfigurator_core.sdk.perf_database import load_system_spec from .capabilities import ModelCapabilityProfile from .types import ParallelTopology @@ -25,9 +27,10 @@ class DTypeMemoryEstimate: estimated_non_kv_bytes: int | None gpu_capacity_bytes: int | None reason: str + provenance: str | None = None def to_dict(self) -> dict[str, object]: - return { + payload = { "kv_cache_dtype": self.kv_cache_dtype, "disposition": self.disposition, "estimated_non_kv_bytes": self.estimated_non_kv_bytes, @@ -39,6 +42,9 @@ def to_dict(self) -> dict[str, object]: ), "reason": self.reason, } + if self.provenance is not None: + payload["provenance"] = self.provenance + return payload @dataclass(frozen=True, slots=True) @@ -48,17 +54,29 @@ class TopologyMemoryDecision: max_new_tokens: int estimates: tuple[DTypeMemoryEstimate, ...] reason: str + source: str = "aic_native_configured_max_new_tokens" + max_batch_size: int = 1 + profile_max_num_tokens: int | None = None def to_dict(self) -> dict[str, object]: + envelope = { + "max_new_tokens": self.max_new_tokens, + "aic_max_num_tokens": self.max_new_tokens, + "max_batch_size": self.max_batch_size, + } + if self.profile_max_num_tokens is not None: + envelope = { + "requested_prefill_tokens": self.max_new_tokens, + "max_num_tokens": self.profile_max_num_tokens, + "max_batch_size": self.max_batch_size, + "scope": "rank_local", + "cuda_graph_reservation": "runtime_profiled_separately", + } return { "topology": self.topology.to_dict(), "disposition": self.disposition, - "source": "aic_native_configured_max_new_tokens", - "activation_envelope": { - "max_new_tokens": self.max_new_tokens, - "aic_max_num_tokens": self.max_new_tokens, - "max_batch_size": 1, - }, + "source": self.source, + "activation_envelope": envelope, "estimates": [estimate.to_dict() for estimate in self.estimates], "reason": self.reason, } @@ -73,6 +91,8 @@ def _estimate_dtype( topology: ParallelTopology, kv_cache_dtype: str, max_new_tokens: int, + fpm_profile: FpmModelProfile | None = None, + max_batch_size: int | None = None, ) -> DTypeMemoryEstimate: # Planner-owned capability data fails closed: resolve_model_capability # guarantees an fmha mapping for every resolved KV dtype, so a missing @@ -84,6 +104,41 @@ def _estimate_dtype( raise ValueError( f"capability profile invariant violated: kv dtype {kv_cache_dtype!r} has no fmha mapping" ) from error + if fpm_profile is not None: + deployment = fpm_profile.select( + model=model_path, + system=system, + backend=backend, + backend_version=capability.aic_database_version, + tp_size=topology.tp, + pp_size=topology.pp, + attention_dp_size=topology.dp, + moe_tp_size=topology.moe_tp, + moe_ep_size=topology.moe_ep, + cp_size=topology.cp, + ) + resources = deployment.resources + resources.validate_envelope( + max_num_tokens=max_new_tokens, + max_batch_size=max_batch_size or resources.max_batch_size, + ) + capacity = math.floor(float(load_system_spec(system)["gpu"]["mem_capacity"])) + if capacity <= 0: + raise ValueError(f"invalid GPU memory capacity for {system!r}: {capacity}") + rejected = resources.non_kv_bytes >= capacity + return DTypeMemoryEstimate( + kv_cache_dtype=kv_cache_dtype, + disposition="rejected" if rejected else "admitted", + estimated_non_kv_bytes=resources.non_kv_bytes, + gpu_capacity_bytes=capacity, + reason=( + "declared rank-local non-KV resource bound is not below GPU capacity" + if rejected + else "declared rank-local non-KV resource bound is below GPU capacity; " + "runtime profiling still determines KV capacity and CUDA-graph memory" + ), + provenance=resources.provenance, + ) try: breakdown = KVCacheEstimator.from_request( model_path, @@ -168,13 +223,16 @@ def filter_memory_infeasible_topologies( capability: ModelCapabilityProfile, topologies: tuple[ParallelTopology, ...], max_new_tokens: int, + fpm_profile: FpmModelProfile | None = None, + max_batch_size: int | None = None, ) -> tuple[tuple[ParallelTopology, ...], tuple[TopologyMemoryDecision, ...]]: """Drop topologies that cannot fit the configured max-new-token envelope. This is intentionally a one-sided generation-time filter. A topology is - rejected only when every requested KV dtype has a successful AIC estimate - and all estimates exceed rank-local physical capacity. Unknown estimates - remain runnable so the target runtime stays authoritative. + rejected only when every requested KV dtype has a successful estimate + and all estimates exceed rank-local physical capacity. Without a profile, + unknown AIC estimates remain runnable. Supplied profiles must explicitly + cover the requested deployment and envelope; validation errors propagate. """ if max_new_tokens < 1: @@ -184,6 +242,22 @@ def filter_memory_infeasible_topologies( admitted = [] rejected_capacity = [] for topology in topologies: + resources = ( + fpm_profile.select( + model=model_path, + system=system, + backend=backend, + backend_version=capability.aic_database_version, + tp_size=topology.tp, + pp_size=topology.pp, + attention_dp_size=topology.dp, + moe_tp_size=topology.moe_tp, + moe_ep_size=topology.moe_ep, + cp_size=topology.cp, + ).resources + if fpm_profile is not None + else None + ) estimates = tuple( _estimate_dtype( backend=backend, @@ -193,6 +267,8 @@ def filter_memory_infeasible_topologies( topology=topology, kv_cache_dtype=kv_cache_dtype, max_new_tokens=max_new_tokens, + fpm_profile=fpm_profile, + max_batch_size=max_batch_size, ) for kv_cache_dtype in capability.dtype.kv_cache_dtypes ) @@ -216,6 +292,9 @@ def filter_memory_infeasible_topologies( max_new_tokens=max_new_tokens, estimates=estimates, reason=reason, + source="fpm_profile_declared" if fpm_profile is not None else "aic_native_configured_max_new_tokens", + max_batch_size=resources.max_batch_size if resources is not None else 1, + profile_max_num_tokens=resources.max_num_tokens if resources is not None else None, ) ) @@ -237,17 +316,19 @@ def filter_memory_infeasible_topologies( f"{best.gpu_capacity_bytes / 2**30:.2f} GiB" ) logger.warning( - "fpm_forward: dropped %d/%d topologies (AIC configured max-new-token non-KV memory " + "fpm_forward: dropped %d/%d topologies (%s configured max-new-token non-KV memory " "exceeds GPU capacity, system=%s, max_new_tokens=%d): %s", len(rejected_capacity), len(topologies), + "FPM profile" if fpm_profile is not None else "AIC", system, max_new_tokens, "; ".join(details), ) if not admitted: + source = "FPM profile" if fpm_profile is not None else "AIC" raise ValueError( - "AIC max-new-token memory admission rejected every FPM topology; " + f"{source} max-new-token memory admission rejected every FPM topology; " f"model={model_path!r}, system={system!r}, max_new_tokens={max_new_tokens}" ) return tuple(admitted), tuple(decisions) diff --git a/python/aisimulate/collector/fpm_forward/planner.py b/python/aisimulate/collector/fpm_forward/planner.py index a02c010e5c..24aa5fca83 100644 --- a/python/aisimulate/collector/fpm_forward/planner.py +++ b/python/aisimulate/collector/fpm_forward/planner.py @@ -15,12 +15,14 @@ import logging import os import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from typing import Any +from aiconfigurator_core.sdk.fpm_profile import FpmDeploymentProfile, FpmModelProfile, load_fpm_profile + from .capabilities import ModelCapabilityProfile, ResolvedDTypeProfile, resolve_model_capability -from .config import FPMCollectionOptions +from .config import PARALLEL_AXES, FPMCollectionOptions from .memory_admission import TopologyMemoryDecision, filter_memory_infeasible_topologies from .topology import enumerate_fpm_topologies, topology_strategy from .types import ParallelTopology @@ -481,9 +483,32 @@ class FPMCollectionPlan: backend_policies: tuple[BackendPolicy, ...] cells: tuple[FPMCell, ...] sha256: str + _fpm_profile_json: str | None = field(default=None, repr=False) + + @property + def fpm_profile(self) -> FpmModelProfile | None: + """Return detached profile metadata from the frozen collection input.""" + return load_fpm_profile(self._fpm_profile_json) if self._fpm_profile_json is not None else None + + def deployment_profile(self, cell: FPMCell) -> FpmDeploymentProfile | None: + profile = self.fpm_profile + if profile is None: + return None + return profile.select( + model=self.model_path, + system=self.system, + backend=self.backend, + backend_version=self.capability.aic_database_version, + tp_size=cell.topology.tp, + pp_size=cell.topology.pp, + attention_dp_size=cell.topology.dp, + moe_tp_size=cell.topology.moe_tp, + moe_ep_size=cell.topology.moe_ep, + cp_size=cell.topology.cp, + ) def to_dict(self) -> dict[str, object]: - return { + payload = { "schema_name": "aic_fpm_collection_plan", "schema_version": 10, "backend": self.backend, @@ -535,6 +560,9 @@ def to_dict(self) -> dict[str, object]: }, "sha256": self.sha256, } + if self._fpm_profile_json is not None: + payload["fpm_profile"] = json.loads(self._fpm_profile_json) + return payload def _cell_id( @@ -572,12 +600,31 @@ def build_collection_plan( model_architecture: str | None = None, has_model_cases: bool = True, model_config_path: str | None = None, + fpm_profile: FpmModelProfile | dict[str, Any] | None = None, collector_config: dict[str, Any] | None = None, generator_overrides: dict[str, Any] | None = None, ) -> FPMCollectionPlan: if backend != "vllm": raise ValueError("FPM Generator V1 currently supports only backend=vllm") - collector_config = collector_config or {} + collector_config = dict(collector_config or {}) + profile = load_fpm_profile(fpm_profile) if fpm_profile is not None else None + if profile is not None: + if profile.model != model_path: + raise ValueError( + f"FPM profile model identity mismatch: requested {model_path!r}, profile={profile.model!r}" + ) + versions = { + deployment.backend_version + for deployment in profile.deployments + if deployment.system == system and deployment.backend == backend + } + if "aic_database_version" not in collector_config: + if len(versions) != 1: + raise ValueError( + "FPM collection profile must identify one runtime version for the target " + f"{system}/{backend}; found {sorted(versions)}" + ) + collector_config["aic_database_version"] = next(iter(versions)) generator_config_sha256 = _canonical_hash(generator_overrides or {}) capability = resolve_model_capability( backend=backend, @@ -592,6 +639,7 @@ def build_collection_plan( database_version=( str(collector_config["aic_database_version"]) if "aic_database_version" in collector_config else None ), + checkpoint_native_dtypes=profile is not None, ) candidate_topologies = enumerate_fpm_topologies( backend=backend, @@ -599,6 +647,9 @@ def build_collection_plan( options=options, allow_pure_tp=capability.allow_pure_tp, ) + policies = _backend_policies(options, collector_config, backend=backend) + if profile is not None: + _validate_profile_identities(profile, capability, candidate_topologies, policies, model_path, system, backend) topologies, topology_memory_admission = filter_memory_infeasible_topologies( backend=backend, model_path=model_path, @@ -606,8 +657,9 @@ def build_collection_plan( capability=capability, topologies=candidate_topologies, max_new_tokens=options.prefill_sampling.max_total_prefill_tokens, + fpm_profile=profile, + max_batch_size=options.max_prefill_batch_size, ) - policies = _backend_policies(options, collector_config, backend=backend) weight_quantization = capability.dtype.gemm_quant_mode runnable_dtype_pairs = { (decision.topology, estimate.kv_cache_dtype) @@ -681,6 +733,10 @@ def build_collection_plan( "policies": [policy.to_dict() for policy in policies], "cells": [cell.to_dict() for cell in cells], } + profile_json = None + if profile is not None: + canonical["fpm_profile"] = profile.model_dump(mode="json") + profile_json = json.dumps(canonical["fpm_profile"], sort_keys=True, separators=(",", ":")) return FPMCollectionPlan( backend=backend, model_path=model_path, @@ -695,4 +751,70 @@ def build_collection_plan( backend_policies=policies, cells=cells, sha256=_canonical_hash(canonical), + _fpm_profile_json=profile_json, ) + + +def _validate_profile_identities( + profile: FpmModelProfile, + capability: ModelCapabilityProfile, + topologies: tuple[ParallelTopology, ...], + policies: tuple[BackendPolicy, ...], + model_path: str, + system: str, + backend: str, +) -> None: + """Require a resource declaration for every requested cell, before admission.""" + if profile.architecture != capability.architecture: + raise ValueError( + "FPM collection profile architecture mismatch: " + f"checkpoint={capability.architecture!r}, profile={profile.architecture!r}" + ) + identity_fields = ( + "gemm_quant_mode", + "moe_quant_mode", + "fmha_quant_mode", + "comm_quant_mode", + "kv_cache_dtype", + *PARALLEL_AXES, + "moe_backend", + "attention_backend", + "enable_wideep", + "enable_eplb", + ) + for topology in topologies: + deployment = profile.select( + model=model_path, + system=system, + backend=backend, + backend_version=capability.aic_database_version, + tp_size=topology.tp, + pp_size=topology.pp, + attention_dp_size=topology.dp, + moe_tp_size=topology.moe_tp, + moe_ep_size=topology.moe_ep, + cp_size=topology.cp, + ) + for kv_dtype in capability.dtype.kv_cache_dtypes: + for policy in policies: + resolved = [ + capability.dtype.gemm_quant_mode, + capability.dtype.moe_quant_mode, + capability.dtype.fmha_by_kv_dtype[kv_dtype], + capability.dtype.comm_quant_mode, + kv_dtype, + *(str(getattr(topology, axis)) for axis in PARALLEL_AXES), + *(str(value) for value in backend_identity_columns(policy).values()), + ] + if deployment.match_identity() != resolved: + conflicts = "; ".join( + f"{name}: resolved={actual!r}, profile={expected!r}" + for name, actual, expected in zip( + identity_fields, resolved, deployment.match_identity(), strict=True + ) + if actual != expected + ) + raise ValueError( + f"FPM collection profile identity mismatch: {conflicts}; supply a profile matching " + "the collection configuration. The profile does not override serving dispatch." + ) diff --git a/python/aisimulate/collector/fpm_forward/runner.py b/python/aisimulate/collector/fpm_forward/runner.py index ee9eed8deb..f8964b8280 100644 --- a/python/aisimulate/collector/fpm_forward/runner.py +++ b/python/aisimulate/collector/fpm_forward/runner.py @@ -503,6 +503,7 @@ def prepare_attempt( cell_id: str, plan_sha256: str, attempt_id: str, + expected_backend_version: str | None = None, ) -> None: """Clear stale results and bind every Pod to this Collector attempt.""" @@ -525,6 +526,14 @@ def prepare_attempt( f"path = pathlib.Path('{FPM_RESULTS_DIR}') / sys.argv[2]; " "path.write_text(json.dumps(payload, sort_keys=True) + '\\n')" ) + version_args = [] + if expected_backend_version is not None: + script += ( + "\nactual = payload['runtime']['backend_version']\n" + "if actual != sys.argv[3]:\n" + " raise RuntimeError(f'FPM profile runtime mismatch: actual={actual!r}, expected={sys.argv[3]!r}')\n" + ) + version_args.append(expected_backend_version) for pod in pods: self._exec_checked( pod, @@ -546,7 +555,7 @@ def prepare_attempt( ) self._exec_checked( pod, - ["python3", "-c", script, payload, COLLECTOR_PROVENANCE_FILENAME], + ["python3", "-c", script, payload, COLLECTOR_PROVENANCE_FILENAME, *version_args], timeout=60, ) @@ -967,6 +976,16 @@ def _cell_generator_overrides( "2", ] ) + if getattr(plan, "fpm_profile", None) is not None: + deployment_profile = plan.deployment_profile(cell) + assert deployment_profile is not None + resources = deployment_profile.resources + # Native point generation sees the declared scheduler bounds before + # any cases are queued. Never discard runtime cases to fit a profile. + if "--max-num-batched-tokens" not in scheduler_args: + scheduler_args.extend(["--max-num-batched-tokens", str(resources.max_num_tokens)]) + if "--max-num-seqs" not in scheduler_args: + scheduler_args.extend(["--max-num-seqs", str(resources.max_batch_size)]) model_args = [] architecture = getattr(getattr(plan, "capability", None), "architecture", None) if architecture == "GlmMoeDsaForCausalLM": @@ -1645,11 +1664,17 @@ def _run_collection_impl( runtime_preflight, ], ) + profile_version = ( + {"expected_backend_version": plan.capability.aic_database_version} + if getattr(plan, "fpm_profile", None) is not None + else {} + ) resource.prepare_attempt( pods, cell_id=cell.cell_id, plan_sha256=plan.sha256, attempt_id=attempt_id, + **profile_version, ) phase_marks["stage_s"] = round(time.monotonic() - mark, 3) mark = time.monotonic() diff --git a/python/aisimulate/src/aiconfigurator_core/sdk/engine.py b/python/aisimulate/src/aiconfigurator_core/sdk/engine.py index 3b5609f992..e8114cb1ed 100644 --- a/python/aisimulate/src/aiconfigurator_core/sdk/engine.py +++ b/python/aisimulate/src/aiconfigurator_core/sdk/engine.py @@ -12,6 +12,10 @@ plain-data ``OpSpec`` wire form, and ships the whole thing across the boundary as a bincode-serialised ``EngineSpec``. +With direct FPM interpolation, an explicit ``FpmModelProfile`` supplies identity +and resources instead. Compilation emits whole-forward timing operations with +empty SOL lists and never constructs an analytical model. + Since the pyo3 op unification the ``Operation`` objects the model layer builds ARE engine ops (Rust pyclasses; ``operations/*.py`` keeps thin data-plane shells). Serialization is therefore Rust-to-Rust: @@ -45,6 +49,12 @@ import aiconfigurator_core from aiconfigurator_core.sdk.config_builders import apply_nextn, build_model_config +from aiconfigurator_core.sdk.fpm_profile import ( + FpmDeploymentProfile, + FpmModelProfile, + load_fpm_profile, + resolve_fpm_interpolation, +) from aiconfigurator_core.sdk.models import get_model from aiconfigurator_core.sdk.operations import FPMForwardOp from aiconfigurator_core.sdk.operations.base import Operation @@ -127,6 +137,7 @@ # - 18 (speculation migration): Generation attention gained verify_query_tokens # and FPM forward gained verify_width, both positional bincode fields. # TokenScale was appended to remap draft query widths before op lookup. +# - 19: FPM forward gained the SOL/direct interpolation selector. # Single owner: the Rust crate constant. Python re-exports it for # diagnostics/tests instead of declaring a twin to keep in sync. ENGINE_SPEC_SCHEMA_VERSION = aiconfigurator_core.engine_spec_schema_version() @@ -418,6 +429,9 @@ def compile_engine( kv_block_size: int | None = None, systems_path: str | None = None, forward_model: str | None = None, + fpm_profile: dict | str | FpmModelProfile | None = None, + fpm_interpolation: str = "auto", + cp_size: int = 1, database_mode: str | None = None, shared_layer: bool | None = None, transfer_policy: str | list[str] | None = None, @@ -430,7 +444,64 @@ def compile_engine( inside ``get_model``) to build the model, then walks ``encoder_ops`` (vision decomposed), ``context_ops`` and ``generation_ops`` into OpSpecs and returns the bytes produced by the Rust ``engine_spec_bincode_from_json`` pyfunction. + + ``fpm_profile`` supplies independent decoder metadata and rank-local + resource bounds. ``fpm_interpolation='direct'`` bypasses model construction; + ``'auto'`` retains SOL for registered architectures and selects direct for + unknown ones. Profile precision is authoritative on either route, with + conflicting caller overrides rejected. ``cp_size`` completes the profile + identity; profiles currently support only CP1/PP1 and no speculation. """ + if type(cp_size) is not int or cp_size != 1: + raise ValueError("cp_size must be the integer 1; this SDK entry point does not support context parallelism") + profile = load_fpm_profile(fpm_profile) if fpm_profile is not None else None + if (profile is not None or fpm_interpolation != "auto") and forward_model != "fpm": + raise ValueError("fpm_profile and fpm_interpolation require forward_model='fpm'") + interpolation = resolve_fpm_interpolation(profile, fpm_interpolation) + deployment = None + if profile is not None: + if nextn: + raise ValueError("FPM profiles support plain autoregressive decoder-only execution; nextn must be 0") + literal_version = _literal_backend_version(system, backend, backend_version, systems_path, None) + deployment = profile.select( + model=model_path, + system=system, + backend=backend, + backend_version=literal_version, + tp_size=tp_size, + pp_size=pp_size, + attention_dp_size=attention_dp_size, + moe_tp_size=moe_tp_size, + moe_ep_size=moe_ep_size, + cp_size=cp_size, + ) + deployment.validate_overrides( + gemm_quant_mode=gemm_quant_mode, + moe_quant_mode=moe_quant_mode, + kvcache_quant_mode=kvcache_quant_mode, + fmha_quant_mode=fmha_quant_mode, + comm_quant_mode=comm_quant_mode, + attention_backend=attention_backend, + ) + if interpolation == "direct": + spec_json = _direct_fpm_spec_json( + profile, + deployment, + kv_block_size=kv_block_size, + systems_path=systems_path, + database_mode=database_mode, + shared_layer=shared_layer, + transfer_policy=transfer_policy, + strict_provenance=strict_provenance, + ) + return bytes(aiconfigurator_core.engine_spec_bincode_from_json(spec_json)) + gemm_quant_mode = deployment.gemm_quant_mode + moe_quant_mode = deployment.moe_quant_mode + kvcache_quant_mode = deployment.kv_cache_dtype + fmha_quant_mode = deployment.fmha_quant_mode + comm_quant_mode = deployment.comm_quant_mode + attention_backend = None if deployment.attention_backend == "auto" else deployment.attention_backend + # `_build_model_config` resolves MoE parallelism defaults internally and # does not take a model_path (quant inference is done inside `get_model`). resolved_moe_tp = moe_tp_size if moe_tp_size is not None else 1 @@ -452,7 +523,13 @@ def compile_engine( # Apply MTP BEFORE get_model so the walked op lists carry the # (L+nextn)/L compute scale; accepted-token progress is applied above core. apply_nextn(model_config, nextn) + if deployment is not None: + model_config.moe_backend = None if deployment.moe_backend == "auto" else deployment.moe_backend model = get_model(model_path, model_config, backend) + if deployment is not None: + for op in (*model.context_ops, *model.generation_ops): + if not isinstance(op, FPMForwardOp) or list(op._match_identity) != deployment.match_identity(): + raise ValueError("registered SOL construction does not preserve the requested FPM profile identity") # Slot policy FIRST, tolerance second: resolve the requested version to a # literal (raising on unlisted versions / unpopulated aliases) before the @@ -486,9 +563,79 @@ def compile_engine( strict_provenance=strict_provenance, ) + if profile is not None: + spec = json.loads(spec_json) + spec["engine"]["forward_model"] = "fpm" + spec["engine"]["extra"].update(fpm_profile=profile.model_dump_json(), fpm_interpolation=interpolation) + spec_json = json.dumps(spec) + return bytes(aiconfigurator_core.engine_spec_bincode_from_json(spec_json)) +def _direct_fpm_spec_json( + profile: FpmModelProfile, + deployment: FpmDeploymentProfile, + *, + kv_block_size: int | None, + systems_path: str | None, + database_mode: str | None, + shared_layer: bool | None, + transfer_policy: str | list[str] | None, + strict_provenance: bool | None, +) -> str: + """Build whole-forward timing operations from metadata, without a graph.""" + if _database_mode_name(None, database_mode) != "SILICON": + raise ValueError("direct FPM interpolation requires database_mode='SILICON'; it has no analytical SOL model") + engine = { + "schema_version": ENGINE_CONFIG_SCHEMA_VERSION, + "model_name": profile.model, + "system_name": deployment.system, + "systems_path": systems_path, + "backend": deployment.backend, + "backend_version": deployment.backend_version, + "forward_model": "fpm", + "kv_block_size": kv_block_size, + "tp_size": deployment.tp, + "pp_size": deployment.pp, + "attention_dp_size": deployment.dp, + "moe_tp_size": deployment.moe_tp, + "moe_ep_size": deployment.moe_ep, + "cp_size": deployment.cp, + "weight_dtype": _rust_quant_to_dtype(deployment.gemm_quant_mode), + "moe_dtype": _rust_moe_quant_to_dtype(deployment.moe_quant_mode), + "activation_dtype": _rust_quant_to_dtype(deployment.fmha_quant_mode), + "kv_cache_dtype": _rust_quant_to_dtype(deployment.kv_cache_dtype), + "enable_shared_layer": shared_layer, + "strict_provenance": bool(strict_provenance), + "database_mode": "SILICON", + "transfer_policy": _transfer_policy_tokens(None, transfer_policy), + "extra": {"fpm_profile": profile.model_dump_json(), "fpm_interpolation": "direct"}, + } + + def phase_op(phase: str) -> dict: + return { + "FpmForward": { + "name": f"fpm_forward_{phase}", + "phase": phase, + "model_path": profile.model, + "match_identity": deployment.match_identity(), + "weight_bytes": deployment.resources.weights_bytes, + "verify_width": 1, + "sol_ops": [], + "interpolation": "direct", + } + } + + return json.dumps( + { + "schema_version": ENGINE_SPEC_SCHEMA_VERSION, + "engine": engine, + "context_ops": [phase_op("prefill")], + "generation_ops": [phase_op("decode")], + } + ) + + def build_ops_json(ops: Any) -> str: """Serialize an op list to the OpSpec JSON array the ad-hoc op-list evaluation FFI (``AicEngine.evaluate_ops_json``) consumes. diff --git a/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py b/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py new file mode 100644 index 0000000000..63d720584d --- /dev/null +++ b/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Explicit identity and resource bounds for decoder-only FPM simulation. + +Profiles describe every rank of a deployment without constructing operations. +Resource values are declarations supplied by the caller, not GPU measurements +or automatically inferred checkpoint metadata. Their provenance must say how +they were obtained; CUDA graph reservations are supplied separately at runtime. +""" + +from __future__ import annotations + +import json +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from aiconfigurator_core.sdk import common + +_PositiveInt = Annotated[int, Field(gt=0)] +_Bytes = Annotated[int, Field(ge=0, le=2**53)] +_Nonempty = Annotated[str, Field(min_length=1, pattern=r"\S")] +_MUTABLE_REFERENCES = {"main", "master", "head", "latest", "current", "unknown", "tbd", "unpinned"} + + +class _ProfileModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class FpmResourceProfile(_ProfileModel): + """Conservative rank-local bounds at a declared scheduler envelope. + + ``max_num_tokens`` and ``max_batch_size`` apply to each attention-DP rank, + not the summed worker iteration used to query FPM timings. Overheads must + exclude the separately configured CUDA graph reservation. + """ + + weights_bytes: _Bytes + activations_bytes: _Bytes + runtime_overhead_bytes: _Bytes + comm_overhead_bytes: _Bytes + kv_bytes_per_token: Annotated[int, Field(gt=0, le=2**53)] + cache_layout: Literal["linear"] + max_num_tokens: _PositiveInt + max_batch_size: _PositiveInt + provenance: _Nonempty + + @model_validator(mode="after") + def _exact_total(self) -> FpmResourceProfile: + if self.non_kv_bytes > 2**53: + raise ValueError("total non-KV resource bytes must not exceed 2**53") + return self + + @property + def non_kv_bytes(self) -> int: + return self.weights_bytes + self.activations_bytes + self.runtime_overhead_bytes + self.comm_overhead_bytes + + def validate_envelope(self, *, max_num_tokens: int, max_batch_size: int) -> None: + for name, requested in (("max_num_tokens", max_num_tokens), ("max_batch_size", max_batch_size)): + if isinstance(requested, bool) or not isinstance(requested, int) or requested <= 0: + raise ValueError(f"{name} must be a positive integer, got {requested!r}") + limit = getattr(self, name) + if requested > limit: + raise ValueError( + f"FPM resource envelope exceeded: requested rank-local {name}={requested}, " + f"profile bound={limit}; provide resource bounds for the larger scheduler envelope" + ) + + +class FpmDeploymentProfile(_ProfileModel): + """One exact hardware, runtime, topology, precision and resource identity.""" + + system: _Nonempty + backend: Literal["vllm"] + backend_version: _Nonempty + tp: _PositiveInt + pp: Literal[1] = 1 + dp: _PositiveInt + moe_tp: _PositiveInt + moe_ep: _PositiveInt + cp: Literal[1] = 1 + gemm_quant_mode: _Nonempty + moe_quant_mode: _Nonempty + fmha_quant_mode: _Nonempty + comm_quant_mode: _Nonempty + kv_cache_dtype: _Nonempty + moe_backend: _Nonempty = "auto" + attention_backend: _Nonempty = "auto" + enable_wideep: Literal[False] = False + enable_eplb: Literal[False] = False + resources: FpmResourceProfile + + @field_validator("pp", "cp", mode="before") + @classmethod + def _integer_dimensions(cls, value: Any) -> int: + if type(value) is not int: + raise ValueError("parallel dimensions must be integers, not booleans or floats") + return value + + @field_validator("enable_wideep", "enable_eplb", mode="before") + @classmethod + def _boolean_flags(cls, value: Any) -> bool: + if type(value) is not bool: + raise ValueError("backend feature flags must be booleans") + return value + + @field_validator("backend_version") + @classmethod + def _literal_version(cls, value: str) -> str: + if value.lower() in _MUTABLE_REFERENCES | {"previous", "next"} or any(c.isspace() for c in value): + raise ValueError("FPM profile backend_version must be a literal runtime version") + return value + + @model_validator(mode="after") + def _deployment_identity(self) -> FpmDeploymentProfile: + modes = { + "gemm_quant_mode": common.GEMMQuantMode, + "moe_quant_mode": common.MoEQuantMode, + "fmha_quant_mode": common.FMHAQuantMode, + "comm_quant_mode": common.CommQuantMode, + "kv_cache_dtype": common.KVCacheQuantMode, + } + for name, enum in modes.items(): + if getattr(self, name) not in enum.__members__: + raise ValueError(f"unknown FPM {name}: {getattr(self, name)!r}") + return self + + @property + def parallel_tuple(self) -> tuple[int, int, int, int, int, int]: + return self.tp, self.pp, self.dp, self.moe_tp, self.moe_ep, self.cp + + def match_identity(self) -> list[str]: + """Return the exact 15-column native FPM identity, with no aliasing.""" + return [ + self.gemm_quant_mode, + self.moe_quant_mode, + self.fmha_quant_mode, + self.comm_quant_mode, + self.kv_cache_dtype, + *(str(value) for value in self.parallel_tuple), + self.moe_backend, + self.attention_backend, + str(self.enable_wideep), + str(self.enable_eplb), + ] + + def quantization_kwargs(self) -> dict[str, str]: + return { + "gemm_quant_mode": self.gemm_quant_mode, + "moe_quant_mode": self.moe_quant_mode, + "fmha_quant_mode": self.fmha_quant_mode, + "comm_quant_mode": self.comm_quant_mode, + "kvcache_quant_mode": self.kv_cache_dtype, + "attention_backend": self.attention_backend, + } + + def validate_overrides(self, **overrides: str | None) -> None: + for name, expected in self.quantization_kwargs().items(): + supplied = overrides.get(name) + if supplied is not None and supplied != expected: + raise ValueError( + f"FPM profile identity conflict: {name}={supplied!r}, profile={expected!r}; " + "use matching measured precision and backend settings" + ) + + +class FpmModelProfile(_ProfileModel): + """Serializable FPM metadata independent of an analytical model class.""" + + schema_version: Literal[1] + model: _Nonempty + model_revision: _Nonempty + architecture: _Nonempty + context_length: _PositiveInt + num_experts: Annotated[int, Field(ge=0)] + provenance: _Nonempty + deployments: Annotated[list[FpmDeploymentProfile], Field(min_length=1)] + + @field_validator("schema_version", mode="before") + @classmethod + def _integer_schema(cls, value: Any) -> int: + if type(value) is not int: + raise ValueError("schema_version must be an integer") + return value + + @field_validator("model_revision") + @classmethod + def _pinned_revision(cls, value: str) -> str: + if value.strip().lower() in _MUTABLE_REFERENCES: + raise ValueError("provide a pinned model_revision, not a mutable or unknown placeholder") + return value + + @model_validator(mode="after") + def _unique_deployments(self) -> FpmModelProfile: + identities = set() + for deployment in self.deployments: + tp = deployment.dp == 1 and deployment.moe_ep == 1 and deployment.moe_tp == deployment.tp + dep = deployment.tp == 1 and deployment.moe_tp == 1 and deployment.moe_ep == deployment.dp + tep = deployment.dp == 1 and deployment.moe_tp == 1 and deployment.moe_ep == deployment.tp + dense_tp = deployment.dp == 1 and deployment.moe_ep == 1 and deployment.moe_tp in (1, deployment.tp) + if not (dense_tp if self.num_experts == 0 else tp or dep or tep): + raise ValueError( + "FPM profile requires dense TP or an MoE TP, DEP, or TEP topology with matching world sizes" + ) + identity = ( + deployment.system, + deployment.backend, + deployment.backend_version, + deployment.parallel_tuple, + ) + if identity in identities: + raise ValueError( + "duplicate FPM deployment identity; specify one precision/resource profile per topology" + ) + identities.add(identity) + if deployment.moe_ep > 1 and (self.num_experts == 0 or self.num_experts % deployment.moe_ep): + raise ValueError("num_experts must be positive and divisible by moe_ep for expert parallelism") + return self + + def select( + self, + *, + model: str, + system: str, + backend: str, + backend_version: str, + tp_size: int = 1, + pp_size: int = 1, + attention_dp_size: int = 1, + moe_tp_size: int | None = None, + moe_ep_size: int | None = None, + cp_size: int = 1, + ) -> FpmDeploymentProfile: + if model != self.model: + raise ValueError(f"FPM profile model identity mismatch: requested {model!r}, profile={self.model!r}") + shape = ( + tp_size, + pp_size, + attention_dp_size, + 1 if moe_tp_size is None else moe_tp_size, + 1 if moe_ep_size is None else moe_ep_size, + cp_size, + ) + if any(type(value) is not int or value <= 0 for value in shape): + raise ValueError("FPM deployment dimensions must be positive integers, not booleans") + for deployment in self.deployments: + if ( + deployment.system == system + and deployment.backend == backend + and deployment.backend_version == backend_version + and deployment.parallel_tuple == shape + ): + return deployment + raise ValueError( + f"no matching FPM deployment profile for {model!r}, {system}/{backend}/{backend_version}, " + f"(tp,pp,dp,moe_tp,moe_ep,cp)={shape}; provide explicit resources and precision for that deployment" + ) + + +def load_fpm_profile(value: dict[str, Any] | str | FpmModelProfile) -> FpmModelProfile: + if isinstance(value, FpmModelProfile): + value = value.model_dump(mode="json") + if isinstance(value, str): + value = json.loads(value) + return FpmModelProfile.model_validate(value) + + +def _quantization_from_engine_config(config_json: str) -> dict[str, str]: + """Restore exact profile modes after checking the native wire dtypes. + + EngineConfig stores dtypes rather than quantization modes: ``sq`` and + ``int8_wo`` both serialize as ``int8``. Both native timing and memory + bridges use the profile to recover the mode, but only when every supplied + wire dtype agrees with the selected deployment's canonical encoding. + """ + from aiconfigurator_core.sdk.rust_engine_step import _moe_quant_to_dtype, _quant_to_dtype + + config = json.loads(config_json) + profile = load_fpm_profile(config["extra"]["fpm_profile"]) + deployment = profile.select( + model=config["model_name"], + system=config["system_name"], + backend=config["backend"], + backend_version=config["backend_version"], + tp_size=config["tp_size"], + pp_size=config["pp_size"], + attention_dp_size=1 if config.get("attention_dp_size") is None else config["attention_dp_size"], + moe_tp_size=config.get("moe_tp_size"), + moe_ep_size=config.get("moe_ep_size"), + cp_size=1 if config.get("cp_size") is None else config["cp_size"], + ) + modes = deployment.quantization_kwargs() + for field, mode, encode in ( + ("weight_dtype", "gemm_quant_mode", _quant_to_dtype), + ("moe_dtype", "moe_quant_mode", _moe_quant_to_dtype), + ("activation_dtype", "fmha_quant_mode", _quant_to_dtype), + ("kv_cache_dtype", "kvcache_quant_mode", _quant_to_dtype), + ): + supplied = config.get(field) + expected = encode(modes[mode]) + if supplied is not None and supplied != expected: + raise ValueError( + f"FPM profile identity conflict: {field}={supplied!r}, " + f"profile {mode}={modes[mode]!r} requires wire dtype {expected!r}" + ) + modes.pop("attention_backend") + return modes + + +def resolve_fpm_interpolation(profile: FpmModelProfile | None, method: str = "auto") -> Literal["sol", "direct"]: + """Choose from explicit registry presence, never from a caught build error.""" + if method not in {"auto", "sol", "direct"}: + raise ValueError(f"unknown FPM interpolation {method!r}; choose auto, sol, or direct") + if method == "direct": + if profile is None: + raise ValueError("direct FPM interpolation requires an fpm_profile with identity and resource metadata") + return "direct" + if profile is None: + return "sol" + from aiconfigurator_core.sdk.models.base import _MODEL_REGISTRY + + family = common.ARCHITECTURE_TO_MODEL_FAMILY.get(profile.architecture, profile.architecture) + registered = family in _MODEL_REGISTRY + if method == "sol" and not registered: + raise ValueError( + f"SOL interpolation requires a registered analytical model class for {profile.architecture!r}; " + "choose fpm_interpolation='direct' to use the supplied profile" + ) + return "sol" if registered else "direct" diff --git a/python/aisimulate/src/aiconfigurator_core/sdk/memory.py b/python/aisimulate/src/aiconfigurator_core/sdk/memory.py index 28fe38377e..2550ee7306 100644 --- a/python/aisimulate/src/aiconfigurator_core/sdk/memory.py +++ b/python/aisimulate/src/aiconfigurator_core/sdk/memory.py @@ -9,7 +9,11 @@ math -- fraction + tolerance validation, the native AIC memory model, the naive fallback, and the tolerance margin -- lives in this module. -Two estimators, each built then asked to ``estimate()``: +Resource sources: + +- **FPM profile**: caller-declared rank-local resource bounds bypass model + construction and timing data. The same budget arithmetic below applies, + after checking the scheduler against the declared resource envelope. - **Native** (:class:`KVCacheEstimator`): ``from_request`` reuses AIC's full backend memory model (``BaseBackend._get_memory_usage`` plus the model's @@ -43,6 +47,7 @@ from aiconfigurator_core.sdk.backends.factory import get_backend from aiconfigurator_core.sdk.common import DefaultHFModels from aiconfigurator_core.sdk.config_builders import apply_nextn, build_model_config, validate_nextn +from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile, load_fpm_profile from aiconfigurator_core.sdk.models import get_model from aiconfigurator_core.sdk.utils import ( _download_hf_config, @@ -986,6 +991,8 @@ def estimate_kv_cache( naive_kv_reservation: float = _DEFAULT_NAIVE_KV_RESERVATION, allow_naive_fallback: bool = False, allow_hf_config_download: bool = False, + fpm_profile: dict | str | FpmModelProfile | None = None, + cp_size: int = 1, ) -> dict[str, Any]: """Compute the KV-cache memory estimate (raw + optional tolerance margin). @@ -1024,11 +1031,17 @@ def estimate_kv_cache( model build is unsupported (default off -> the error propagates). allow_hf_config_download: allow the naive fallback to download a ``config.json`` from HuggingFace when it is not local / pre-cached. + fpm_profile: explicit FPM identity and conservative rank-local resource + bounds. This route reads only hardware specifications, requires the + requested scheduler envelope to fit the profile, and never builds + a model or falls back to naive estimation. CUDA graph bytes remain + a separate reservation in addition to declared profile overheads. + cp_size: profile context-parallel identity; currently only CP1 is supported. Returns: A flat dict with ``total_gpu_capacity_bytes``, ``total_kv_size_bytes``, ``kv_size_per_token_bytes``, ``total_kv_size_tokens``, ``source`` - (``"native"`` | ``"naive_fallback"``), ``memory_breakdown`` (dict on the + (``"native"`` | ``"naive_fallback"`` | ``"profile"``), ``memory_breakdown`` (dict on the native path, ``None`` on the fallback), and ``tolerance_adjusted`` (a dict when ``tolerance_fraction`` is set, else ``None``). @@ -1037,6 +1050,8 @@ def estimate_kv_cache( tolerance, no KV budget, insufficient model metadata, or (with the fallback off) an unsupported model/backend. """ + if type(cp_size) is not int or cp_size != 1: + raise ValueError("cp_size must be the integer 1; this SDK entry point does not support context parallelism") _validate_memory_fraction(backend, memory_fraction_kind, memory_fraction_value) _validate_tolerance(tolerance_fraction) _validate_naive_reservation(naive_kv_reservation) @@ -1047,6 +1062,59 @@ def estimate_kv_cache( # Validate the compute-side MTP depth before any fallback path. validate_nextn(nextn) + if fpm_profile is not None: + if nextn: + raise ValueError("FPM profile resources support plain autoregressive execution; nextn must be 0") + profile = load_fpm_profile(fpm_profile) + if backend_version is None: + raise ValueError("FPM profile memory estimation requires the profile's literal backend_version") + deployment = profile.select( + model=model_path, + system=system, + backend=backend, + backend_version=backend_version, + tp_size=tp_size, + pp_size=pp_size, + attention_dp_size=attention_dp_size, + moe_tp_size=moe_tp_size, + moe_ep_size=moe_ep_size, + cp_size=cp_size, + ) + deployment.validate_overrides( + gemm_quant_mode=gemm_quant_mode, + moe_quant_mode=moe_quant_mode, + kvcache_quant_mode=kvcache_quant_mode, + fmha_quant_mode=fmha_quant_mode, + comm_quant_mode=comm_quant_mode, + ) + resources = deployment.resources + resources.validate_envelope(max_num_tokens=max_num_tokens, max_batch_size=max_batch_size) + if resources.non_kv_bytes + cuda_graph_reserved_bytes > _MAX_EXACT_BYTE_COUNT: + raise ValueError("total FPM non-KV bytes including CUDA graphs must not exceed 2**53") + capacity = gpu_memory_capacity_bytes_override + if capacity is None: + capacity = perf_database.load_system_spec(system, systems_paths=systems_path)["gpu"]["mem_capacity"] + estimate = KVCacheEstimator._estimate_from_breakdown( + { + "weights_bytes": resources.weights_bytes, + "activations_bytes": resources.activations_bytes, + "runtime_overhead_bytes": resources.runtime_overhead_bytes, + "comm_overhead_bytes": resources.comm_overhead_bytes, + "non_kv_bytes": resources.non_kv_bytes, + "kv_size_per_token_bytes": resources.kv_bytes_per_token, + "gpu_memory_capacity_bytes": capacity, + }, + is_of_free=is_of_free, + fraction=fraction, + gpu_memory_capacity_bytes_override=gpu_memory_capacity_bytes_override, + cuda_graph_reserved_bytes=cuda_graph_reserved_bytes, + ) + estimate["source"] = "profile" + estimate["resource_provenance"] = resources.provenance + _apply_tolerance(estimate, tolerance_fraction) + estimate.pop("tokens_from_kv_bytes", None) + return estimate + try: native = KVCacheEstimator.from_request( model_path, @@ -1135,6 +1203,8 @@ def estimate_num_gpu_blocks( allow_naive_fallback: bool = False, allow_hf_config_download: bool = False, diagnostics: dict[str, Any] | None = None, + fpm_profile: dict | str | FpmModelProfile | None = None, + cp_size: int = 1, ) -> int: """Convert the KV-cache token capacity to a scheduler block count. @@ -1170,13 +1240,13 @@ def estimate_num_gpu_blocks( system, backend, backend_version=backend_version, - max_num_tokens=int(max_num_tokens), - max_batch_size=int(max_batch_size), + max_num_tokens=max_num_tokens if fpm_profile is not None else int(max_num_tokens), + max_batch_size=max_batch_size if fpm_profile is not None else int(max_batch_size), memory_fraction_kind=memory_fraction_kind, memory_fraction_value=float(memory_fraction_value), - tp_size=int(tp_size), - pp_size=int(pp_size), - attention_dp_size=int(attention_dp_size), + tp_size=tp_size if fpm_profile is not None else int(tp_size), + pp_size=pp_size if fpm_profile is not None else int(pp_size), + attention_dp_size=attention_dp_size if fpm_profile is not None else int(attention_dp_size), moe_tp_size=moe_tp_size, moe_ep_size=moe_ep_size, gemm_quant_mode=gemm_quant_mode, @@ -1184,7 +1254,7 @@ def estimate_num_gpu_blocks( kvcache_quant_mode=kvcache_quant_mode, fmha_quant_mode=fmha_quant_mode, comm_quant_mode=comm_quant_mode, - nextn=int(nextn), + nextn=nextn if fpm_profile is not None else int(nextn), systems_path=systems_path, gpu_memory_capacity_bytes_override=gpu_memory_capacity_bytes_override, cuda_graph_reserved_bytes=cuda_graph_reserved_bytes, @@ -1192,6 +1262,8 @@ def estimate_num_gpu_blocks( naive_kv_reservation=float(naive_kv_reservation), allow_naive_fallback=allow_naive_fallback, allow_hf_config_download=allow_hf_config_download, + fpm_profile=fpm_profile, + cp_size=cp_size, ) adjusted = estimate.get("tolerance_adjusted") diff --git a/python/aisimulate/src/aisimulate/aic.py b/python/aisimulate/src/aisimulate/aic.py index 53b6fa6cda..0f351af1b2 100644 --- a/python/aisimulate/src/aisimulate/aic.py +++ b/python/aisimulate/src/aisimulate/aic.py @@ -88,6 +88,7 @@ def materialize_aic_num_gpu_blocks( comm_dtype=lowered.get("aic_comm_dtype"), systems_path=lowered.get("systems_path"), cuda_graph_reserved_bytes=lowered.get("cuda_graph_reserved_bytes", 0), + **({"fpm_profile": lowered["aic_fpm_profile"]} if lowered.get("aic_fpm_profile") is not None else {}), **({"diagnostics": memory_diagnostics} if memory_diagnostics is not None else {}), ) return lowered @@ -118,6 +119,7 @@ def estimate_num_gpu_blocks( systems_path: str | None = None, cuda_graph_reserved_bytes: int = 0, diagnostics: dict[str, Any] | None = None, + fpm_profile: dict[str, Any] | None = None, ) -> int: """Estimate per-rank KV blocks using the replay-wide AIC contract. @@ -174,6 +176,7 @@ def estimate_num_gpu_blocks( comm_quant_mode=_quant_mode_name("comm", comm_dtype), systems_path=systems_path, cuda_graph_reserved_bytes=cuda_graph_reserved_bytes, + **({"fpm_profile": fpm_profile} if fpm_profile is not None else {}), **({"diagnostics": diagnostics} if diagnostics is not None else {}), ) ) @@ -186,9 +189,30 @@ def estimate_kv_bytes_per_token( pp_size: int, moe_tp_size: int = 1, moe_ep_size: int = 1, + fpm_profile: dict[str, Any] | None = None, + system: str | None = None, + backend: str = "vllm", + backend_version: str | None = None, + attention_dp_size: int = 1, ) -> int: """Derive per-rank KV bytes/token from the resolved Hugging Face config.""" + if fpm_profile is not None: + from aiconfigurator_core.sdk.fpm_profile import load_fpm_profile + + deployment = load_fpm_profile(fpm_profile).select( + model=model_name, + system=system, + backend=backend, + backend_version=backend_version, + tp_size=tp_size, + pp_size=pp_size, + attention_dp_size=attention_dp_size, + moe_tp_size=moe_tp_size, + moe_ep_size=moe_ep_size, + ) + return deployment.resources.kv_bytes_per_token + from aiconfigurator_core.sdk.memory import NaiveKVCacheEstimator estimator = NaiveKVCacheEstimator.from_model_path( diff --git a/python/aisimulate/src/aisimulate/compiler.py b/python/aisimulate/src/aisimulate/compiler.py index 9c2e5935e8..1a22bb9b47 100644 --- a/python/aisimulate/src/aisimulate/compiler.py +++ b/python/aisimulate/src/aisimulate/compiler.py @@ -319,6 +319,14 @@ def _worker_performance_model_metadata( "moe_ep_size": parallel.moe_expert if sharded_moe else None, "nextn": None, "forward_model": worker.timing.forward_model, + **( + { + "fpm_profile": engine.fpm_profile.model_dump(mode="json"), + "fpm_interpolation": worker.timing.fpm_interpolation, + } + if engine.fpm_profile is not None + else {} + ), **({"systems_path": engine.systems_path} if engine.systems_path is not None else {}), }, } @@ -361,6 +369,11 @@ def _worker_engine_args( payload["aic_backend_version"] = engine.backend_version if engine.systems_path is not None: payload["systems_path"] = engine.systems_path + if engine.fpm_profile is not None: + payload["aic_fpm_profile"] = engine.fpm_profile.model_dump(mode="json") + payload["aic_fpm_interpolation"] = worker.timing.fpm_interpolation + elif worker.timing.fpm_interpolation != "auto": + payload["aic_fpm_interpolation"] = worker.timing.fpm_interpolation if parallel.pipeline != 1: payload["aic_pp_size"] = parallel.pipeline if parallel.moe_tensor * parallel.moe_expert > 1: @@ -373,6 +386,8 @@ def _worker_engine_args( payload["max_model_len"] = ( engine.context_length if isinstance(engine.context_length, int) + else engine.fpm_profile.context_length + if engine.fpm_profile is not None else resolve_model_context_length(engine.model) ) if capacity.type == "fixed": @@ -442,6 +457,17 @@ def _resolve_kv_bytes_per_token( pp_size=parallel.pipeline, moe_tp_size=parallel.moe_tensor, moe_ep_size=parallel.moe_expert, + **( + { + "fpm_profile": engine.fpm_profile.model_dump(mode="json"), + "system": worker.hardware or engine.hardware, + "backend": engine.backend, + "backend_version": engine.backend_version, + "attention_dp_size": parallel.attention_data, + } + if engine.fpm_profile is not None + else {} + ), ) diff --git a/python/aisimulate/src/aisimulate/config/engine.py b/python/aisimulate/src/aisimulate/config/engine.py index 6156f4dc82..fdf7785427 100644 --- a/python/aisimulate/src/aisimulate/config/engine.py +++ b/python/aisimulate/src/aisimulate/config/engine.py @@ -9,6 +9,8 @@ from pydantic import Field, field_validator, model_validator +from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile + from .common import Choices, IntegerRange, NumericRange, StrictModel, SystemsPath PositiveInt = Annotated[int, Field(strict=True, gt=0)] @@ -141,11 +143,14 @@ def _validate_g3(self): class TimingConfig(StrictModel): type: Literal["default", "fixed", "polynomial"] = "default" forward_model: Literal["op_level", "fpm"] = "op_level" + fpm_interpolation: Literal["auto", "sol", "direct"] = "auto" prefill_ms: float | None = Field(default=None, ge=0.0) decode_ms: float | None = Field(default=None, ge=0.0) @model_validator(mode="after") def _validate_timing(self) -> TimingConfig: + if self.fpm_interpolation != "auto" and (self.type != "default" or self.forward_model != "fpm"): + raise ValueError("fpm_interpolation requires default FPM timing") if self.type == "fixed": if self.prefill_ms is None or self.decode_ms is None: raise ValueError("fixed timing requires prefill_ms and decode_ms") @@ -210,6 +215,7 @@ class KvTransferConfig(StrictModel): class EnginePredictionConfig(StrictModel): mode: EngineMode = "aggregated" model: str + fpm_profile: FpmModelProfile | None = None hardware: str backend: Backend = "vllm" backend_version: str | None = None @@ -235,6 +241,7 @@ def _reject_auto_hardware(cls, value: str) -> str: @model_validator(mode="after") def _validate_roles(self) -> EnginePredictionConfig: + _validate_fpm_profile(self, {self.mode}, {self.backend}) _validate_systems_path_modes(self, {self.mode}) _validate_worker_hardware(modes={self.mode}, workers=self.workers) if self.mode == "afd": @@ -387,6 +394,7 @@ class EngineRecommendationConfig(StrictModel): default_factory=lambda: Choices[EngineMode](choices=["aggregated", "disaggregated"]) ) model: str + fpm_profile: FpmModelProfile | None = None hardware: str backend: Backend | Choices[Backend] = Field(default_factory=lambda: Choices[Backend](choices=["vllm", "sglang"])) backend_version: str | None = None @@ -406,6 +414,8 @@ def _validate_nonempty(cls, value: str) -> str: @model_validator(mode="after") def _validate_roles(self) -> EngineRecommendationConfig: modes = set(self.mode.choices) if isinstance(self.mode, Choices) else {self.mode} + backends = set(self.backend.choices) if isinstance(self.backend, Choices) else {self.backend} + _validate_fpm_profile(self, modes, backends) _validate_systems_path_modes(self, modes) _validate_worker_hardware(modes=modes, workers=self.workers) if "afd" in modes: @@ -426,6 +436,47 @@ def _validate_roles(self) -> EngineRecommendationConfig: return self +def _validate_fpm_profile(engine, modes: set[str], backends: set[str]) -> None: + profile = engine.fpm_profile + if profile is None: + for role in ("aggregated", "prefill", "decode"): + worker = getattr(engine.workers, role) + if worker is not None and worker.timing.fpm_interpolation == "direct": + raise ValueError("direct FPM interpolation requires engine.fpm_profile") + return + if engine.model != profile.model: + raise ValueError("engine.model must match engine.fpm_profile.model") + if engine.backend_version is None: + raise ValueError("engine.fpm_profile requires a literal engine.backend_version") + if backends != {"vllm"} or "afd" in modes or engine.workers.encoder is not None: + raise ValueError("FPM profiles support vLLM aggregated/disaggregated decoder workers without AFD or encoders") + if isinstance(engine.context_length, int) and engine.context_length > profile.context_length: + raise ValueError("engine.context_length exceeds the FPM profile context_length") + for role in ("aggregated", "prefill", "decode"): + worker = getattr(engine.workers, role) + if worker is None: + continue + if worker.timing.type != "default" or worker.timing.forward_model != "fpm": + raise ValueError("engine.fpm_profile requires default FPM timing for every worker") + if isinstance(worker, WorkerPredictionConfig): + parallel = worker.parallelism + deployment = profile.select( + model=engine.model, + system=worker.hardware or engine.hardware, + backend=engine.backend, + backend_version=engine.backend_version, + tp_size=parallel.tensor, + pp_size=parallel.pipeline, + attention_dp_size=parallel.attention_data, + moe_tp_size=parallel.moe_tensor, + moe_ep_size=parallel.moe_expert, + ) + deployment.resources.validate_envelope( + max_num_tokens=worker.scheduler.max_batched_tokens, + max_batch_size=worker.scheduler.max_sequences, + ) + + def _validate_systems_path_modes(engine, modes: set[str]) -> None: if engine.systems_path is None: return diff --git a/python/aisimulate/src/aisimulate/recommend.py b/python/aisimulate/src/aisimulate/recommend.py index ad7557cc91..6f53432ae7 100644 --- a/python/aisimulate/src/aisimulate/recommend.py +++ b/python/aisimulate/src/aisimulate/recommend.py @@ -117,8 +117,16 @@ def recommendation_to_sweeper( "hardware_sku": hardware, "gpu_budget": optimization.constraints.max_candidate_gpus, "min_gpu_budget": optimization.constraints.min_candidate_gpus, - "context_length": (resolve_model_context_length(model) if context == "max" else context), + "context_length": ( + config.engine.fpm_profile.context_length + if context == "max" and config.engine.fpm_profile is not None + else resolve_model_context_length(model) + if context == "max" + else context + ), } + if engine.get("fpm_profile") is not None: + search_space["fpm_profile"] = engine["fpm_profile"] for role in ("prefill", "decode"): if workers.get(role, {}).get("hardware") is not None: search_space[f"{role}_hardware_sku"] = workers[role]["hardware"] @@ -368,6 +376,8 @@ def _role_search_space( else: result[f"{legacy_role}_timing_model"] = None result[f"{legacy_role}_forward_model"] = timing.get("forward_model", "op_level") + if timing.get("fpm_interpolation", "auto") != "auto": + result[f"{legacy_role}_fpm_interpolation"] = timing["fpm_interpolation"] result[f"{legacy_role}_startup_time"] = raw.get("startup_seconds", 0) # Remove empty internal maps so legacy serialization remains concise. if not result["engine_float_ranges"]: @@ -711,6 +721,8 @@ def _candidate_prediction( } if sample.get("systems_path") is not None: engine["systems_path"] = sample["systems_path"] + if sample.get("fpm_profile") is not None: + engine["fpm_profile"] = deepcopy(sample["fpm_profile"]) raw_engine = source.engine.model_dump(mode="python", exclude_none=True) if deployment.encoder is not None: from .config.epd import encoder_prediction_fields @@ -762,6 +774,8 @@ def _candidate_prediction( timing = deepcopy(timing_model) else: timing = {"type": "default", "forward_model": sample.get(f"{role}_forward_model") or "op_level"} + if sample.get("fpm_profile") is not None or sample.get(f"{role}_fpm_interpolation", "auto") != "auto": + timing["fpm_interpolation"] = sample.get(f"{role}_fpm_interpolation", "auto") kv_cache = { "block_size": block_size, "prefix_caching": sample[f"{role}_enable_prefix_caching"], diff --git a/python/aisimulate/src/aisimulate/runner.py b/python/aisimulate/src/aisimulate/runner.py index 76ac7214b9..9bbc5a9832 100644 --- a/python/aisimulate/src/aisimulate/runner.py +++ b/python/aisimulate/src/aisimulate/runner.py @@ -88,6 +88,8 @@ "comm_dtype": ("comm_dtype", "aic_comm_dtype"), "systems_path": ("systems_path",), "forward_model": ("forward_model", "aic_forward_model"), + "fpm_profile": ("fpm_profile", "aic_fpm_profile"), + "fpm_interpolation": ("fpm_interpolation", "aic_fpm_interpolation"), } _AIC_FORWARD_MODELS = frozenset({"op_level", "fpm"}) @@ -1261,12 +1263,18 @@ def _materialize_engine_role( value = rank.pop(configured[0]) if target in {"pp", "moe_tp_size", "moe_ep_size"}: value = _positive_int(value, f"engine provider {role} {target}") + elif target == "fpm_profile": + from aiconfigurator_core.sdk.fpm_profile import load_fpm_profile + + value = load_fpm_profile(value).model_dump(mode="json") elif not isinstance(value, str) or not value: raise ValueError(f"engine provider {role} {target} must be a string") if target == "forward_model" and value not in _AIC_FORWARD_MODELS: raise ValueError( f"engine provider {role} forward_model must be one of {sorted(_AIC_FORWARD_MODELS)}, got {value!r}" ) + if target == "fpm_interpolation" and value not in {"auto", "sol", "direct"}: + raise ValueError(f"engine provider {role} fpm_interpolation must be auto, sol, or direct") aic_timing_overrides[target] = value timing_model = rank.get("timing_model") diff --git a/python/aisimulate/src/aisimulate/support/cli.py b/python/aisimulate/src/aisimulate/support/cli.py index e5a06ce1dc..08dc3c2c6b 100644 --- a/python/aisimulate/src/aisimulate/support/cli.py +++ b/python/aisimulate/src/aisimulate/support/cli.py @@ -17,6 +17,8 @@ import yaml from pydantic import ValidationError +from aisimulate.config.common import load_yaml + from .fpm import run_fpm from .plan import create_plan, request_id from .schema import FPMDeployment, SearchProfile, SloSpec, SupportIdentity, SupportRequest, WorkloadSpec @@ -57,7 +59,11 @@ def add_support_parser(subparsers: Any) -> None: init.add_argument("--tokenizer-revision") init.add_argument("--chat-template-revision") init.add_argument("--aisimulate-revision", help="Optional pinned AISimulate source revision.") - init.add_argument("--tensor-parallel", type=int, help="GPUs per pure-TP worker (default: 1).") + init.add_argument("--tensor-parallel", type=int, help="Attention tensor-parallel size (default: 1).") + init.add_argument("--attention-data-parallel", type=int, help="Attention data-parallel size (default: 1).") + init.add_argument("--moe-tensor-parallel", type=int, help="Expert tensor-parallel size (default: TP for MoE).") + init.add_argument("--moe-expert-parallel", type=int, help="Expert-parallel size (default: 1).") + init.add_argument("--fpm-profile", help="JSON/YAML identity and resource profile for class-independent FPM.") init.add_argument("--input-tokens", type=int, help="Input tokens per request (default: 1024).") init.add_argument("--output-tokens", type=int, help="Output tokens per request (default: 128).") init.add_argument("--concurrency", type=int, help="Concurrent requests (default: 1).") @@ -121,6 +127,7 @@ def _request_from_args(args: argparse.Namespace) -> SupportRequest: "identity": identity, "workload": {**_values(args, WorkloadSpec), "slo": _values(args, SloSpec)}, "search": _values(args, SearchProfile), + **({"fpm_profile": load_yaml(args.fpm_profile)} if getattr(args, "fpm_profile", None) else {}), } ) @@ -196,6 +203,9 @@ def _print(value: dict[str, Any], output_format: str = "table") -> None: "max_candidates": ("Maximum candidates (1 or 2)", int), "objective": ("Recommendation objective", str), "seed": ("Recommendation search seed", int), + "attention_data_parallel": ("Attention data-parallel size", int), + "moe_tensor_parallel": ("Expert tensor-parallel size", int), + "moe_expert_parallel": ("Expert-parallel size", int), } @@ -266,12 +276,12 @@ def _init(args: argparse.Namespace) -> int: f"Scope: FPM simulation for {request.identity.model} ({request.identity.model_kind}), " f"vLLM {request.identity.framework_version}, " f"{request.identity.gpu_count} {request.identity.gpu} GPU(s) on {request.identity.node_count} node(s); " - f"TP{request.search.tensor_parallel}, {workload.input_tokens}/{workload.output_tokens} tokens, " + f"{request.parallel_preset} {request.parallelism()}, {workload.input_tokens}/{workload.output_tokens} tokens, " f"concurrency {workload.concurrency}, {workload.request_count} requests, " f"up to {request.search.max_candidates} candidate(s)." ) print( - "Request saved. Model integration, runtime compatibility and FPM data are unchecked; " + "Request saved. Model metadata/resources, runtime compatibility and FPM data are unchecked; " "accuracy is not assessed. Setup has not launched GPU work." ) _print( @@ -294,7 +304,8 @@ def _plan(args: argparse.Namespace) -> int: "candidate_count": plan["search"]["candidate_count"], "plan": str(root / "support-plan.json"), "commands": plan["outputs"]["commands"], - "prerequisites": "unchecked", + "prerequisites": "runtime_and_fpm_data_unchecked" if request.fpm_profile is not None else "unchecked", + **({"resources": "estimated_from_declared_profile"} if request.fpm_profile is not None else {}), "accuracy": "not assessed", "next": shlex.join( [ diff --git a/python/aisimulate/src/aisimulate/support/fpm.py b/python/aisimulate/src/aisimulate/support/fpm.py index b13e4cd3ef..e8ca547b92 100644 --- a/python/aisimulate/src/aisimulate/support/fpm.py +++ b/python/aisimulate/src/aisimulate/support/fpm.py @@ -30,6 +30,17 @@ def fpm_cli_args( checkpoint = Path(checkpoint_dir).expanduser().resolve() if checkpoint_dir else root / "fpm-checkpoint" if checkpoint != root / "fpm-checkpoint" and root / "fpm-checkpoint" not in checkpoint.parents: raise ValueError("checkpoint_dir must stay within the plan's fpm-checkpoint directory") + profile = request.profile_deployment() + max_prefill_tokens = max(2, request.workload.input_tokens * request.workload.concurrency) + max_prefill_batch = request.workload.concurrency + if profile is not None: + scheduler = request.scheduler_limits() + max_prefill_batch = scheduler["max_sequences"] + max_prefill_tokens = min( + max(2, request.workload.input_tokens * max_prefill_batch), scheduler["max_batched_tokens"] + ) + if max_prefill_tokens < 2: + raise ValueError("FPM collection requires a rank-local token limit of at least 2 in the resource profile") command = [ "python3", "-m", @@ -41,15 +52,15 @@ def fpm_cli_args( "--gpu", request.identity.gpu, "--fpm-max-gpus", - str(request.search.tensor_parallel), + str(request.worker_gpus), "--fpm-gpu-counts", - str(request.search.tensor_parallel), + str(request.worker_gpus), "--fpm-parallel-presets", - "pure_tp" if request.identity.model_kind == "moe" else "tp", + request.parallel_preset, "--fpm-max-prefill-isl", - str(max(2, request.workload.input_tokens * request.workload.concurrency)), + str(max_prefill_tokens), "--fpm-max-prefill-batch-size", - str(request.workload.concurrency), + str(max_prefill_batch), "--checkpoint-dir", str(checkpoint), "--fpm-artifact-root", @@ -57,6 +68,23 @@ def fpm_cli_args( "--fpm-database-root", str(root / "systems/data"), ] + if profile is not None: + command.extend( + ( + "--model-architecture", + request.fpm_profile.architecture, + "--fpm-weight-quantizations", + profile.gemm_quant_mode, + "--fpm-kv-cache-dtypes", + profile.kv_cache_dtype, + "--fpm-attention-backend", + profile.attention_backend, + "--fpm-moe-backend", + profile.moe_backend, + "--fpm-model-profile", + str(root / "fpm-model-profile.json"), + ) + ) if request.identity.sm is not None: command.extend(("--sm", str(request.identity.sm))) if deployment is not None: diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index 8f436107c8..3ef8be650b 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -33,20 +33,13 @@ def request_id(request: SupportRequest) -> str: def _parallelism(request: SupportRequest, replicas: int) -> dict[str, int]: - return { - "replicas": replicas, - "tensor": request.search.tensor_parallel, - "pipeline": 1, - "attention_data": 1, - "moe_tensor": request.search.tensor_parallel if request.identity.model_kind == "moe" else 1, - "moe_expert": 1, - } + return request.parallelism(replicas) def _configs( request: SupportRequest, root: Path ) -> tuple[dict[str, Any], dict[str, dict[str, Any]], list[dict[str, int]]]: - max_replicas = request.identity.node_count * (request.identity.gpus_per_node // request.search.tensor_parallel) + max_replicas = request.identity.node_count * (request.identity.gpus_per_node // request.worker_gpus) replicas = list(dict.fromkeys((1, max_replicas)))[: request.search.max_candidates] presets = [_parallelism(request, count) for count in replicas] workload = request.workload @@ -73,9 +66,12 @@ def _configs( } # Recommendation otherwise expands scheduler defaults into extra domains. worker = { - "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + "scheduler": request.scheduler_limits(), "timing": {"type": "default", "forward_model": "fpm"}, } + if request.fpm_profile is not None: + engine["fpm_profile"] = request.fpm_profile.model_dump(mode="json") + worker["timing"]["fpm_interpolation"] = "direct" prediction = { **common, "engine": {**engine, "workers": {"aggregated": {**worker, "parallelism": presets[0]}}}, @@ -155,7 +151,7 @@ def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any] "candidate_count": len(presets), "max_candidates": request.search.max_candidates, "candidates": [ - {"parallelism": preset, "total_gpus": preset["replicas"] * preset["tensor"]} for preset in presets + {"parallelism": preset, "total_gpus": preset["replicas"] * request.worker_gpus} for preset in presets ], "detail": ( "A single selected worker is planned; recommendation does not compare alternative configurations." @@ -168,7 +164,7 @@ def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any] }, "fpm": { "status": "planned", - "worker_gpus": request.search.tensor_parallel, + "worker_gpus": request.worker_gpus, "plan_command": commands["fpm_plan_local"], "sampling": ( "Prefill is bounded by max(2, input_tokens * concurrency) and concurrency. Decode sampling and " @@ -181,10 +177,10 @@ def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any] "id": "model_integration", "status": "not_checked", "detail": ( - "Verify pinned model metadata, memory/cache accounting, and chosen parallelism. This revision's " - "FPM predict/recommend uses registered analytical classes and SOL transfer; follow " - "python/aisimulate/docs/add_a_new_model.md for that route. Class-independent FPM with direct " - "timing interpolation is planned separately. Per-operation silicon data is not required." + "Verify pinned model metadata, memory/cache accounting, and chosen parallelism. Without an " + "FPM profile, predict/recommend uses a registered analytical class and SOL transfer; follow " + "python/aisimulate/docs/add_a_new_model.md for that route. Supply an FPM identity/resource " + "profile to use direct interpolation without a class. Per-operation silicon data is not required." ), }, { @@ -217,12 +213,70 @@ def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any] "systems_root": str(root / "systems"), }, } + if request.fpm_profile is not None: + from aiconfigurator_core.sdk.memory import estimate_kv_cache + + deployment = request.profile_deployment() + scheduler = prediction["engine"]["workers"]["aggregated"]["scheduler"] + estimate = estimate_kv_cache( + request.identity.model, + request.identity.gpu, + request.identity.framework, + backend_version=request.identity.framework_version, + max_num_tokens=scheduler["max_batched_tokens"], + max_batch_size=scheduler["max_sequences"], + memory_fraction_kind="of_total", + memory_fraction_value=0.9, + tp_size=deployment.tp, + pp_size=deployment.pp, + attention_dp_size=deployment.dp, + moe_tp_size=deployment.moe_tp, + moe_ep_size=deployment.moe_ep, + fpm_profile=request.fpm_profile, + ) + if estimate["total_kv_size_tokens"] <= request.search.context_length: + raise ValueError("declared FPM resources leave insufficient rank-local KV capacity for the pilot context") + plan["resources"] = estimate + plan["search"]["baseline_rule"] = ( + "Selected profile deployment; declared topology and rank-local resource bounds pass CPU admission. " + "Serving-runtime compatibility and silicon accuracy remain unchecked." + ) + plan["prerequisites"][0].update( + status="declared_profile_validated", + detail=( + "The pinned identity, declared deployment, scheduler envelope and resource budget validate without " + "an analytical model class. Verify the profile's resource provenance against the serving runtime. " + "Generated configs use direct interpolation; matching timing coverage is still required." + ), + ) + plan["prerequisites"][1]["detail"] = ( + "Prepare and verify the declared model/tokenizer revisions, visible GPUs, model access, and the " + "packaged Dynamo/Kubernetes/Generator collector runtime. This plan does not download a pinned " + "checkpoint or inspect the running runtime. During collection execution, the collector checks " + "the observed Pod vLLM version against the profile's literal backend version before benchmarking." + ) + plan["fpm"]["sampling"] = ( + "Prefill sampling is capped by the pilot input length and the generated rank-local scheduler limits. " + "These bounds do not establish complete timing coverage of worker-wide FPM queries, including DEP. " + "Decode uses the collector's existing sampling profile within the declared resource envelope. " + "The synthetic request count does not bound timing samples. Smoke and limited runs do not publish " + "formal FPM data." + ) + plan["prerequisites"][2]["detail"] = ( + "Collect matching prefill/decode timings into the local systems tree before prediction or recommendation. " + "The profile resource estimate does not establish timing coverage or measured accuracy." + ) + plan["outputs"]["fpm_model_profile"] = str(root / "fpm-model-profile.json") yaml_documents = { Path("request.yaml"): request.model_dump(mode="json", exclude_none=True), Path("predict/pilot.yaml"): prediction, **{Path(f"recommend/{name}.yaml"): config for name, config in recommendations.items()}, } documents = {path: yaml.safe_dump(data, sort_keys=False).encode() for path, data in yaml_documents.items()} + if request.fpm_profile is not None: + documents[Path("fpm-model-profile.json")] = ( + json.dumps(request.fpm_profile.model_dump(mode="json"), indent=2, sort_keys=True) + "\n" + ).encode() documents[Path("systems") / source_spec.name] = source_spec.read_bytes() documents[Path("commands.json")] = (json.dumps(commands, indent=2, sort_keys=True) + "\n").encode() documents[Path("support-plan.json")] = (json.dumps(plan, indent=2, sort_keys=True) + "\n").encode() @@ -251,6 +305,7 @@ def _check_paths(root: Path) -> None: for relative in ( "request.yaml", "support-plan.json", + "fpm-model-profile.json", "commands.json", "predict", "predict/pilot.yaml", @@ -290,6 +345,15 @@ def check_plan(request: SupportRequest, root: Path) -> None: raise ValueError(f"saved request in {root} was modified or cannot be read") from exc if saved != request: raise ValueError(f"saved request identity in {root} differs from the requested plan") + if request.fpm_profile is not None and (root / "fpm-model-profile.json").exists(): + from aiconfigurator_core.sdk.fpm_profile import load_fpm_profile + + try: + profile = load_fpm_profile((root / "fpm-model-profile.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"saved FPM model profile in {root} is missing or invalid") from exc + if profile != request.fpm_profile: + raise ValueError(f"saved FPM model profile in {root} differs from the requested identity") def create_plan(request: SupportRequest, output_dir: str | Path, *, overwrite: bool = False) -> dict[str, Any]: diff --git a/python/aisimulate/src/aisimulate/support/schema.py b/python/aisimulate/src/aisimulate/support/schema.py index 603f6356c1..4c6ed37a78 100644 --- a/python/aisimulate/src/aisimulate/support/schema.py +++ b/python/aisimulate/src/aisimulate/support/schema.py @@ -12,6 +12,7 @@ from packaging.version import Version from pydantic import Field, field_validator, model_validator +from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile from aisimulate.config.common import PositiveFiniteFloat, PositiveStrictInt, StrictModel, load_yaml @@ -96,6 +97,9 @@ def _request_count(self) -> WorkloadSpec: class SearchProfile(StrictModel): tensor_parallel: PositiveStrictInt = 1 + attention_data_parallel: PositiveStrictInt | None = None + moe_tensor_parallel: PositiveStrictInt | None = None + moe_expert_parallel: PositiveStrictInt | None = None context_length: PositiveStrictInt = 16384 max_candidates: int = Field(default=1, strict=True, ge=1, le=2) objective: Literal[ @@ -152,13 +156,84 @@ class SupportRequest(StrictModel): identity: SupportIdentity workload: WorkloadSpec = Field(default_factory=WorkloadSpec) search: SearchProfile = Field(default_factory=SearchProfile) + fpm_profile: FpmModelProfile | None = None + + def parallelism(self, replicas: int = 1) -> dict[str, int]: + search = self.search + return { + "replicas": replicas, + "tensor": search.tensor_parallel, + "pipeline": 1, + "attention_data": search.attention_data_parallel or 1, + "moe_tensor": search.moe_tensor_parallel + or (search.tensor_parallel if self.identity.model_kind == "moe" else 1), + "moe_expert": search.moe_expert_parallel or 1, + } + + @property + def worker_gpus(self) -> int: + parallel = self.parallelism() + return parallel["tensor"] * parallel["attention_data"] + + @property + def parallel_preset(self) -> str: + parallel = self.parallelism() + if self.identity.model_kind == "dense": + return "tp" + if parallel["attention_data"] > 1: + return "dep" + return "tep" if parallel["moe_expert"] > 1 else "pure_tp" + + def profile_deployment(self): + if self.fpm_profile is None: + return None + parallel = self.parallelism() + return self.fpm_profile.select( + model=self.identity.model, + system=self.identity.gpu, + backend=self.identity.framework, + backend_version=self.identity.framework_version, + tp_size=parallel["tensor"], + pp_size=parallel["pipeline"], + attention_dp_size=parallel["attention_data"], + moe_tp_size=parallel["moe_tensor"], + moe_ep_size=parallel["moe_expert"], + ) + + def scheduler_limits(self) -> dict[str, int]: + """Rank-local limits shared by generated configs and collection bounds.""" + deployment = self.profile_deployment() + if deployment is None: + return {"max_batched_tokens": 8192, "max_sequences": 256} + return { + "max_batched_tokens": min(8192, deployment.resources.max_num_tokens), + "max_sequences": min(self.workload.concurrency, deployment.resources.max_batch_size), + } @model_validator(mode="after") def _shape(self) -> SupportRequest: - if self.search.tensor_parallel > self.identity.gpus_per_node: - raise ValueError("search.tensor_parallel must fit within gpus_per_node") + parallel = self.parallelism() + tp, dp, mtp, ep = (parallel[name] for name in ("tensor", "attention_data", "moe_tensor", "moe_expert")) + if self.identity.model_kind == "dense": + valid = dp == mtp == ep == 1 + else: + valid = ( + (dp == 1 and mtp == tp and ep == 1) or (tp == mtp == 1 and ep == dp) or (dp == mtp == 1 and ep == tp) + ) + if not valid: + raise ValueError("onboarding requires a complete TP, DEP, or TEP topology; set the attention and MoE sizes") + if self.worker_gpus > self.identity.gpus_per_node: + raise ValueError("search.tensor_parallel * attention_data_parallel must fit within gpus_per_node") if self.workload.input_tokens + self.workload.output_tokens > self.search.context_length: raise ValueError("search.context_length must cover the input and output tokens") + if self.fpm_profile is not None: + self.profile_deployment() + if self.identity.model_revision != self.fpm_profile.model_revision: + raise ValueError("identity.model_revision must match fpm_profile.model_revision") + if (self.fpm_profile.num_experts > 0) != (self.identity.model_kind == "moe"): + raise ValueError("identity.model_kind must match fpm_profile.num_experts") + if self.search.context_length > self.fpm_profile.context_length: + raise ValueError("search.context_length exceeds fpm_profile.context_length") return self @classmethod diff --git a/python/aisimulate/src/aisimulate/sweeper/config.py b/python/aisimulate/src/aisimulate/sweeper/config.py index b92682614b..7cfbd7bcf4 100644 --- a/python/aisimulate/src/aisimulate/sweeper/config.py +++ b/python/aisimulate/src/aisimulate/sweeper/config.py @@ -485,6 +485,7 @@ class SearchSpace(BaseModel): model_name: str # HF id or private model name hardware_sku: str # e.g. "h200_sxm" systems_path: SystemsPath | None = None + fpm_profile: dict[str, Any] | None = None prefill_hardware_sku: str | None = Field(default=None, min_length=1) decode_hardware_sku: str | None = Field(default=None, min_length=1) gpu_budget: int = 32 # max GPUs per candidate @@ -521,6 +522,7 @@ class SearchSpace(BaseModel): prefill_num_gpu_blocks: int | None = None prefill_timing_model: dict[str, Any] | None = None prefill_forward_model: str = "op_level" # AIC forward-pass model: op_level | fpm + prefill_fpm_interpolation: Literal["auto", "sol", "direct"] = "auto" prefill_startup_time: float | None = None # decode engine (disagg branch): scheduler batching capacity @@ -535,6 +537,7 @@ class SearchSpace(BaseModel): decode_num_gpu_blocks: int | None = None decode_timing_model: dict[str, Any] | None = None decode_forward_model: str = "op_level" # AIC forward-pass model: op_level | fpm + decode_fpm_interpolation: Literal["auto", "sol", "direct"] = "auto" decode_startup_time: float | None = None # agg engine (agg branch): scheduler batching capacity @@ -549,6 +552,7 @@ class SearchSpace(BaseModel): agg_num_gpu_blocks: int | None = None agg_timing_model: dict[str, Any] | None = None agg_forward_model: str = "op_level" # AIC forward-pass model: op_level | fpm + agg_fpm_interpolation: Literal["auto", "sol", "direct"] = "auto" agg_startup_time: float | None = None kv_transfer_bytes_per_token: int | str | None = None kv_transfer_bandwidth: float | None = None diff --git a/python/aisimulate/src/aisimulate/sweeper/deploy.py b/python/aisimulate/src/aisimulate/sweeper/deploy.py index 30edbfbe1a..7ac6a8a061 100644 --- a/python/aisimulate/src/aisimulate/sweeper/deploy.py +++ b/python/aisimulate/src/aisimulate/sweeper/deploy.py @@ -46,6 +46,9 @@ def _performance_model_metadata(sample: dict[str, Any], role: str, *, backend_ve } if sample.get("systems_path") is not None: config["systems_path"] = sample["systems_path"] + if sample.get("fpm_profile") is not None: + config["fpm_profile"] = sample["fpm_profile"] + config["fpm_interpolation"] = sample.get(f"{role}_fpm_interpolation", "auto") return {"provider": "aic", "config": config} @@ -85,6 +88,11 @@ def _engine_args_payload(sample: dict[str, Any], role: str, *, backend_version: } if sample.get("systems_path") is not None: payload["systems_path"] = sample["systems_path"] + if sample.get("fpm_profile") is not None: + payload["aic_fpm_profile"] = sample["fpm_profile"] + method = sample.get(f"{role}_fpm_interpolation", "auto") + if sample.get("fpm_profile") is not None or method != "auto": + payload["aic_fpm_interpolation"] = method if backend == "vllm" and sample.get("context_length") is not None: payload["max_model_len"] = int(sample["context_length"]) if moe_tp * moe_ep > 1: @@ -127,6 +135,7 @@ def _engine_args_payload(sample: dict[str, Any], role: str, *, backend_version: pp_size=int(sample[f"{prefix}pp"]), moe_tp_size=moe_tp, moe_ep_size=moe_ep, + **_profile_kv_args(sample, role, backend_version), ) if configured_bytes == "auto" else int(configured_bytes) @@ -140,6 +149,7 @@ def _engine_args_payload(sample: dict[str, Any], role: str, *, backend_version: pp_size=int(sample["prefill_pp"]), moe_tp_size=int(sample["prefill_moe_tp"]), moe_ep_size=int(sample["prefill_moe_ep"]), + **_profile_kv_args(sample, "prefill", backend_version), ) if transfer_geometry == "auto" else int(transfer_geometry) @@ -154,6 +164,19 @@ def _engine_args_payload(sample: dict[str, Any], role: str, *, backend_version: return payload +def _profile_kv_args(sample: dict[str, Any], role: str, backend_version: str) -> dict[str, Any]: + if sample.get("fpm_profile") is None: + return {} + prefix = _role_prefix(role) + return { + "fpm_profile": sample["fpm_profile"], + "system": _role_hardware_sku(sample, role), + "backend": sample["backend"], + "backend_version": backend_version, + "attention_dp_size": int(sample[f"{prefix}attention_dp"]), + } + + def build_backend_deployment( sample: dict[str, Any], *, diff --git a/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py b/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py index 4154936bb1..a51c56fc46 100644 --- a/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py +++ b/python/aisimulate/src/aisimulate/sweeper/kv_estimate.py @@ -20,11 +20,16 @@ (:func:`get_latest_database_version` resolves the version). SKUs without a perf DB raise :class:`NoPerfDatabase`; the naive fallback is intentionally disabled because it mis-models MoE expert sharding. + +An explicit FPM profile instead supplies per-deployment resource bounds. That +memory path requires a hardware specification and literal profile version, +but no timing database or analytical model. """ from __future__ import annotations from collections.abc import Iterable +from typing import Any from aiconfigurator_core.sdk.memory import estimate_kv_cache from aiconfigurator_core.sdk.perf_database import get_latest_database_version @@ -69,6 +74,7 @@ def estimate_kv_tokens( memory_fraction: float = DEFAULT_MEMORY_FRACTION, nextn: int = 0, systems_path: str | None = None, + fpm_profile: dict[str, Any] | None = None, ) -> int | None: """Per-rank KV-cache capacity (in tokens) for ``shape``, or ``None`` when the shape leaves no KV budget (weights + activations already fill VRAM -> OOM). @@ -93,6 +99,7 @@ def estimate_kv_tokens( nextn=nextn, systems_path=systems_path, allow_naive_fallback=False, + **({"fpm_profile": fpm_profile} if fpm_profile is not None else {}), ) except ValueError as exc: msg = str(exc) @@ -120,6 +127,7 @@ def feasible_shape_tokens( max_batch_size: int = DEFAULT_MAX_BATCH_SIZE, memory_fraction: float = DEFAULT_MEMORY_FRACTION, systems_path: str | None = None, + fpm_profile: dict[str, Any] | None = None, ) -> dict[ParallelShape, int]: """Map each *feasible* shape to its KV-cache token capacity. @@ -141,6 +149,7 @@ def feasible_shape_tokens( max_batch_size=max_batch_size, memory_fraction=memory_fraction, systems_path=systems_path, + **({"fpm_profile": fpm_profile} if fpm_profile is not None else {}), ) if tokens is not None and tokens > max_seq_len: feasible[shape] = tokens diff --git a/python/aisimulate/src/aisimulate/sweeper/model_hw.py b/python/aisimulate/src/aisimulate/sweeper/model_hw.py index 064aa43c04..2542488bb0 100644 --- a/python/aisimulate/src/aisimulate/sweeper/model_hw.py +++ b/python/aisimulate/src/aisimulate/sweeper/model_hw.py @@ -15,11 +15,16 @@ workload's ``max_seq_len`` (:mod:`aisimulate.sweeper.kv_estimate`). That is per-shape (TEP / DEP / TP differ at the same GPU count) and uses the real quantized weights — it replaces the old BF16 min-GPU weight floor entirely. + +With an explicit FPM profile, enumeration uses only its declared deployment +tuples and rank-local resource bounds. It does not resolve an analytical model +or infer another topology's resources from its GPU count. """ from __future__ import annotations from dataclasses import dataclass +from typing import Any from aiconfigurator.generator.naive import _estimate_model_weight_bytes from aiconfigurator_core.sdk import perf_database @@ -34,6 +39,7 @@ ) from .parallel_enum import ( DisaggParallelConfig, + ParallelShape, ReplicaParallelConfig, enumerate_disagg_configs, enumerate_parallel_configs, @@ -120,6 +126,7 @@ def parallel_configs_for( memory_fraction: float = DEFAULT_MEMORY_FRACTION, role_runtime: dict[str, tuple[int, int, float] | tuple[int, int, float, int | None]] | None = None, systems_path: str | None = None, + fpm_profile: dict[str, Any] | None = None, ) -> list[ReplicaParallelConfig] | list[DisaggParallelConfig]: """Resolve the model/hardware, then enumerate the parallel configs that fit the GPU budget and can hold a ``max_seq_len``-token sequence. @@ -139,8 +146,28 @@ def parallel_configs_for( :class:`NoViableParallelConfig` when no shape can hold the sequence within the budget. """ - mh = resolve_model_hardware(model_name, hardware_sku, backend=backend, systems_path=systems_path) - seq_len = max_seq_len if max_seq_len is not None else mh.max_context + profile = None + if fpm_profile is not None: + from aiconfigurator_core.sdk.fpm_profile import load_fpm_profile + + profile = load_fpm_profile(fpm_profile) + if profile.model != model_name: + raise ValueError("FPM profile model identity does not match the requested model") + seq_len = max_seq_len if max_seq_len is not None else profile.context_length + if seq_len > profile.context_length: + raise ValueError("max_seq_len exceeds FPM profile context_length") + configs = _profile_parallel_configs( + profile, + hardware_sku=hardware_sku, + backend=backend, + backend_version=backend_version, + gpu_budget=gpu_budget, + min_gpu_budget=min_gpu_budget, + deployment_mode=deployment_mode, + ) + else: + mh = resolve_model_hardware(model_name, hardware_sku, backend=backend, systems_path=systems_path) + seq_len = max_seq_len if max_seq_len is not None else mh.max_context if seq_len is None: raise ValueError(f"max_seq_len is required: {model_name} config exposes no max context length") @@ -148,20 +175,21 @@ def parallel_configs_for( # MoE tensor-parallel (moe_ep == 1) is enabled for every MoE model, MLA # included: real deployments (e.g. InferenceX GLM-5, reported as EP=1) run it, # so the search must be able to find it rather than have it filtered out here. - common = dict( - is_moe=mh.is_moe, - backend=backend, - gpu_budget=gpu_budget, - min_gpu_budget=min_gpu_budget, - enable_wideep=mh.enable_wideep, - allow_moe_pure_tp=True, - ) - if deployment_mode == "disagg": - configs = enumerate_disagg_configs(**common) - elif deployment_mode == "agg": - configs = enumerate_parallel_configs(**common) - else: - raise ValueError(f"deployment_mode must be 'agg' or 'disagg', got {deployment_mode!r}") + if profile is None: + common = dict( + is_moe=mh.is_moe, + backend=backend, + gpu_budget=gpu_budget, + min_gpu_budget=min_gpu_budget, + enable_wideep=mh.enable_wideep, + allow_moe_pure_tp=True, + ) + if deployment_mode == "disagg": + configs = enumerate_disagg_configs(**common) + elif deployment_mode == "agg": + configs = enumerate_parallel_configs(**common) + else: + raise ValueError(f"deployment_mode must be 'agg' or 'disagg', got {deployment_mode!r}") # KV-cache validity: keep configs whose every role-shape holds a max_seq_len sequence. def feasible_for(role: str, shapes): @@ -175,6 +203,20 @@ def feasible_for(role: str, shapes): raise ValueError( "role_runtime values must be (tokens, batch, memory) or (tokens, batch, memory, fixed_tokens)" ) + if profile is not None: + for shape in dict.fromkeys(shapes): + deployment = profile.select( + model=model_name, + system=hardware_sku, + backend=backend, + backend_version=backend_version, + tp_size=shape.tp, + pp_size=shape.pp, + attention_dp_size=shape.dp, + moe_tp_size=shape.moe_tp, + moe_ep_size=shape.moe_ep, + ) + deployment.resources.validate_envelope(max_num_tokens=role_tokens, max_batch_size=role_batch) if fixed_tokens is not None: return {shape: fixed_tokens for shape in dict.fromkeys(shapes) if fixed_tokens > seq_len} return feasible_shape_tokens( @@ -188,6 +230,7 @@ def feasible_for(role: str, shapes): max_batch_size=role_batch, memory_fraction=role_memory, systems_path=systems_path, + **({"fpm_profile": fpm_profile} if fpm_profile is not None else {}), ) if deployment_mode == "agg": @@ -203,3 +246,50 @@ def feasible_for(role: str, shapes): f"sequence within {gpu_budget} GPUs ({backend} KV-cache estimate)" ) return kept + + +def _profile_parallel_configs( + profile, + *, + hardware_sku: str, + backend: str, + backend_version: str | None, + gpu_budget: int, + min_gpu_budget: int | None, + deployment_mode: str, +): + """Enumerate only declared deployment shapes; replicas do not change a cell.""" + deployments = [ + deployment + for deployment in profile.deployments + if deployment.system == hardware_sku + and deployment.backend == backend + and deployment.backend_version == backend_version + ] + if not deployments: + raise ValueError(f"no FPM deployment profile for {hardware_sku}/{backend}/{backend_version}") + replicas = [] + for deployment in deployments: + shape = ParallelShape( + tp=deployment.tp, + pp=deployment.pp, + dp=deployment.dp, + moe_tp=deployment.moe_tp, + moe_ep=deployment.moe_ep, + ) + replicas.extend( + ReplicaParallelConfig(shape=shape, replicas=count) + for count in range(1, gpu_budget // shape.gpus_per_worker + 1) + ) + if deployment_mode == "agg": + configs = replicas + elif deployment_mode == "disagg": + configs = [ + DisaggParallelConfig(prefill=prefill, decode=decode) + for prefill in replicas + for decode in replicas + if prefill.total_gpus + decode.total_gpus <= gpu_budget + ] + else: + raise ValueError("FPM profiles support only agg or disagg deployment modes") + return [config for config in configs if min_gpu_budget is None or config.total_gpus >= min_gpu_budget] diff --git a/python/aisimulate/src/aisimulate/sweeper/sample.py b/python/aisimulate/src/aisimulate/sweeper/sample.py index 157354caa2..5a6104361b 100644 --- a/python/aisimulate/src/aisimulate/sweeper/sample.py +++ b/python/aisimulate/src/aisimulate/sweeper/sample.py @@ -132,6 +132,8 @@ def unroll_sample( sample[key] = getattr(search_space, key) if search_space.systems_path is not None: sample["systems_path"] = search_space.systems_path + if search_space.fpm_profile is not None: + sample["fpm_profile"] = search_space.fpm_profile if mode == "disagg": sample["prefill_hardware_sku"] = search_space.hardware_sku_for("prefill") @@ -156,6 +158,18 @@ def unroll_sample( sample[key] = selection[key] for key in pinned: sample[key] = selection.get(key, getattr(search_space, key)) + for role in ("agg", "prefill", "decode"): + if f"{role}_forward_model" not in sample: + continue + method = getattr(search_space, f"{role}_fpm_interpolation") + if search_space.fpm_profile is not None: + from aiconfigurator_core.sdk.fpm_profile import load_fpm_profile, resolve_fpm_interpolation + + # Pin the method before execution so replay cannot change it when + # another process has a different model registry. + method = resolve_fpm_interpolation(load_fpm_profile(search_space.fpm_profile), method) + if method != "auto" or search_space.fpm_profile is not None: + sample[f"{role}_fpm_interpolation"] = method if mode == "disagg": for key in ( "kv_transfer_bytes_per_token", diff --git a/python/aisimulate/src/aisimulate/sweeper/search_space.py b/python/aisimulate/src/aisimulate/sweeper/search_space.py index 3a34eeb18e..4f96085874 100644 --- a/python/aisimulate/src/aisimulate/sweeper/search_space.py +++ b/python/aisimulate/src/aisimulate/sweeper/search_space.py @@ -272,6 +272,7 @@ def _heterogeneous_disagg_configs( max_seq_len=max_seq_len, role_runtime={"agg": _role_runtime(search_space, backend, role)}, systems_path=search_space.systems_path, + **({"fpm_profile": search_space.fpm_profile} if search_space.fpm_profile is not None else {}), ) except (NoPerfDatabase, NoViableParallelConfig) as exc: raise type(exc)(f"{role} hardware_sku={hardware!r}: {exc}") from exc @@ -596,6 +597,7 @@ def matches_custom(config: _ParallelConfig) -> bool: max_seq_len=max_seq_len, role_runtime=_runtime_by_role(ss, backend, deployment_mode), systems_path=ss.systems_path, + **({"fpm_profile": ss.fpm_profile} if ss.fpm_profile is not None else {}), ) except (NoPerfDatabase, NoViableParallelConfig): continue # backend unusable for this mode -> drop it from the search diff --git a/python/aisimulate/tests/cross_package/test_core_public_api.py b/python/aisimulate/tests/cross_package/test_core_public_api.py index f5430143b9..a7730d2c22 100644 --- a/python/aisimulate/tests/cross_package/test_core_public_api.py +++ b/python/aisimulate/tests/cross_package/test_core_public_api.py @@ -83,6 +83,8 @@ def test_stable_function_signatures() -> None: "kv_block_size: 'int | None' = None, " "systems_path: 'str | None' = None, " "forward_model: 'str | None' = None, " + "fpm_profile: 'dict | str | FpmModelProfile | None' = None, " + "fpm_interpolation: 'str' = 'auto', cp_size: 'int' = 1, " "database_mode: 'str | None' = None, shared_layer: 'bool | None' = None, " "transfer_policy: 'str | list[str] | None' = None, " "strict_provenance: 'bool | None' = None) -> 'bytes'" diff --git a/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py b/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py new file mode 100644 index 0000000000..354753b986 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py @@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +import importlib.metadata +import json +import sys +from dataclasses import replace + +import pytest +import yaml +from collector.fpm_forward import capabilities, cli, memory_admission, planner, runner +from collector.fpm_forward.config import FPMCollectionOptions, reject_fpm_arguments_without_fpm +from collector.fpm_forward.model_capability import load_model_config + +pytestmark = pytest.mark.unit + + +def _profile(): + """Real GLM deployment identities with explicitly declared test-only bounds.""" + return { + "schema_version": 1, + "model": "nvidia/GLM-5.2-NVFP4", + "model_revision": "collector-test-snapshot", + "architecture": "GlmMoeDsaForCausalLM", + "context_length": 8192, + "num_experts": 256, + "provenance": "Collector test declaration; not a measured memory qualification.", + "deployments": [ + { + "system": "b200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "tp": tp, + "dp": dp, + "moe_tp": 1, + "moe_ep": 8, + "gemm_quant_mode": "nvfp4", + "moe_quant_mode": "nvfp4", + "fmha_quant_mode": "fp8", + "comm_quant_mode": "half", + "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 100_000_000_000, + "activations_bytes": 1_000_000_000, + "runtime_overhead_bytes": 1_000_000_000, + "comm_overhead_bytes": 1_000_000_000, + "kv_bytes_per_token": 60_216, + "cache_layout": "linear", + "max_num_tokens": 8192, + "max_batch_size": 1024, + "provenance": "Independent test envelope: 100 GB weights plus three 1 GB overhead bounds.", + }, + } + for tp, dp in ((1, 8), (8, 1)) + ], + } + + +def _argv(profile): + return [ + "--model-path", + profile["model"], + "--gpu", + "b200_sxm", + "--fpm-max-gpus", + "8", + "--fpm-gpu-counts", + "8", + "--fpm-parallel-presets", + "dep", + "tep", + "--fpm-kv-cache-dtypes", + "fp8", + ] + + +def _plan(profile, **overrides): + args = cli._parser().parse_args(_argv(profile)) + kwargs = { + "backend": "vllm", + "model_path": profile["model"], + "system": "b200_sxm", + "selected_ops": {"dsa_context_module", "dsa_generation_module"}, + "options": FPMCollectionOptions.from_args(args), + "fpm_profile": profile, + } + kwargs.update(overrides) + return planner.build_collection_plan(**kwargs) + + +@pytest.fixture +def no_models_or_timing_data(monkeypatch): + def forbidden(*_args, **_kwargs): + pytest.fail("profile collection constructed a model or consulted timing data") + + from aiconfigurator_core.sdk import engine, memory, models, perf_database + from aiconfigurator_core.sdk.models import base + + monkeypatch.setattr(models, "_MODEL_REGISTRY", {}) + monkeypatch.setattr(base, "_MODEL_REGISTRY", {}) + for module in (models, memory, engine): + monkeypatch.setattr(module, "get_model", forbidden) + monkeypatch.setattr(memory_admission.KVCacheEstimator, "from_request", forbidden) + for module in (capabilities, perf_database): + monkeypatch.setattr(module, "get_database", forbidden) + monkeypatch.setattr(module, "get_latest_database_version", forbidden) + monkeypatch.setattr(capabilities, "context_fmha_supported_modes", forbidden) + monkeypatch.setattr(planner, "_git_revision", lambda: "profile-collection-test") + + +@pytest.mark.parametrize("file_format", ["json", "yaml"]) +def test_glm_cli_plans_without_model_classes_or_any_timing_data( + tmp_path, monkeypatch, capsys, no_models_or_timing_data, file_format +): + profile = _profile() + path = tmp_path / f"profile.{file_format}" + path.write_text(json.dumps(profile) if file_format == "json" else yaml.safe_dump(profile)) + monkeypatch.setenv("COLLECTOR_MODEL_PATH", "") + + assert cli.main([*_argv(profile), "--fpm-model-profile", str(path), "--plan-only"]) == 0 + + output = json.loads(capsys.readouterr().out) + assert output["counts"]["cells"] == 4 + assert {cell["cell_id"] for cell in output["cells"]} == { + "fpm-6efe9f97155a97e4", + "fpm-093958a8e9575a8a", + "fpm-272431d375ba264f", + "fpm-8ccaea64a44c2882", + } + assert {cell["parallel_strategy"] for cell in output["cells"]} == {"dep", "tep"} + assert {cell["resolved_dtypes"]["fmha_quant_mode"] for cell in output["cells"]} == {"fp8"} + assert {cell["resolved_dtypes"]["fmha_resolution"] for cell in output["cells"]} == {"checkpoint_native"} + assert output["fpm_profile"]["model_revision"] == "collector-test-snapshot" + assert {decision["source"] for decision in output["topology_memory_admission"]} == {"fpm_profile_declared"} + for decision in output["topology_memory_admission"]: + assert decision["estimates"][0]["estimated_non_kv_bytes"] == 103_000_000_000 + assert decision["estimates"][0]["provenance"] == profile["deployments"][0]["resources"]["provenance"] + assert decision["activation_envelope"]["max_batch_size"] == 1024 + assert decision["activation_envelope"]["scope"] == "rank_local" + + +def test_unknown_architecture_plans_and_renders_from_real_config(tmp_path, no_models_or_timing_data): + profile = _profile() + config = load_model_config(profile["model"]).payload + config["architectures"] = ["UnregisteredMoeForCausalLM"] + profile.update(model="private-org/new-model", architecture=config["architectures"][0]) + path = tmp_path / "config.json" + path.write_text(json.dumps(config)) + + plan = _plan(profile, model_config_path=str(path), has_model_cases=False) + assert plan.capability.support_level == "bootstrap_template" + cell_dir = tmp_path / "rendered" + cell_dir.mkdir() + runner._render_cell(plan, plan.cells[0], cell_dir, {}) + + script = (cell_dir / "run.sh").read_text() + assert "--model private-org/new-model" in script + assert "--max-num-batched-tokens 8192" in script + assert "--max-num-seqs 1024" in script + + +def test_frozen_profile_content_invalidates_resume_without_changing_cell_ids(tmp_path, no_models_or_timing_data): + source = _profile() + first = _plan(source) + original = first.to_dict() + source["model_revision"] = "different-test-snapshot" + second = _plan(source) + detached = first.fpm_profile + detached.deployments[0].resources.weights_bytes += 10 + + assert first.to_dict() == original + assert first.sha256 != second.sha256 + assert [cell.cell_id for cell in first.cells] == [cell.cell_id for cell in second.cells] + assert _plan(original["fpm_profile"]).sha256 == first.sha256 + checkpoint = tmp_path / "checkpoint.json" + checkpoint.write_text(json.dumps({"schema": runner.CHECKPOINT_SCHEMA, "plan_sha256": first.sha256, "cells": {}})) + with pytest.raises(ValueError, match="checkpoint does not match the current frozen plan"): + runner._load_checkpoint(checkpoint, second, resume=True) + + +@pytest.mark.parametrize("field,value", [("fmha_quant_mode", "bfloat16"), ("moe_backend", "flashinfer_cutlass")]) +def test_profile_cannot_override_resolved_precision_or_backend(field, value, no_models_or_timing_data): + profile = _profile() + profile["deployments"][0][field] = value + with pytest.raises(ValueError, match="profile identity mismatch"): + _plan(profile) + + +def test_historical_minimax_transfer_precision_cannot_relabel_new_collection(no_models_or_timing_data): + profile = _profile() + profile.update(model="MiniMaxAI/MiniMax-M2.7", architecture="MiniMaxM2ForCausalLM") + deployment = profile["deployments"][0] + deployment.update( + system="h200_sxm", + tp=4, + dp=1, + moe_tp=4, + moe_ep=1, + gemm_quant_mode="fp8_block", + moe_quant_mode="fp8_block", + fmha_quant_mode="bfloat16", + ) + profile["deployments"] = [deployment] + options = replace( + FPMCollectionOptions.from_args(cli._parser().parse_args(_argv(profile))), + max_gpus=4, + gpu_counts=(4,), + parallel_presets=("pure_tp",), + ) + with pytest.raises(ValueError, match="fmha_quant_mode: resolved='fp8', profile='bfloat16'"): + _plan(profile, options=options, system="h200_sxm", selected_ops={"attention_context", "attention_generation"}) + + +@pytest.mark.parametrize("field,value", [("model", "other/model"), ("architecture", "OtherArchitecture")]) +def test_profile_model_identity_must_match_real_checkpoint(field, value, no_models_or_timing_data): + profile = _profile() + profile[field] = value + with pytest.raises(ValueError, match="profile (model identity|architecture) mismatch"): + _plan(profile, model_path="nvidia/GLM-5.2-NVFP4") + + +def test_missing_requested_topology_raises_before_memory_admission(monkeypatch, no_models_or_timing_data): + profile = _profile() + profile["deployments"] = profile["deployments"][:1] + monkeypatch.setattr(memory_admission, "load_system_spec", lambda *_args: pytest.fail("admission ran")) + with pytest.raises(ValueError, match="no matching FPM deployment profile"): + _plan(profile) + + +def test_profile_collection_rejects_non_native_kv_dtype(no_models_or_timing_data): + profile = _profile() + options = replace( + FPMCollectionOptions.from_args(cli._parser().parse_args(_argv(profile))), kv_cache_dtypes=("bfloat16",) + ) + with pytest.raises(ValueError, match="checkpoint-native KV dtype"): + _plan(profile, options=options) + + +@pytest.mark.parametrize("field,value", [("max_num_tokens", 4096), ("max_batch_size", 1)]) +def test_resource_envelope_errors_do_not_fail_open(field, value, no_models_or_timing_data): + profile = _profile() + profile["deployments"][0]["resources"][field] = value + options = replace( + FPMCollectionOptions.from_args(cli._parser().parse_args(_argv(profile))), max_prefill_batch_size=4 + ) + with pytest.raises(ValueError, match="resource envelope exceeded"): + _plan(profile, options=options) + + +def test_declared_size_capacity_filter_counts_rejected_topology(caplog, no_models_or_timing_data): + profile = _profile() + profile["deployments"][0]["resources"]["weights_bytes"] = 200_000_000_000 + plan = _plan(profile) + assert {cell.parallel_strategy for cell in plan.cells} == {"tep"} + assert [decision.disposition for decision in plan.topology_memory_admission] == ["rejected", "admitted"] + assert "dropped 1/2 topologies" in caplog.text + + +@pytest.mark.parametrize("phase", ["prefill", "decode"]) +@pytest.mark.parametrize("smoke", [False, True]) +def test_profile_bounds_native_scheduling_before_cases_are_queued(phase, smoke, no_models_or_timing_data): + plan = _plan(_profile()) + cell = next(cell for cell in plan.cells if cell.workload_kind == phase) + arguments = runner._cell_generator_overrides(plan, cell, {}, smoke=smoke)["params"]["agg"]["extra_cli_args"] + assert arguments.count("--max-num-batched-tokens") == 1 + assert arguments[arguments.index("--max-num-batched-tokens") + 1] == "8192" + assert arguments.count("--max-num-seqs") == 1 + assert arguments[arguments.index("--max-num-seqs") + 1] == "1024" + + +def test_profile_preserves_explicit_prefill_envelope(no_models_or_timing_data): + profile = _profile() + options = replace( + FPMCollectionOptions.from_args(cli._parser().parse_args(_argv(profile))), + max_prefill_isl=2048, + max_prefill_batch_size=4, + ) + plan = _plan(profile, options=options) + for cell in plan.cells: + arguments = runner._cell_generator_overrides(plan, cell, {})["params"]["agg"]["extra_cli_args"] + expected_tokens, expected_batch = ("2048", "4") if cell.workload_kind == "prefill" else ("8192", "1024") + assert arguments[arguments.index("--max-num-batched-tokens") + 1] == expected_tokens + assert arguments[arguments.index("--max-num-seqs") + 1] == expected_batch + + +@pytest.mark.parametrize("actual_version", ["0.25.1", "0.24.0"]) +def test_profile_runtime_version_is_observed_before_execution(tmp_path, monkeypatch, actual_version): + resource = object.__new__(runner.KubernetesCellRunner) + monkeypatch.setattr(runner, "FPM_RESULTS_DIR", str(tmp_path)) + monkeypatch.setattr(importlib.metadata, "version", lambda _: actual_version) + + def execute(_pod, command, timeout): + if command[:2] == ["python3", "-c"]: + monkeypatch.setattr(sys, "argv", ["-c", *command[3:]]) + exec(command[2], {}) + + resource._exec_checked = execute + arguments = dict(cell_id="cell", plan_sha256="plan", attempt_id="attempt", expected_backend_version="0.25.1") + if actual_version != "0.25.1": + with pytest.raises(RuntimeError, match="profile runtime mismatch"): + resource.prepare_attempt(["pod-0"], **arguments) + else: + resource.prepare_attempt(["pod-0"], **arguments) + provenance = json.loads((tmp_path / "collector-provenance.json").read_text()) + assert provenance["runtime"]["backend_version"] == actual_version + + +def test_profile_input_is_not_valid_for_op_collection(): + from argparse import Namespace + + with pytest.raises(ValueError, match="--fpm-model-profile"): + reject_fpm_arguments_without_fpm(Namespace(ops=["gemm"], fpm_model_profile="profile.json")) + + +def test_multiple_profile_runtime_versions_fail_explicitly(no_models_or_timing_data): + profile = _profile() + other = copy.deepcopy(profile["deployments"][0]) + other["backend_version"] = "0.24.0" + profile["deployments"].append(other) + with pytest.raises(ValueError, match="must identify one runtime version"): + _plan(profile) diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py new file mode 100644 index 0000000000..e05c41aeee --- /dev/null +++ b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py @@ -0,0 +1,612 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FPM profiles keep identity, memory admission and compilation graph-independent.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from aiconfigurator_core.sdk import engine, memory +from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile, load_fpm_profile, resolve_fpm_interpolation + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def profile_dict(): + return { + "schema_version": 1, + "model": "test/unknown-decoder", + "model_revision": "profile-test-fixture-v1", + "architecture": "UnregisteredDecoderForCausalLM", + "context_length": 4096, + "num_experts": 64, + "provenance": "Synthetic metadata for SDK behavior tests; not silicon qualification.", + "deployments": [ + { + "system": "test_gpu", + "backend": "vllm", + "backend_version": "0.25.1", + "tp": 2, + "dp": 1, + "moe_tp": 2, + "moe_ep": 1, + "gemm_quant_mode": "nvfp4", + "moe_quant_mode": "nvfp4", + "fmha_quant_mode": "fp8", + "comm_quant_mode": "half", + "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 100, + "activations_bytes": 20, + "runtime_overhead_bytes": 30, + "comm_overhead_bytes": 50, + "kv_bytes_per_token": 10, + "cache_layout": "linear", + "max_num_tokens": 8192, + "max_batch_size": 256, + "provenance": "Declared per-rank arithmetic fixture; excludes CUDA graphs.", + }, + } + ], + } + + +def _fail_graph(*_args, **_kwargs): + raise AssertionError("analytical model construction or timing database was accessed") + + +@pytest.fixture +def direct_compile(monkeypatch): + monkeypatch.setattr(engine, "get_model", _fail_graph) + monkeypatch.setattr(engine, "build_model_config", _fail_graph) + monkeypatch.setattr(engine, "_maybe_load_database", _fail_graph) + monkeypatch.setattr(engine, "_literal_backend_version", lambda _s, _b, version, *_args: version) + monkeypatch.setattr(engine.aiconfigurator_core, "engine_spec_bincode_from_json", lambda value: value.encode()) + + def compile_profile(profile, **kwargs): + return json.loads( + engine.compile_engine( + "test/unknown-decoder", + "test_gpu", + "vllm", + "0.25.1", + tp_size=2, + moe_tp_size=2, + forward_model="fpm", + fpm_profile=profile, + **kwargs, + ) + ) + + return compile_profile + + +def test_unknown_profile_auto_compiles_without_operations(profile_dict, direct_compile): + spec = direct_compile(profile_dict) + profile = FpmModelProfile.model_validate(profile_dict) + deployment = profile.deployments[0] + assert json.loads(spec["engine"]["extra"]["fpm_profile"]) == profile.model_dump(mode="json") + assert spec["engine"]["extra"]["fpm_interpolation"] == "direct" + assert spec["engine"]["activation_dtype"] == "fp8" + for key, phase in (("context_ops", "prefill"), ("generation_ops", "decode")): + assert len(spec[key]) == 1 + op = spec[key][0]["FpmForward"] + assert op["phase"] == phase + assert op["model_path"] == profile.model + assert op["match_identity"] == deployment.match_identity() + assert op["interpolation"] == "direct" + assert op["sol_ops"] == [] + assert op["weight_bytes"] == 100 + + +def test_direct_json_transport_preserves_all_identity_fields(profile_dict, direct_compile): + profile_dict["deployments"][0].update(moe_backend="pinned_moe", attention_backend="pinned_attention") + spec = direct_compile(json.dumps(profile_dict), fpm_interpolation="direct") + assert spec["context_ops"][0]["FpmForward"]["match_identity"] == [ + "nvfp4", + "nvfp4", + "fp8", + "half", + "fp8", + "2", + "1", + "1", + "2", + "1", + "1", + "pinned_moe", + "pinned_attention", + "False", + "False", + ] + + +@pytest.mark.parametrize("override", [{"fmha_quant_mode": "bfloat16"}, {"gemm_quant_mode": "fp8"}]) +def test_precision_conflicts_fail_before_build(profile_dict, direct_compile, override): + with pytest.raises(ValueError, match="FPM profile identity conflict"): + direct_compile(profile_dict, **override) + + +@pytest.mark.parametrize("cp_size", [2, 0, True, 1.0, "1", None]) +@pytest.mark.parametrize("with_profile", [False, True]) +def test_sdk_entry_points_reject_unsupported_cp_before_construction(profile_dict, monkeypatch, cp_size, with_profile): + monkeypatch.setattr(engine, "build_model_config", _fail_graph) + monkeypatch.setattr(memory.KVCacheEstimator, "from_request", _fail_graph) + monkeypatch.setattr(memory.NaiveKVCacheEstimator, "from_model_path", _fail_graph) + kwargs = {"cp_size": cp_size, "fpm_profile": profile_dict if with_profile else None} + with pytest.raises(ValueError, match="cp_size must be the integer 1"): + engine.compile_engine("test/unknown-decoder", "test_gpu", "vllm", "0.25.1", forward_model="fpm", **kwargs) + with pytest.raises(ValueError, match="cp_size must be the integer 1"): + memory.estimate_kv_cache( + "test/unknown-decoder", + "test_gpu", + "vllm", + "0.25.1", + max_num_tokens=8192, + max_batch_size=256, + memory_fraction_kind="of_total", + memory_fraction_value=0.8, + allow_naive_fallback=True, + **kwargs, + ) + + +@pytest.mark.parametrize("kwargs", [{"nextn": 1}, {"cp_size": 2}, {"database_mode": "SOL"}]) +def test_direct_rejects_unsupported_execution(profile_dict, direct_compile, kwargs): + with pytest.raises(ValueError): + direct_compile(profile_dict, **kwargs) + + +def test_direct_requires_explicit_resources(direct_compile): + with pytest.raises(ValueError, match="requires an fpm_profile"): + direct_compile(None, fpm_interpolation="direct") + + +def test_explicit_sol_does_not_fall_back_on_unknown_architecture(profile_dict, direct_compile): + with pytest.raises(ValueError, match="requires a registered analytical model class"): + direct_compile(profile_dict, fpm_interpolation="sol") + + +def test_registered_auto_uses_registry_presence_without_constructing(profile_dict, monkeypatch): + from aiconfigurator_core.sdk.models.base import _MODEL_REGISTRY + + profile = load_fpm_profile(profile_dict) + monkeypatch.setitem(_MODEL_REGISTRY, profile.architecture, object()) + assert resolve_fpm_interpolation(profile) == "sol" + assert resolve_fpm_interpolation(profile, "direct") == "direct" + assert resolve_fpm_interpolation(None) == "sol" + + +def test_registered_build_failure_is_not_direct_fallback(profile_dict, monkeypatch): + from aiconfigurator_core.sdk.models.base import _MODEL_REGISTRY + + monkeypatch.setitem(_MODEL_REGISTRY, profile_dict["architecture"], object()) + monkeypatch.setattr(engine, "_literal_backend_version", lambda _s, _b, version, *_args: version) + observed = [] + + def broken_model(_path, config, _backend): + observed.append(config.fmha_quant_mode.name) + raise RuntimeError("registered model has an unrelated implementation error") + + monkeypatch.setattr(engine, "get_model", broken_model) + with pytest.raises(RuntimeError, match="unrelated implementation error"): + engine.compile_engine( + profile_dict["model"], + "test_gpu", + "vllm", + "0.25.1", + tp_size=2, + moe_tp_size=2, + forward_model="fpm", + fpm_profile=profile_dict, + ) + assert observed == ["fp8"] + + +@pytest.mark.parametrize( + "mutation", + [ + lambda p: p.update(unrecognized_field=1), + lambda p: p.update(model_revision="main"), + lambda p: p.update(model_revision="unknown"), + lambda p: p.update(context_length=True), + lambda p: p.update(schema_version=True), + lambda p: p.update(schema_version=1.0), + lambda p: p.update(provenance=" "), + lambda p: p.update(deployments=[]), + lambda p: p["deployments"].append(copy.deepcopy(p["deployments"][0])), + lambda p: p["deployments"][0].update(tp="2"), + lambda p: p["deployments"][0].update(pp=2), + lambda p: p["deployments"][0].update(cp=2), + lambda p: p["deployments"][0].update(cp=True), + lambda p: p["deployments"][0].update(pp=1.0), + lambda p: p["deployments"][0].update(dp=2), + lambda p: p["deployments"][0].update(backend_version="current"), + lambda p: p["deployments"][0].update(fmha_quant_mode="bf16"), + lambda p: p["deployments"][0].update(enable_wideep=True), + lambda p: p["deployments"][0].update(enable_eplb=0), + lambda p: p["deployments"][0]["resources"].update(cache_layout="sliding_window"), + lambda p: p["deployments"][0]["resources"].update(kv_bytes_per_token=0), + lambda p: p["deployments"][0]["resources"].update(weights_bytes=-1), + lambda p: p["deployments"][0]["resources"].update(weights_bytes=1.5), + lambda p: p["deployments"][0]["resources"].update(weights_bytes=2**53), + ], +) +def test_profiles_reject_ambiguous_or_unsupported_metadata(profile_dict, mutation): + mutation(profile_dict) + with pytest.raises(ValidationError): + load_fpm_profile(profile_dict) + + +def test_topologies_have_separate_resources_and_expert_admission(profile_dict): + tep = profile_dict["deployments"][0] + tep.update(tp=8, dp=1, moe_tp=1, moe_ep=8) + dep = copy.deepcopy(tep) + dep.update(tp=1, dp=8) + dep["resources"]["weights_bytes"] = 180 + profile_dict["deployments"].append(dep) + profile = load_fpm_profile(profile_dict) + selected = profile.select( + model=profile.model, + system="test_gpu", + backend="vllm", + backend_version="0.25.1", + tp_size=1, + attention_dp_size=8, + moe_tp_size=1, + moe_ep_size=8, + ) + assert selected.resources.weights_bytes == 180 + assert profile.deployments[0].resources.weights_bytes == 100 + with pytest.raises(ValueError, match="no matching FPM deployment"): + profile.select(model=profile.model, system="test_gpu", backend="vllm", backend_version="0.25.1") + with pytest.raises(ValueError, match="model identity mismatch"): + profile.select(model="other/model", system="test_gpu", backend="vllm", backend_version="0.25.1") + profile_dict["num_experts"] = 63 + with pytest.raises(ValidationError, match="divisible by moe_ep"): + load_fpm_profile(profile_dict) + + +def test_dense_tp_does_not_require_expert_tensor_parallelism(profile_dict): + profile_dict["num_experts"] = 0 + profile_dict["deployments"][0]["moe_tp"] = 1 + assert load_fpm_profile(profile_dict).deployments[0].parallel_tuple == (2, 1, 1, 1, 1, 1) + + +@pytest.mark.parametrize("kwargs", [{"moe_tp_size": 0}, {"moe_ep_size": False}, {"cp_size": True}]) +def test_select_rejects_zero_or_boolean_dimensions(profile_dict, kwargs): + profile = load_fpm_profile(profile_dict) + with pytest.raises(ValueError, match="positive integers"): + profile.select(model=profile.model, system="test_gpu", backend="vllm", backend_version="0.25.1", **kwargs) + + +@pytest.fixture +def profile_memory(profile_dict, monkeypatch): + monkeypatch.setattr(memory, "get_model", _fail_graph) + monkeypatch.setattr(memory, "build_model_config", _fail_graph) + monkeypatch.setattr(memory.perf_database, "get_database", _fail_graph) + monkeypatch.setattr(memory.perf_database, "resolve_query_version", _fail_graph) + monkeypatch.setattr(memory.perf_database, "load_system_spec", lambda *_a, **_k: {"gpu": {"mem_capacity": 1000}}) + monkeypatch.setattr(engine, "_literal_backend_version", lambda _s, _b, version, *_args: version) + + def estimate(**overrides): + args = dict( + max_num_tokens=8192, + max_batch_size=256, + memory_fraction_kind="of_total", + memory_fraction_value=0.8, + tp_size=2, + moe_tp_size=2, + fpm_profile=profile_dict, + ) + args.update(overrides) + return memory.estimate_kv_cache("test/unknown-decoder", "test_gpu", "vllm", "0.25.1", **args) + + return estimate + + +def test_profile_memory_uses_declared_bounds_without_graph_or_timing_data(profile_memory): + result = profile_memory(cuda_graph_reserved_bytes=20, tolerance_fraction=0.1) + assert result["source"] == "profile" + assert result["total_gpu_capacity_bytes"] == 1000 + assert result["total_kv_size_bytes"] == 580 # 1000 * .8 - 200 - 20 + assert result["kv_size_per_token_bytes"] == 10 + assert result["total_kv_size_tokens"] == 58 + assert result["tolerance_adjusted"]["total_kv_size_bytes"] == 522 + assert result["tolerance_adjusted"]["total_kv_size_tokens"] == 52 + assert result["memory_breakdown"]["cuda_graph_reserved_bytes"] == 20 + assert "Declared per-rank" in result["resource_provenance"] + json.dumps(result) # The result is replayable; it contains no callables. + + +def test_profile_memory_capacity_override_does_not_read_hardware(profile_memory, monkeypatch): + monkeypatch.setattr(memory.perf_database, "load_system_spec", _fail_graph) + result = profile_memory(gpu_memory_capacity_bytes_override=2000) + assert result["total_kv_size_bytes"] == 1400 + + +def test_profile_block_budget_preserves_resource_provenance(profile_dict, profile_memory): + diagnostics = {} + blocks = memory.estimate_num_gpu_blocks( + "test/unknown-decoder", + "test_gpu", + "vllm", + "0.25.1", + scheduler_block_size=4, + max_num_tokens=8192, + max_batch_size=256, + memory_fraction_kind="of_total", + memory_fraction_value=0.8, + tp_size=2, + moe_tp_size=2, + fpm_profile=profile_dict, + diagnostics=diagnostics, + ) + assert blocks == 15 # (1000 * .8 - 200) / 10 / 4 + assert diagnostics["source"] == "profile" + assert diagnostics["resource_provenance"] == profile_dict["deployments"][0]["resources"]["provenance"] + with pytest.raises(ValueError, match="positive integer"): + memory.estimate_num_gpu_blocks( + "test/unknown-decoder", + "test_gpu", + "vllm", + "0.25.1", + scheduler_block_size=4, + max_num_tokens=True, + max_batch_size=256, + memory_fraction_kind="of_total", + memory_fraction_value=0.8, + tp_size=2, + moe_tp_size=2, + fpm_profile=profile_dict, + ) + + +@pytest.mark.parametrize( + ("kwargs", "reason"), + [ + ({"max_num_tokens": 8193}, "resource envelope exceeded"), + ({"max_batch_size": 257}, "resource envelope exceeded"), + ({"max_num_tokens": True}, "positive integer"), + ({"fmha_quant_mode": "bfloat16"}, "identity conflict"), + ({"cp_size": 2}, "cp_size must be the integer 1"), + ({"nextn": 1}, "nextn must be 0"), + ({"gpu_memory_capacity_bytes_override": 200}, "no KV budget"), + ({"memory_fraction_kind": "of_free"}, "incompatible memory fraction"), + ], +) +def test_profile_memory_fails_closed(profile_memory, kwargs, reason): + with pytest.raises(ValueError, match=reason): + profile_memory(**kwargs) + + +@pytest.fixture +def native_profile_config(profile_dict, monkeypatch): + # Use a bundled system/version for native engine construction. Resource and + # precision variants below are transport fixtures, not measured timing cells. + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + profile_dict.update(model="MiniMaxAI/MiniMax-M2.7") + profile_dict["deployments"][0].update(system="h200_sxm") + monkeypatch.setattr(engine, "get_model", _fail_graph) + specs = [] + encode = engine.aiconfigurator_core.engine_spec_bincode_from_json + + def record_spec(value): + specs.append(json.loads(value)) + return encode(value) + + monkeypatch.setattr(engine.aiconfigurator_core, "engine_spec_bincode_from_json", record_spec) + + def compile_config(): + engine.compile_engine( + profile_dict["model"], + "h200_sxm", + "vllm", + "0.25.1", + tp_size=2, + moe_tp_size=2, + forward_model="fpm", + fpm_profile=profile_dict, + fpm_interpolation="direct", + ) + return specs[-1]["engine"] + + return compile_config, specs + + +@pytest.mark.parametrize( + ("gemm_mode", "moe_mode"), + [ + ("fp8_block", "nvfp4"), + ("int8_wo", "nvfp4"), + ("int4_wo", "nvfp4"), + ("sq", "nvfp4"), + ("fp8_ootb", "nvfp4"), + ("nvfp4", "int4_wo"), + ], +) +def test_native_recompilation_preserves_profile_precision(profile_dict, native_profile_config, gemm_mode, moe_mode): + profile_dict["deployments"][0].update(gemm_quant_mode=gemm_mode, moe_quant_mode=moe_mode) + compile_config, specs = native_profile_config + config = compile_config() + native = engine.aiconfigurator_core.RustForwardPassPerfModel.from_native(json.dumps(config)) + assert json.loads(native.diagnostics())["source"] == "aic" + assert len(specs) == 2 + for phase in ("context_ops", "generation_ops"): + assert specs[0][phase] == specs[1][phase] + + +@pytest.mark.parametrize("field", ["weight_dtype", "moe_dtype", "activation_dtype", "kv_cache_dtype"]) +def test_native_profile_rejects_different_wire_dtype(native_profile_config, field): + compile_config, _ = native_profile_config + config = compile_config() + config[field] = "bfloat16" + with pytest.raises(ValueError, match="FPM profile identity conflict"): + engine.aiconfigurator_core.RustForwardPassPerfModel.best_available(json.dumps(config), "prefill") + + +@pytest.mark.parametrize( + "case", + ["malformed_profile", "invalid_method", "missing_profile", "unrelated_construction_error", "cp_without_profile"], +) +def test_explicit_fpm_requests_never_become_regression_fallback(profile_dict, native_profile_config, monkeypatch, case): + compile_config, _ = native_profile_config + config = compile_config() + if case == "malformed_profile": + config["extra"]["fpm_profile"] = "{}" + elif case == "invalid_method": + config["extra"]["fpm_interpolation"] = "dierct" + elif case == "missing_profile": + config["extra"].pop("fpm_profile") + elif case == "cp_without_profile": + config["extra"] = {} + config["cp_size"] = 2 + else: + profile_dict["architecture"] = "MiniMaxM2ForCausalLM" + config["extra"]["fpm_profile"] = json.dumps(profile_dict) + config["extra"]["fpm_interpolation"] = "sol" + + def unrelated_error(*_args, **_kwargs): + raise RuntimeError("unrelated construction error") + + monkeypatch.setattr(engine, "get_model", unrelated_error) + with pytest.raises(ValueError, match="invalid engine config"): + engine.aiconfigurator_core.RustForwardPassPerfModel.best_available(json.dumps(config), "prefill") + + +def test_legacy_unsupported_model_can_still_use_regression(native_profile_config, monkeypatch): + compile_config, _ = native_profile_config + config = compile_config() + config["extra"] = {} + + def unsupported(*_args, **_kwargs): + raise ValueError("legacy model is unavailable") + + monkeypatch.setattr(engine, "get_model", unsupported) + model = engine.aiconfigurator_core.RustForwardPassPerfModel.best_available(json.dumps(config), "prefill") + assert json.loads(model.diagnostics())["source"] == "fallback_regression" + + +@pytest.mark.parametrize( + ("model", "system", "tp", "dp", "moe_tp", "moe_ep", "gemm", "fmha"), + [ + ("MiniMaxAI/MiniMax-M2.7", "h200_sxm", 4, 1, 4, 1, "fp8_block", "bfloat16"), + ("nvidia/GLM-5.2-NVFP4", "b200_sxm", 1, 8, 1, 8, "nvfp4", "fp8"), + ("nvidia/GLM-5.2-NVFP4", "b200_sxm", 8, 1, 1, 8, "nvfp4", "fp8"), + ], +) +def test_real_fpm_cells_compile_and_query_with_model_construction_disabled( + profile_dict, monkeypatch, model, system, tp, dp, moe_tp, moe_ep, gemm, fmha +): + import pyarrow.parquet as pq + + import aiconfigurator_core + + # Synthetic resource declarations deliberately isolate timing compilation; + # this test does not qualify real GPU memory. The measured model identity + # remains unchanged, while every analytical construction entry point fails. + # This retained dataset is outside the current/next serving-version slots. + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + profile_dict.update(model=model) + profile_dict["deployments"][0].update( + system=system, + tp=tp, + dp=dp, + moe_tp=moe_tp, + moe_ep=moe_ep, + gemm_quant_mode=gemm, + moe_quant_mode=gemm, + fmha_quant_mode=fmha, + ) + monkeypatch.setattr(engine, "get_model", _fail_graph) + monkeypatch.setattr(engine, "build_model_config", _fail_graph) + monkeypatch.setattr(engine, "_maybe_load_database", _fail_graph) + blob = engine.compile_engine( + model, + system, + "vllm", + "0.25.1", + tp_size=tp, + attention_dp_size=dp, + moe_tp_size=moe_tp, + moe_ep_size=moe_ep, + forward_model="fpm", + fpm_profile=profile_dict, + fpm_interpolation="direct", + ) + native = aiconfigurator_core.AicEngine.from_spec(blob) + data_path = Path(aiconfigurator_core.__file__).parent / "systems/data" / system / "vllm/0.25.1" + rows = pq.read_table( + data_path / "fpm_forward_perf.parquet", + filters=[("model_path", "=", model), ("tp", "=", tp), ("dp", "=", dp), ("batch_size", "=", 1)], + ).to_pylist() + prefill = next( + row + for row in rows + if row["workload_kind"] == "prefill" and row["total_prefill_tokens"] == 1 and row["total_kv_read_tokens"] == 0 + ) + decode = min( + (row for row in rows if row["workload_kind"] == "decode" and row["kv_seed_regime"] != "fake_fallback"), + key=lambda row: row["total_kv_read_tokens"], + ) + assert native.predict_prefill_latency(1, 1, 0) == pytest.approx(prefill["latency_ms"]) + assert native.predict_decode_latency_total(1, decode["total_kv_read_tokens"]) == pytest.approx(decode["latency_ms"]) + # The modular native builder crosses back through Python with the profile + # encoded in EngineConfig.extra's existing string map. Exercise this second + # transport independently of Python's direct compile_engine entry point. + config = { + "schema_version": 1, + "model_name": model, + "system_name": system, + "backend": "vllm", + "backend_version": "0.25.1", + "forward_model": "fpm", + "tp_size": tp, + "pp_size": 1, + "attention_dp_size": dp, + "moe_tp_size": moe_tp, + "moe_ep_size": moe_ep, + "cp_size": 1, + "extra": {"fpm_profile": json.dumps(profile_dict), "fpm_interpolation": "direct"}, + } + forward = aiconfigurator_core.RustForwardPassPerfModel.from_native(json.dumps(config)) + assert json.loads(forward.diagnostics())["source"] == "aic" + + +def test_registered_glm_auto_retains_sol_with_explicit_fp8_fmha(profile_dict, monkeypatch): + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + profile_dict.update(model="nvidia/GLM-5.2-NVFP4", architecture="GlmMoeDsaForCausalLM") + profile_dict["deployments"][0].update(system="b200_sxm", tp=1, dp=8, moe_tp=1, moe_ep=8) + real_get_model = engine.get_model + built_models = [] + + def record_model(*args): + model = real_get_model(*args) + built_models.append(model) + return model + + monkeypatch.setattr(engine, "get_model", record_model) + monkeypatch.setattr(engine, "_direct_fpm_spec_json", _fail_graph) + blob = engine.compile_engine( + "nvidia/GLM-5.2-NVFP4", + "b200_sxm", + "vllm", + "0.25.1", + tp_size=1, + attention_dp_size=8, + moe_tp_size=1, + moe_ep_size=8, + forward_model="fpm", + fpm_profile=profile_dict, + ) + assert len(built_models) == 1 + assert built_models[0].config.fmha_quant_mode.name == "fp8" + assert built_models[0].context_ops[0]._sol_ops + assert engine.EngineHandle(blob).predict_prefill_latency(1, 1) > 0 diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index 6a24bc85a2..58f3827257 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -932,7 +932,11 @@ def test_recommendation_candidate_yaml_round_trips_forward_model() -> None: adapter_sections={}, ) - assert prediction["engine"]["workers"]["aggregated"]["timing"] == {"type": "default", "forward_model": "fpm"} + assert prediction["engine"]["workers"]["aggregated"]["timing"] == { + "type": "default", + "forward_model": "fpm", + "fpm_interpolation": "auto", + } CorePredictionConfig.model_validate(prediction) @@ -962,7 +966,11 @@ def test_recommendation_candidate_yaml_spells_out_op_level_like_other_defaults() ) # Normalization materializes every schema default into the candidate; forward_model is no exception. - assert prediction["engine"]["workers"]["aggregated"]["timing"] == {"type": "default", "forward_model": "op_level"} + assert prediction["engine"]["workers"]["aggregated"]["timing"] == { + "type": "default", + "forward_model": "op_level", + "fpm_interpolation": "auto", + } def _pd_hardware_config(*, recommend=False, **overrides): diff --git a/tests/test_fpm_profile_workflow.py b/tests/test_fpm_profile_workflow.py new file mode 100644 index 0000000000..931b751213 --- /dev/null +++ b/tests/test_fpm_profile_workflow.py @@ -0,0 +1,440 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Class-independent public plumbing; real timing validation is separate.""" + +from __future__ import annotations + +import json +from copy import deepcopy + +import pytest +from pydantic import ValidationError + +from aisimulate.aic import materialize_aic_num_gpu_blocks +from aisimulate.compiler import prediction_to_replay_spec +from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig +from aisimulate.recommend import _candidate_prediction, recommendation_to_sweeper +from aisimulate.support.fpm import fpm_cli_args +from aisimulate.support.plan import check_plan, create_plan +from aisimulate.support.schema import SupportRequest +from aisimulate.sweeper.deploy import build_backend_deployment +from aisimulate.sweeper.model_hw import parallel_configs_for +from aisimulate.sweeper.replay import ReplaySpec +from aisimulate.sweeper.sample import unroll_sample + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def profile(): + # Deliberately fictional metadata, with declared byte bounds, proves the + # workflow cannot recover resources through an existing model registration. + common = { + "system": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "gemm_quant_mode": "fp8_block", + "moe_quant_mode": "fp8_block", + "fmha_quant_mode": "bfloat16", + "comm_quant_mode": "half", + "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 20 * 1024**3, + "activations_bytes": 1024**3, + "runtime_overhead_bytes": 1024**3, + "comm_overhead_bytes": 1024**3, + "kv_bytes_per_token": 512, + "cache_layout": "linear", + "max_num_tokens": 8192, + "max_batch_size": 8, + "provenance": "Synthetic rank-local bounds for workflow tests; excludes CUDA graph reservation.", + }, + } + return { + "schema_version": 1, + "model": "test-only/UnregisteredMoe", + "model_revision": "workflow-fixture-v1", + "architecture": "UnregisteredMoeForCausalLM", + "context_length": 4096, + "num_experts": 8, + "provenance": "Fictional model, no timing or silicon accuracy claim.", + "deployments": [ + {**deepcopy(common), "tp": 2, "dp": 1, "moe_tp": 2, "moe_ep": 1}, + { + **deepcopy(common), + "tp": 1, + "dp": 2, + "moe_tp": 1, + "moe_ep": 2, + "resources": {**common["resources"], "kv_bytes_per_token": 1024, "weights_bytes": 30 * 1024**3}, + }, + ], + } + + +@pytest.fixture(autouse=True) +def forbid_registered_model(monkeypatch): + from aiconfigurator_core.sdk import engine, memory + from aisimulate import aic, compiler, recommend + from aisimulate.sweeper import model_hw + + def fail(*args, **kwargs): + pytest.fail("class-independent workflow reached model/config construction") + + monkeypatch.setattr(engine, "get_model", fail) + monkeypatch.setattr(memory.KVCacheEstimator, "from_request", fail) + monkeypatch.setattr(memory.NaiveKVCacheEstimator, "from_model_path", fail) + monkeypatch.setattr(aic, "resolve_model_context_length", fail) + monkeypatch.setattr(compiler, "resolve_model_context_length", fail) + monkeypatch.setattr(recommend, "resolve_model_context_length", fail) + monkeypatch.setattr(model_hw, "get_model_config_from_model_path", fail) + monkeypatch.setattr(model_hw, "_estimate_model_weight_bytes", fail) + + +def _worker(*, dep=False): + return { + "parallelism": { + "tensor": 1 if dep else 2, + "attention_data": 2 if dep else 1, + "moe_tensor": 1 if dep else 2, + "moe_expert": 2 if dep else 1, + }, + "scheduler": {"max_batched_tokens": 1024, "max_sequences": 4}, + "timing": {"forward_model": "fpm", "fpm_interpolation": "direct"}, + } + + +def _engine(profile): + return { + "model": profile["model"], + "fpm_profile": profile, + "hardware": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "workers": {"aggregated": _worker()}, + } + + +def test_prediction_materializes_rank_local_capacity_without_a_model(profile): + config = CorePredictionConfig.model_validate({"engine": _engine(profile)}) + spec = prediction_to_replay_spec(config) + args = spec.backend_deployment.agg_engine_args + assert args["max_model_len"] == 4096 + assert args["aic_fpm_interpolation"] == "direct" + assert args["aic_fpm_profile"]["model"] == profile["model"] + diagnostics = {} + lowered = materialize_aic_num_gpu_blocks(args, memory_diagnostics=diagnostics) + assert lowered["num_gpu_blocks"] > 0 + assert diagnostics["source"] == "profile" + reserved = materialize_aic_num_gpu_blocks({**args, "cuda_graph_reserved_bytes": 1024**3}) + assert reserved["num_gpu_blocks"] < lowered["num_gpu_blocks"] + + +def test_disaggregated_transfer_uses_profile_cache_geometry(profile): + engine = _engine(profile) + engine.update( + mode="disaggregated", + workers={"prefill": _worker(dep=True), "decode": _worker()}, + kv_transfer={"bytes_per_token": "auto"}, + ) + spec = prediction_to_replay_spec(CorePredictionConfig.model_validate({"engine": engine})) + assert spec.backend_deployment.prefill_engine_args["kv_transfer_bytes_per_token"] == 1024 + assert spec.backend_deployment.decode_engine_args["kv_transfer_bytes_per_token"] == 1024 + + +@pytest.mark.parametrize("mode,budget,expected", [("agg", 2, 2), ("disagg", 4, 4)]) +def test_profile_candidates_use_only_declared_topologies(profile, mode, budget, expected): + candidates = parallel_configs_for( + profile["model"], + "h200_sxm", + backend="vllm", + backend_version="0.25.1", + deployment_mode=mode, + gpu_budget=budget, + max_num_tokens=1024, + max_batch_size=4, + fpm_profile=profile, + ) + assert len(candidates) == expected + assert all(candidate.total_gpus <= budget for candidate in candidates) + if mode == "agg": + assert {candidate.shape.strategy for candidate in candidates} == {"tp", "dep"} + + +@pytest.mark.parametrize("method", ["auto", "direct"]) +def test_recommendation_preserves_profile_in_exported_prediction(profile, monkeypatch, method): + engine = _engine(profile) + engine["workers"]["aggregated"]["parallelism"] = {"preset": "default"} + engine["workers"]["aggregated"]["timing"]["fpm_interpolation"] = method + engine["mode"] = "aggregated" + source = CoreRecommendationConfig.model_validate( + {"engine": engine, "optimization": {"constraints": {"max_candidate_gpus": 2}}} + ) + smart = recommendation_to_sweeper(source) + candidates = parallel_configs_for( + profile["model"], + "h200_sxm", + backend="vllm", + backend_version="0.25.1", + deployment_mode="agg", + gpu_budget=2, + max_num_tokens=1024, + max_batch_size=4, + fpm_profile=profile, + ) + sample = unroll_sample( + search_space=smart.search_space, + selection={ + "deployment_mode": "agg", + "backend": "vllm", + "agg_max_num_batched_tokens": 1024, + "agg_max_num_seqs": 4, + }, + parallel_config=candidates[1], + ) + sample["backend_version"] = "0.25.1" + assert sample["agg_fpm_interpolation"] == "direct" + from aiconfigurator_core.sdk.models.base import _MODEL_REGISTRY + + # A fresh environment may now have an analytical class for the same model. + # Export must retain the method actually selected for the evaluated sample. + monkeypatch.setitem(_MODEL_REGISTRY, profile["architecture"], object()) + replay = ReplaySpec( + backend_deployment=build_backend_deployment(sample, backend_version="0.25.1"), workload={}, goal={} + ) + exported = _candidate_prediction(source, sample, replay, adapter_sections={}) + reloaded = CorePredictionConfig.model_validate(exported) + assert reloaded.engine.fpm_profile == source.engine.fpm_profile + assert reloaded.engine.workers.aggregated.timing.fpm_interpolation == "direct" + assert reloaded.engine.workers.aggregated.parallelism.attention_data == 2 + args = prediction_to_replay_spec(reloaded).backend_deployment.agg_engine_args + assert materialize_aic_num_gpu_blocks(args)["num_gpu_blocks"] > 0 + + +@pytest.mark.parametrize( + "update,error", + [ + ({"model": "another/model"}, "match"), + ({"backend_version": None}, "literal"), + ({"context_length": 8192}, "exceeds"), + ({"fpm_profile": None}, "requires engine.fpm_profile"), + ({"workers": {"aggregated": {**_worker(), "scheduler": {"max_sequences": 32}}}}, "envelope exceeded"), + ({"workers": {"aggregated": {**_worker(), "parallelism": {"tensor": 4, "moe_tensor": 4}}}}, "no matching"), + ], +) +def test_invalid_profile_predictions_fail_before_runtime(profile, update, error): + with pytest.raises(ValidationError, match=error): + CorePredictionConfig.model_validate({"engine": {**_engine(profile), **update}}) + + +def test_onboard_dep_keeps_full_identity_and_checks_resources_before_collection(profile, tmp_path): + request = SupportRequest.model_validate( + { + "identity": { + "model": profile["model"], + "model_revision": profile["model_revision"], + "model_kind": "moe", + "framework_version": "0.25.1", + "gpu": "h200_sxm", + "gpu_count": 2, + "gpus_per_node": 2, + "interconnect": "NVLink", + }, + "search": { + "tensor_parallel": 1, + "attention_data_parallel": 2, + "moe_tensor_parallel": 1, + "moe_expert_parallel": 2, + "context_length": 4096, + }, + "fpm_profile": profile, + } + ) + plan = create_plan(request, tmp_path) + assert plan["fpm"]["worker_gpus"] == 2 + assert plan["search"]["candidates"][0]["total_gpus"] == 2 + assert plan["resources"]["source"] == "profile" + assert plan["resources"]["total_kv_size_tokens"] > 4096 + loaded = CorePredictionConfig.from_yaml(tmp_path / "predict/pilot.yaml") + assert loaded.engine.workers.aggregated.parallelism.attention_data == 2 + assert loaded.engine.workers.aggregated.timing.fpm_interpolation == "direct" + command = fpm_cli_args(request, output_dir=tmp_path, plan_only=True) + assert command[command.index("--fpm-parallel-presets") + 1] == "dep" + assert command[command.index("--fpm-gpu-counts") + 1] == "2" + assert command[command.index("--fpm-kv-cache-dtypes") + 1] == "fp8" + profile_path = tmp_path / "fpm-model-profile.json" + assert command[command.index("--fpm-model-profile") + 1] == str(profile_path) + tampered = json.loads(profile_path.read_text()) + tampered["deployments"][0]["fmha_quant_mode"] = "fp8" + profile_path.write_text(json.dumps(tampered)) + with pytest.raises(ValueError, match="differs from the requested identity"): + check_plan(request, tmp_path) + + +@pytest.mark.parametrize( + "preset,max_tokens,max_sequences,moe_backend", + [ + ("dep", 4096, 8, "auto"), + ("dep", 8192, 4, "auto"), + ("dep", 8192, 8, "flashinfer_cutlass"), + ("tep", 4096, 4, "flashinfer_cutlass"), + ], +) +def test_generated_glm_collector_plan_preserves_backend_and_rank_local_bounds( + profile, tmp_path, monkeypatch, capsys, preset, max_tokens, max_sequences, moe_backend +): + from collector.fpm_forward import cli as collector_cli + + # Real GLM identity, with synthetic resource bounds: this exercises the + # generated command and collector admission without claiming GPU accuracy. + profile.update( + model="nvidia/GLM-5.2-NVFP4", + architecture="GlmMoeDsaForCausalLM", + context_length=8192, + num_experts=256, + provenance="Real GLM identity with synthetic resource bounds for guided collection tests.", + ) + deployment = profile["deployments"][0] + deployment.update( + system="b200_sxm", + tp=1 if preset == "dep" else 8, + dp=8 if preset == "dep" else 1, + moe_tp=1, + moe_ep=8, + gemm_quant_mode="nvfp4", + moe_quant_mode="nvfp4", + fmha_quant_mode="fp8", + moe_backend=moe_backend, + ) + deployment["resources"].update(max_num_tokens=max_tokens, max_batch_size=max_sequences) + profile["deployments"] = [deployment] + request = SupportRequest.model_validate( + { + "identity": { + "model": profile["model"], + "model_revision": profile["model_revision"], + "model_kind": "moe", + "framework_version": "0.25.1", + "gpu": "b200_sxm", + "gpu_count": 8, + "gpus_per_node": 8, + "interconnect": "NVLink", + }, + "search": { + "tensor_parallel": deployment["tp"], + "attention_data_parallel": deployment["dp"], + "moe_tensor_parallel": 1, + "moe_expert_parallel": 8, + "context_length": 8192, + }, + "workload": {"input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "request_count": 8}, + "fpm_profile": profile, + } + ) + plan = create_plan(request, tmp_path) + command = json.loads((tmp_path / "commands.json").read_text())["fpm_plan_local"] + monkeypatch.delenv("COLLECTOR_MODEL_PATH", raising=False) + assert collector_cli.main(command[3:]) == 0 + collected = json.loads(capsys.readouterr().out) + assert collected["counts"]["cells"] == 2 + assert {cell["parallel_strategy"] for cell in collected["cells"]} == {preset} + assert collected["options"]["moe_backend"] == moe_backend + sampling = collected["options"]["prefill_sampling"] + expected_tokens = min(max_tokens, 1024 * max_sequences) + assert sampling["max_total_prefill_tokens"] == expected_tokens + assert sampling["max_batch_size"] == max_sequences + assert collected["topology_memory_admission"][0]["activation_envelope"]["scope"] == "rank_local" + prediction = CorePredictionConfig.from_yaml(tmp_path / "predict/pilot.yaml") + scheduler = prediction.engine.workers.aggregated.scheduler + assert scheduler.max_batched_tokens == max_tokens + assert scheduler.max_sequences == max_sequences + assert prediction.traffic.load.concurrency == 8 + assert prediction.traffic.stop.requests == 8 + assert "timing coverage" in plan["fpm"]["sampling"] + + +def test_onboard_rejects_profile_below_collector_minimum_token_axis(profile, tmp_path): + profile["deployments"][0]["resources"]["max_num_tokens"] = 1 + request = SupportRequest.model_validate( + { + "identity": { + "model": profile["model"], + "model_revision": profile["model_revision"], + "model_kind": "moe", + "framework_version": "0.25.1", + "gpu": "h200_sxm", + "gpu_count": 2, + "gpus_per_node": 2, + "interconnect": "NVLink", + }, + "search": {"tensor_parallel": 2, "context_length": 4096}, + "fpm_profile": profile, + } + ) + with pytest.raises(ValueError, match="FPM collection requires.*at least 2"): + create_plan(request, tmp_path) + + +@pytest.mark.parametrize( + "tp,dp,moe_tp,moe_ep,preset", [(2, 1, 2, 1, "pure_tp"), (1, 2, 1, 2, "dep"), (2, 1, 1, 2, "tep")] +) +def test_onboard_cli_embeds_profile_and_preserves_topology(profile, tmp_path, tp, dp, moe_tp, moe_ep, preset): + import aisimulate.main as cli + + profile["deployments"].append({**deepcopy(profile["deployments"][0]), "moe_tp": 1, "moe_ep": 2}) + source = tmp_path / "model-profile.json" + source.write_text(json.dumps(profile)) + request_path = tmp_path / "request.yaml" + assert ( + cli.main( + [ + "onboard", + "init", + "--model", + profile["model"], + "--model-revision", + profile["model_revision"], + "--model-kind", + "moe", + "--framework-version", + "0.25.1", + "--gpu", + "h200_sxm", + "--gpu-count", + "2", + "--interconnect", + "NVLink", + "--context-length", + "4096", + "--fpm-profile", + str(source), + "--tensor-parallel", + str(tp), + "--attention-data-parallel", + str(dp), + "--moe-tensor-parallel", + str(moe_tp), + "--moe-expert-parallel", + str(moe_ep), + "--output", + str(request_path), + ] + ) + == 0 + ) + source.unlink() + request = SupportRequest.from_yaml(request_path) + assert request.fpm_profile.model == profile["model"] + assert request.worker_gpus == 2 + assert request.parallel_preset == preset + assert request.profile_deployment().parallel_tuple == (tp, 1, dp, moe_tp, moe_ep, 1) + plan = tmp_path / "plan" + assert cli.main(["onboard", "plan", "--config", str(request_path), "--output-dir", str(plan)]) == 0 + prediction = CorePredictionConfig.from_yaml(plan / "predict/pilot.yaml") + assert prediction.engine.fpm_profile == request.fpm_profile + profile_output = plan / "fpm-model-profile.json" + original = profile_output.read_bytes() + profile_output.unlink() + create_plan(request, plan, overwrite=True) + assert profile_output.read_bytes() == original From 081eb51024ac1de9582c5ac1c613dfb55565bb70 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Thu, 17 Sep 2026 06:15:01 +0800 Subject: [PATCH 13/27] fix: preserve FPM profile identity across SDK namespaces Signed-off-by: Yiming Liu --- .../src/aiconfigurator/sdk/fpm_profile.py | 8 ++ .../src/aisimulate_core/sdk/fpm_profile.py | 9 +++ .../cross_package/test_import_contract.py | 78 +++++++++++++++++++ .../unit/sdk/database/test_attention_lanes.py | 6 +- 4 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 python/aisimulate/src/aiconfigurator/sdk/fpm_profile.py create mode 100644 python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py diff --git a/python/aisimulate/src/aiconfigurator/sdk/fpm_profile.py b/python/aisimulate/src/aiconfigurator/sdk/fpm_profile.py new file mode 100644 index 0000000000..e723245a56 --- /dev/null +++ b/python/aisimulate/src/aiconfigurator/sdk/fpm_profile.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility alias for aiconfigurator_core.sdk.fpm_profile.""" + +from aiconfigurator.sdk._compat import alias_module as _alias_module + +_alias_module(__name__, "aiconfigurator_core.sdk.fpm_profile") diff --git a/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py b/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py new file mode 100644 index 0000000000..09a96fa8d6 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Identity-preserving alias for the migrated FPM profiles.""" + +import sys as _sys +from importlib import import_module as _import_module + +_sys.modules[__name__] = _import_module("aiconfigurator_core.sdk.fpm_profile") diff --git a/python/aisimulate/tests/cross_package/test_import_contract.py b/python/aisimulate/tests/cross_package/test_import_contract.py index 76fb26fc6e..e8976c3bc3 100644 --- a/python/aisimulate/tests/cross_package/test_import_contract.py +++ b/python/aisimulate/tests/cross_package/test_import_contract.py @@ -10,6 +10,7 @@ import importlib import importlib.resources import sys +from pathlib import Path from unittest.mock import patch import pytest @@ -30,6 +31,7 @@ "engine", "engine_table_view", "errors", + "fpm_profile", "inference_summary", "memory", "models.base", @@ -126,6 +128,82 @@ def test_legacy_leaf_module_is_canonical_module(module_suffix: str) -> None: assert sys.modules[legacy_name] is sys.modules[canonical_name] +@pytest.mark.parametrize("namespace", ["aiconfigurator.sdk", "aisimulate_core.sdk"]) +def test_fpm_profile_alias_preserves_module_and_type_identity(namespace: str) -> None: + """All public profile imports must share the canonical classes and state.""" + alias_name = f"{namespace}.fpm_profile" + alias = importlib.import_module(alias_name) + canonical = importlib.import_module("aiconfigurator_core.sdk.fpm_profile") + + assert alias is canonical + assert sys.modules[alias_name] is canonical + for name in ("FpmModelProfile", "FpmDeploymentProfile", "FpmResourceProfile"): + assert getattr(alias, name) is getattr(canonical, name) + + +@pytest.mark.parametrize("namespace", ["aiconfigurator.sdk", "aisimulate_core.sdk"]) +def test_fpm_profile_alias_instances_load_and_compile(namespace: str, tmp_path: Path) -> None: + """Profiles created through either facade must reach native compilation.""" + from aiconfigurator_core.sdk.engine import compile_engine + from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile, load_fpm_profile + + alias = importlib.import_module(f"{namespace}.fpm_profile") + profile = alias.FpmModelProfile.model_validate( + { + "schema_version": 1, + "model": "test/unknown-decoder", + "model_revision": "import-contract-fixture-v1", + "architecture": "ImportContractDecoderForCausalLM", + "context_length": 4096, + "num_experts": 0, + "provenance": "Synthetic metadata for import compatibility; not a GPU measurement.", + "deployments": [ + { + "system": "test_gpu", + "backend": "vllm", + "backend_version": "0.25.1", + "tp": 1, + "dp": 1, + "moe_tp": 1, + "moe_ep": 1, + "gemm_quant_mode": "fp8", + "moe_quant_mode": "fp8", + "fmha_quant_mode": "fp8", + "comm_quant_mode": "half", + "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 100, + "activations_bytes": 20, + "runtime_overhead_bytes": 30, + "comm_overhead_bytes": 50, + "kv_bytes_per_token": 10, + "cache_layout": "linear", + "max_num_tokens": 8192, + "max_batch_size": 256, + "provenance": "Synthetic rank-local resource bounds; excludes CUDA graphs.", + }, + } + ], + } + ) + + loaded = load_fpm_profile(profile) + assert type(loaded) is FpmModelProfile + assert loaded.model_dump() == profile.model_dump() + + compiled = compile_engine( + profile.model, + "test_gpu", + "vllm", + "0.25.1", + systems_path=str(tmp_path), + forward_model="fpm", + fpm_profile=profile, + ) + assert isinstance(compiled, bytes) + assert compiled + + @pytest.mark.parametrize("package_suffix", ["models", "operations", "speculation"]) def test_legacy_package_reexports_canonical_public_surface(package_suffix: str) -> None: """Package facades preserve child wrappers and export canonical objects.""" diff --git a/python/aisimulate/tests/unit/sdk/database/test_attention_lanes.py b/python/aisimulate/tests/unit/sdk/database/test_attention_lanes.py index c9fe4ca26d..c9e4bd1b83 100644 --- a/python/aisimulate/tests/unit/sdk/database/test_attention_lanes.py +++ b/python/aisimulate/tests/unit/sdk/database/test_attention_lanes.py @@ -243,11 +243,11 @@ def _fetch(database, attribute, *, shared_layer=None): monkeypatch.setattr(_etv, "fetch_attention_lane_density", _fetch) -def test_engine_spec_schema_version_is_eighteen(): - """Verification width fields extend the positional operation payload.""" +def test_engine_spec_schema_version_is_nineteen(): + """FpmForward interpolation extends the positional operation payload.""" from aiconfigurator.sdk import engine - assert engine.ENGINE_SPEC_SCHEMA_VERSION == 18 + assert engine.ENGINE_SPEC_SCHEMA_VERSION == 19 def test_lanes_outside_the_known_vocabulary_stay_reachable(): From cdafa007cce0668850a073b09ee83f49ab63f6aa Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Thu, 17 Sep 2026 06:21:22 +0800 Subject: [PATCH 14/27] ci: assign ownership for FPM profile workflow tests Signed-off-by: Yiming Liu --- .github/codeowners/areas.yaml | 3 +++ CODEOWNERS | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 31131f46e9..4004aa4120 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -32,6 +32,7 @@ areas: - python/aisimulate/ - python/aisimulate/src/aisimulate/aic.py - tests/test_aic.py + - tests/test_fpm_profile_workflow.py - label: sweeper github_team: "@ai-dynamo/aisimulate-sweeper-codeowners" path_globs: @@ -126,6 +127,8 @@ shared: owners: [forward-pass-engine, maintainers] - glob: tests/test_aic.py owners: [forward-pass-engine, maintainers] + - glob: tests/test_fpm_profile_workflow.py + owners: [forward-pass-engine, maintainers] - glob: python/aisimulate/src/aisimulate/sweeper/ owners: [sweeper, maintainers] - glob: docs/sweeper/ diff --git a/CODEOWNERS b/CODEOWNERS index 00df34ef33..6a30e917a4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -44,6 +44,7 @@ /crates/tests/public-api/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners /crates/core/src/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners /crates/core/tests/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners +/tests/test_fpm_profile_workflow.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners /crates/core/parity_tests/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners /python/aisimulate/src/aisimulate/aic.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners @@ -164,6 +165,7 @@ /crates/tests/public-api/Cargo.toml @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain /python/aisimulate/.coderabbit.yaml @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain /python/aisimulate/eslint.config.js @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain +/tests/test_fpm_profile_workflow.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/access-aisimulate-maintain /crates/core/parity_tests/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/access-aisimulate-maintain /tests/test_ci_workflow_contracts.py @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain /tests/test_e2e_accuracy_overview.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain From fd2d20f57e46acb3476454a1d0255fc4a58798b0 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Fri, 18 Sep 2026 23:17:10 +0800 Subject: [PATCH 15/27] Keep FPM configuration parsing inside the lightweight supervision boundary Signed-off-by: Yiming Liu --- .../src/aiconfigurator_core/sdk/common.py | 101 +------ .../aiconfigurator_core/sdk/fpm_profile.py | 265 +---------------- .../src/aisimulate/config/engine.py | 2 +- .../aisimulate/src/aisimulate/fpm_profile.py | 266 ++++++++++++++++++ .../aisimulate/src/aisimulate/quantization.py | 100 +++++++ .../src/aisimulate/support/schema.py | 2 +- .../cross_package/test_import_contract.py | 26 ++ .../tests/unit/test_fpm_config_imports.py | 177 ++++++++++++ tests/test_unified_traffic_runtime.py | 28 +- 9 files changed, 610 insertions(+), 357 deletions(-) create mode 100644 python/aisimulate/src/aisimulate/fpm_profile.py create mode 100644 python/aisimulate/src/aisimulate/quantization.py create mode 100644 python/aisimulate/tests/unit/test_fpm_config_imports.py diff --git a/python/aisimulate/src/aiconfigurator_core/sdk/common.py b/python/aisimulate/src/aiconfigurator_core/sdk/common.py index 27eeae29fc..a8535f0a5f 100644 --- a/python/aisimulate/src/aiconfigurator_core/sdk/common.py +++ b/python/aisimulate/src/aiconfigurator_core/sdk/common.py @@ -4,7 +4,6 @@ import csv import json import math -from collections import namedtuple from dataclasses import dataclass, field from enum import Enum, StrEnum from functools import cache @@ -13,6 +12,12 @@ from packaging.version import InvalidVersion, Version from aiconfigurator_core.sdk.errors import MissingSystemFlopsError +from aisimulate.quantization import CommQuantMode as CommQuantMode +from aisimulate.quantization import FMHAQuantMode as FMHAQuantMode +from aisimulate.quantization import GEMMQuantMode as GEMMQuantMode +from aisimulate.quantization import KVCacheQuantMode as KVCacheQuantMode +from aisimulate.quantization import MoEQuantMode as MoEQuantMode +from aisimulate.quantization import QuantMapping as QuantMapping def parse_support_matrix_version(version: str | None) -> Version | None: @@ -1376,100 +1381,6 @@ class PerfDataFilename(Enum): fpm_forward = "fpm_forward_perf.parquet" -# compute_dtype names the tensor-core pipeline the mode's MMA actually executes -# on ("bfloat16" | "int8" | "fp8" | "fp4"), i.e. which _tc_flops entry of -# the system YAML governs its SOL math. Weight-only modes (int8_wo, int4_wo, -# w4a16_*) dequantize to bf16 before the MMA, so they map to "bfloat16". None -# marks memory-only modes (KV cache, comm) that never query FLOPS. -QuantMapping = namedtuple("QuantMapping", ["memory", "compute", "name", "compute_dtype"]) - - -class GEMMQuantMode(Enum): - """ - GEMM quant mode. - """ - - bfloat16 = QuantMapping(2, 1, "bfloat16", "bfloat16") # w16a16 - int8_wo = QuantMapping(1, 1, "int8_wo", "bfloat16") # w8a16 - int4_wo = QuantMapping(0.5, 1, "int4_wo", "bfloat16") # w4a16 - fp8 = QuantMapping(1, 2, "fp8", "fp8") # w8fp8 - fp8_static = QuantMapping( - 1, 2, "fp8_static", "fp8" - ) # fp8 with static quantization (compute_scale/scale_matrix modeled) - sq = QuantMapping(1, 2, "sq", "int8") # w8int8 - fp8_block = QuantMapping(1, 2, "fp8_block", "fp8") # specific for trtllm torch ds fp8 - fp8_ootb = QuantMapping( - 1, 2, "fp8_ootb", "fp8" - ) # in future, should deprecate this mode as it's specific for trtllm trt backend - nvfp4 = QuantMapping(9 / 16, 4, "nvfp4", "fp4") # nvfp4 on blackwell. 1 fp8 scale per 16 nvfp4 weights. - nvfp4_wo = QuantMapping(9 / 16, 1, "nvfp4_wo", "bfloat16") # nvfp4 sw dequant to bf16 (non-Blackwell) - w4a16_nvfp4 = QuantMapping( - 9 / 16, 1, "w4a16_nvfp4", "bfloat16" - ) # NVFP4 weights + 1 fp8 scale per 16 weights, dequantized into the bf16 MMA lane. - - -class MoEQuantMode(Enum): - """ - MoE quant mode. - """ - - bfloat16 = QuantMapping(2, 1, "bfloat16", "bfloat16") # w16a16 - fp8 = QuantMapping(1, 2, "fp8", "fp8") # w8fp8 - int4_wo = QuantMapping(0.5, 1, "int4_wo", "bfloat16") # w4a16 - fp8_block = QuantMapping(1, 2, "fp8_block", "fp8") # specific for trtllm torch ds fp8 - w4afp8 = QuantMapping(0.5, 2, "w4afp8", "fp8") # specific for trtllm torch ds w4a8 - nvfp4 = QuantMapping(9 / 16, 4, "nvfp4", "fp4") # nvfp4 on blackwell. 1 fp8 scale per 16 nvfp4 weights. - nvfp4_wo = QuantMapping(9 / 16, 1, "nvfp4_wo", "bfloat16") # nvfp4 sw dequant to bf16 (non-Blackwell) - w4a16_mxfp4 = QuantMapping(0.5, 1, "w4a16_mxfp4", "bfloat16") # native data format for gpt oss - w4a8_mxfp4_mxfp8 = QuantMapping(0.5, 2, "w4a8_mxfp4_mxfp8", "fp8") - # mxfp4 weights, mxfp8 activations (recommended for Blackwell) - w4a8_mxfp4_mxfp8_trtllm = QuantMapping(0.5, 2, "w4a8_mxfp4_mxfp8_trtllm", "fp8") - # Blackwell trtllm-gen fused MoE: MXFP4 (E2M1, block-32) weights x MXFP8 (E4M3) - # activations -- the kernel DeepSeek-V4-Pro actually runs in prefill on sm100 - # (bmm_MxE4m3_MxE2m1MxE4m3 ... sm100f, flashinfer trtllm_fp4_block_scale_moe). - # Distinct backend from w4a8_mxfp4_mxfp8 above (flashinfer cutedsl). DSV4 MoE - # weights are stored MXFP4 (I8-packed E2M1 + E8M0 scales), so sglang dispatches - # by GPU: sm100 -> this (trtllm-gen); sm90 -> w4a16_mxfp4_cutlass below. - w4a16_mxfp4_cutlass = QuantMapping(0.5, 1, "w4a16_mxfp4_cutlass", "bfloat16") - # Hopper (sm90) DeepSeek-V4-Pro MoE: flashinfer cutlass SM90 mixed GEMM - # (cutlass_fused_moe(use_w4_group_scaling=True)) -- MXFP4 weights x BF16 - # activations (weight-only). Distinct backend from w4a16_mxfp4 above, which is - # GPT-OSS's triton_kernels mxfp4 path. (DSV4 Hopper silicon data pending.) - w4a16_nvfp4 = QuantMapping(9 / 16, 1, "w4a16_nvfp4", "bfloat16") - # Scale-aware NVFP4 weights dequantized into the BF16 MoE compute lane. - - -class FMHAQuantMode(Enum): - """ - FMHA quant mode. - """ - - bfloat16 = QuantMapping(2, 1, "bfloat16", "bfloat16") - float16 = QuantMapping(2, 1, "float16", "bfloat16") # sglang decode attention uses float16 compute - fp8 = QuantMapping(1, 2, "fp8", "fp8") - fp8_block = QuantMapping(1, 2, "fp8_block", "fp8") # FIXME: specific for sglang wideep - - -class KVCacheQuantMode(Enum): - """ - KVCache quant mode. - """ - - bfloat16 = QuantMapping(2, 0, "bfloat16", None) - int8 = QuantMapping(1, 0, "int8", None) - fp8 = QuantMapping(1, 0, "fp8", None) - - -class CommQuantMode(Enum): - """ - Comm quant mode. - """ - - half = QuantMapping(2, 0, "half", None) - int8 = QuantMapping(1, 0, "int8", None) - fp8 = QuantMapping(1, 0, "fp8", None) - - # compute_dtype -> system YAML key. A key's absence from the YAML means the # platform has no hardware for that dtype (see get_quant_tc_flops). _COMPUTE_DTYPE_TO_FLOPS_KEY = { diff --git a/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py b/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py index 7934af5dcb..4f32adaf8c 100644 --- a/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py +++ b/python/aisimulate/src/aiconfigurator_core/sdk/fpm_profile.py @@ -1,269 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Explicit identity and resource bounds for decoder-only FPM simulation. - -Profiles describe every rank of a deployment without constructing operations. -Resource values are declarations supplied by the caller, not GPU measurements -or automatically inferred checkpoint metadata. Their provenance must say how -they were obtained; CUDA graph reservations are supplied separately at runtime. -""" +"""FPM profile compatibility exports and runtime-facing validation bridges.""" from __future__ import annotations import json -from typing import Annotated, Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator - -from aiconfigurator_core.sdk import common - -_PositiveInt = Annotated[int, Field(gt=0)] -_Bytes = Annotated[int, Field(ge=0, le=2**53)] -_Nonempty = Annotated[str, Field(min_length=1, pattern=r"\S")] -_MUTABLE_REFERENCES = {"main", "master", "head", "latest", "current", "unknown", "tbd", "unpinned"} - - -class _ProfileModel(BaseModel): - model_config = ConfigDict(extra="forbid", strict=True) - - -class FpmResourceProfile(_ProfileModel): - """Conservative rank-local bounds at a declared scheduler envelope. - - ``max_num_tokens`` and ``max_batch_size`` apply to each attention-DP rank, - not the summed worker iteration used to query FPM timings. Overheads must - exclude the separately configured CUDA graph reservation. - """ - - weights_bytes: _Bytes - activations_bytes: _Bytes - runtime_overhead_bytes: _Bytes - comm_overhead_bytes: _Bytes - kv_bytes_per_token: Annotated[int, Field(gt=0, le=2**53)] - cache_layout: Literal["linear"] - max_num_tokens: _PositiveInt - max_batch_size: _PositiveInt - provenance: _Nonempty - - @model_validator(mode="after") - def _exact_total(self) -> FpmResourceProfile: - if self.non_kv_bytes > 2**53: - raise ValueError("total non-KV resource bytes must not exceed 2**53") - return self - - @property - def non_kv_bytes(self) -> int: - return self.weights_bytes + self.activations_bytes + self.runtime_overhead_bytes + self.comm_overhead_bytes - - def validate_envelope(self, *, max_num_tokens: int, max_batch_size: int) -> None: - for name, requested in (("max_num_tokens", max_num_tokens), ("max_batch_size", max_batch_size)): - if isinstance(requested, bool) or not isinstance(requested, int) or requested <= 0: - raise ValueError(f"{name} must be a positive integer, got {requested!r}") - limit = getattr(self, name) - if requested > limit: - raise ValueError( - f"FPM resource envelope exceeded: requested rank-local {name}={requested}, " - f"profile bound={limit}; provide resource bounds for the larger scheduler envelope" - ) - - -class FpmDeploymentProfile(_ProfileModel): - """One exact hardware, runtime, topology, precision and resource identity.""" - - system: _Nonempty - backend: Literal["vllm"] - backend_version: _Nonempty - tp: _PositiveInt - pp: Literal[1] = 1 - dp: _PositiveInt - moe_tp: _PositiveInt - moe_ep: _PositiveInt - cp: Literal[1] = 1 - gemm_quant_mode: _Nonempty - moe_quant_mode: _Nonempty - fmha_quant_mode: _Nonempty - comm_quant_mode: _Nonempty - kv_cache_dtype: _Nonempty - moe_backend: _Nonempty = "auto" - attention_backend: _Nonempty = "auto" - enable_wideep: Literal[False] = False - enable_eplb: Literal[False] = False - resources: FpmResourceProfile - - @field_validator("pp", "cp", mode="before") - @classmethod - def _integer_dimensions(cls, value: Any) -> int: - if type(value) is not int: - raise ValueError("parallel dimensions must be integers, not booleans or floats") - return value - - @field_validator("enable_wideep", "enable_eplb", mode="before") - @classmethod - def _boolean_flags(cls, value: Any) -> bool: - if type(value) is not bool: - raise ValueError("backend feature flags must be booleans") - return value - - @field_validator("backend_version") - @classmethod - def _literal_version(cls, value: str) -> str: - if value.lower() in _MUTABLE_REFERENCES | {"previous", "next"} or any(c.isspace() for c in value): - raise ValueError("FPM profile backend_version must be a literal runtime version") - return value - - @model_validator(mode="after") - def _deployment_identity(self) -> FpmDeploymentProfile: - modes = { - "gemm_quant_mode": common.GEMMQuantMode, - "moe_quant_mode": common.MoEQuantMode, - "fmha_quant_mode": common.FMHAQuantMode, - "comm_quant_mode": common.CommQuantMode, - "kv_cache_dtype": common.KVCacheQuantMode, - } - for name, enum in modes.items(): - if getattr(self, name) not in enum.__members__: - raise ValueError(f"unknown FPM {name}: {getattr(self, name)!r}") - return self - - @property - def parallel_tuple(self) -> tuple[int, int, int, int, int, int]: - return self.tp, self.pp, self.dp, self.moe_tp, self.moe_ep, self.cp - - def match_identity(self) -> list[str]: - """Return the exact 15-column native FPM identity, with no aliasing.""" - return [ - self.gemm_quant_mode, - self.moe_quant_mode, - self.fmha_quant_mode, - self.comm_quant_mode, - self.kv_cache_dtype, - *(str(value) for value in self.parallel_tuple), - self.moe_backend, - self.attention_backend, - str(self.enable_wideep), - str(self.enable_eplb), - ] - - def quantization_kwargs(self) -> dict[str, str]: - return { - "gemm_quant_mode": self.gemm_quant_mode, - "moe_quant_mode": self.moe_quant_mode, - "fmha_quant_mode": self.fmha_quant_mode, - "comm_quant_mode": self.comm_quant_mode, - "kvcache_quant_mode": self.kv_cache_dtype, - "attention_backend": self.attention_backend, - } - - def validate_overrides(self, **overrides: str | None) -> None: - for name, expected in self.quantization_kwargs().items(): - supplied = overrides.get(name) - if supplied is not None and supplied != expected: - raise ValueError( - f"FPM profile identity conflict: {name}={supplied!r}, profile={expected!r}; " - "use matching measured precision and backend settings" - ) - - -class FpmModelProfile(_ProfileModel): - """Serializable FPM metadata independent of an analytical model class.""" - - schema_version: Literal[1] - model: _Nonempty - model_revision: _Nonempty - architecture: _Nonempty - context_length: _PositiveInt - num_experts: Annotated[int, Field(ge=0)] - provenance: _Nonempty - deployments: Annotated[list[FpmDeploymentProfile], Field(min_length=1)] - - @field_validator("schema_version", mode="before") - @classmethod - def _integer_schema(cls, value: Any) -> int: - if type(value) is not int: - raise ValueError("schema_version must be an integer") - return value - - @field_validator("model_revision") - @classmethod - def _pinned_revision(cls, value: str) -> str: - if value.strip().lower() in _MUTABLE_REFERENCES: - raise ValueError("provide a pinned model_revision, not a mutable or unknown placeholder") - return value - - @model_validator(mode="after") - def _unique_deployments(self) -> FpmModelProfile: - identities = set() - for deployment in self.deployments: - tp = deployment.dp == 1 and deployment.moe_ep == 1 and deployment.moe_tp == deployment.tp - dep = deployment.tp == 1 and deployment.moe_tp == 1 and deployment.moe_ep == deployment.dp - tep = deployment.dp == 1 and deployment.moe_tp == 1 and deployment.moe_ep == deployment.tp - dense_tp = deployment.dp == 1 and deployment.moe_ep == 1 and deployment.moe_tp in (1, deployment.tp) - if not (dense_tp if self.num_experts == 0 else tp or dep or tep): - raise ValueError( - "FPM profile requires dense TP or an MoE TP, DEP, or TEP topology with matching world sizes" - ) - identity = ( - deployment.system, - deployment.backend, - deployment.backend_version, - deployment.parallel_tuple, - ) - if identity in identities: - raise ValueError( - "duplicate FPM deployment identity; specify one precision/resource profile per topology" - ) - identities.add(identity) - if deployment.moe_ep > 1 and (self.num_experts == 0 or self.num_experts % deployment.moe_ep): - raise ValueError("num_experts must be positive and divisible by moe_ep for expert parallelism") - return self - - def select( - self, - *, - model: str, - system: str, - backend: str, - backend_version: str, - tp_size: int = 1, - pp_size: int = 1, - attention_dp_size: int = 1, - moe_tp_size: int | None = None, - moe_ep_size: int | None = None, - cp_size: int = 1, - ) -> FpmDeploymentProfile: - if model != self.model: - raise ValueError(f"FPM profile model identity mismatch: requested {model!r}, profile={self.model!r}") - shape = ( - tp_size, - pp_size, - attention_dp_size, - 1 if moe_tp_size is None else moe_tp_size, - 1 if moe_ep_size is None else moe_ep_size, - cp_size, - ) - if any(type(value) is not int or value <= 0 for value in shape): - raise ValueError("FPM deployment dimensions must be positive integers, not booleans") - for deployment in self.deployments: - if ( - deployment.system == system - and deployment.backend == backend - and deployment.backend_version == backend_version - and deployment.parallel_tuple == shape - ): - return deployment - raise ValueError( - f"no matching FPM deployment profile for {model!r}, {system}/{backend}/{backend_version}, " - f"(tp,pp,dp,moe_tp,moe_ep,cp)={shape}; provide explicit resources and precision for that deployment" - ) - -def load_fpm_profile(value: dict[str, Any] | str | FpmModelProfile) -> FpmModelProfile: - if isinstance(value, FpmModelProfile): - value = value.model_dump(mode="json") - if isinstance(value, str): - value = json.loads(value) - return FpmModelProfile.model_validate(value) +from aisimulate.fpm_profile import _MUTABLE_REFERENCES +from aisimulate.fpm_profile import FpmDeploymentProfile as FpmDeploymentProfile +from aisimulate.fpm_profile import FpmModelProfile as FpmModelProfile +from aisimulate.fpm_profile import FpmResourceProfile as FpmResourceProfile +from aisimulate.fpm_profile import load_fpm_profile as load_fpm_profile def _quantization_from_engine_config(config_json: str) -> dict[str, str]: @@ -314,6 +62,7 @@ def _validate_forward_pass_profile(config_json: str) -> str: This path reads no timing data and does not construct an analytical model. Estimator defaults and interpolation selection belong to Rust. """ + from aiconfigurator_core.sdk import common from aiconfigurator_core.sdk.models.base import _MODEL_REGISTRY config = json.loads(config_json) diff --git a/python/aisimulate/src/aisimulate/config/engine.py b/python/aisimulate/src/aisimulate/config/engine.py index 5a327e2144..db964df012 100644 --- a/python/aisimulate/src/aisimulate/config/engine.py +++ b/python/aisimulate/src/aisimulate/config/engine.py @@ -9,7 +9,7 @@ from pydantic import Field, field_validator, model_validator -from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile +from aisimulate.fpm_profile import FpmModelProfile from .common import Choices, IntegerRange, NumericRange, StrictModel, SystemsPath, requested_backend_version diff --git a/python/aisimulate/src/aisimulate/fpm_profile.py b/python/aisimulate/src/aisimulate/fpm_profile.py new file mode 100644 index 0000000000..634a1969b7 --- /dev/null +++ b/python/aisimulate/src/aisimulate/fpm_profile.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Explicit identity and resource bounds for decoder-only FPM simulation. + +Profiles describe every rank of a deployment without constructing operations. +Resource values are declarations supplied by the caller, not GPU measurements +or automatically inferred checkpoint metadata. Their provenance must say how +they were obtained; CUDA graph reservations are supplied separately at runtime. +""" + +from __future__ import annotations + +import json +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from . import quantization + +_PositiveInt = Annotated[int, Field(gt=0)] +_Bytes = Annotated[int, Field(ge=0, le=2**53)] +_Nonempty = Annotated[str, Field(min_length=1, pattern=r"\S")] +_MUTABLE_REFERENCES = {"main", "master", "head", "latest", "current", "unknown", "tbd", "unpinned"} + + +class _ProfileModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class FpmResourceProfile(_ProfileModel): + """Conservative rank-local bounds at a declared scheduler envelope. + + ``max_num_tokens`` and ``max_batch_size`` apply to each attention-DP rank, + not the summed worker iteration used to query FPM timings. Overheads must + exclude the separately configured CUDA graph reservation. + """ + + weights_bytes: _Bytes + activations_bytes: _Bytes + runtime_overhead_bytes: _Bytes + comm_overhead_bytes: _Bytes + kv_bytes_per_token: Annotated[int, Field(gt=0, le=2**53)] + cache_layout: Literal["linear"] + max_num_tokens: _PositiveInt + max_batch_size: _PositiveInt + provenance: _Nonempty + + @model_validator(mode="after") + def _exact_total(self) -> FpmResourceProfile: + if self.non_kv_bytes > 2**53: + raise ValueError("total non-KV resource bytes must not exceed 2**53") + return self + + @property + def non_kv_bytes(self) -> int: + return self.weights_bytes + self.activations_bytes + self.runtime_overhead_bytes + self.comm_overhead_bytes + + def validate_envelope(self, *, max_num_tokens: int, max_batch_size: int) -> None: + for name, requested in (("max_num_tokens", max_num_tokens), ("max_batch_size", max_batch_size)): + if isinstance(requested, bool) or not isinstance(requested, int) or requested <= 0: + raise ValueError(f"{name} must be a positive integer, got {requested!r}") + limit = getattr(self, name) + if requested > limit: + raise ValueError( + f"FPM resource envelope exceeded: requested rank-local {name}={requested}, " + f"profile bound={limit}; provide resource bounds for the larger scheduler envelope" + ) + + +class FpmDeploymentProfile(_ProfileModel): + """One exact hardware, runtime, topology, precision and resource identity.""" + + system: _Nonempty + backend: Literal["vllm"] + backend_version: _Nonempty + tp: _PositiveInt + pp: Literal[1] = 1 + dp: _PositiveInt + moe_tp: _PositiveInt + moe_ep: _PositiveInt + cp: Literal[1] = 1 + gemm_quant_mode: _Nonempty + moe_quant_mode: _Nonempty + fmha_quant_mode: _Nonempty + comm_quant_mode: _Nonempty + kv_cache_dtype: _Nonempty + moe_backend: _Nonempty = "auto" + attention_backend: _Nonempty = "auto" + enable_wideep: Literal[False] = False + enable_eplb: Literal[False] = False + resources: FpmResourceProfile + + @field_validator("pp", "cp", mode="before") + @classmethod + def _integer_dimensions(cls, value: Any) -> int: + if type(value) is not int: + raise ValueError("parallel dimensions must be integers, not booleans or floats") + return value + + @field_validator("enable_wideep", "enable_eplb", mode="before") + @classmethod + def _boolean_flags(cls, value: Any) -> bool: + if type(value) is not bool: + raise ValueError("backend feature flags must be booleans") + return value + + @field_validator("backend_version") + @classmethod + def _literal_version(cls, value: str) -> str: + if value.lower() in _MUTABLE_REFERENCES | {"previous", "next"} or any(c.isspace() for c in value): + raise ValueError("FPM profile backend_version must be a literal runtime version") + return value + + @model_validator(mode="after") + def _deployment_identity(self) -> FpmDeploymentProfile: + modes = { + "gemm_quant_mode": quantization.GEMMQuantMode, + "moe_quant_mode": quantization.MoEQuantMode, + "fmha_quant_mode": quantization.FMHAQuantMode, + "comm_quant_mode": quantization.CommQuantMode, + "kv_cache_dtype": quantization.KVCacheQuantMode, + } + for name, enum in modes.items(): + if getattr(self, name) not in enum.__members__: + raise ValueError(f"unknown FPM {name}: {getattr(self, name)!r}") + return self + + @property + def parallel_tuple(self) -> tuple[int, int, int, int, int, int]: + return self.tp, self.pp, self.dp, self.moe_tp, self.moe_ep, self.cp + + def match_identity(self) -> list[str]: + """Return the exact 15-column native FPM identity, with no aliasing.""" + return [ + self.gemm_quant_mode, + self.moe_quant_mode, + self.fmha_quant_mode, + self.comm_quant_mode, + self.kv_cache_dtype, + *(str(value) for value in self.parallel_tuple), + self.moe_backend, + self.attention_backend, + str(self.enable_wideep), + str(self.enable_eplb), + ] + + def quantization_kwargs(self) -> dict[str, str]: + return { + "gemm_quant_mode": self.gemm_quant_mode, + "moe_quant_mode": self.moe_quant_mode, + "fmha_quant_mode": self.fmha_quant_mode, + "comm_quant_mode": self.comm_quant_mode, + "kvcache_quant_mode": self.kv_cache_dtype, + "attention_backend": self.attention_backend, + } + + def validate_overrides(self, **overrides: str | None) -> None: + for name, expected in self.quantization_kwargs().items(): + supplied = overrides.get(name) + if supplied is not None and supplied != expected: + raise ValueError( + f"FPM profile identity conflict: {name}={supplied!r}, profile={expected!r}; " + "use matching measured precision and backend settings" + ) + + +class FpmModelProfile(_ProfileModel): + """Serializable FPM metadata independent of an analytical model class.""" + + schema_version: Literal[1] + model: _Nonempty + model_revision: _Nonempty + architecture: _Nonempty + context_length: _PositiveInt + num_experts: Annotated[int, Field(ge=0)] + provenance: _Nonempty + deployments: Annotated[list[FpmDeploymentProfile], Field(min_length=1)] + + @field_validator("schema_version", mode="before") + @classmethod + def _integer_schema(cls, value: Any) -> int: + if type(value) is not int: + raise ValueError("schema_version must be an integer") + return value + + @field_validator("model_revision") + @classmethod + def _pinned_revision(cls, value: str) -> str: + if value.strip().lower() in _MUTABLE_REFERENCES: + raise ValueError("provide a pinned model_revision, not a mutable or unknown placeholder") + return value + + @model_validator(mode="after") + def _unique_deployments(self) -> FpmModelProfile: + identities = set() + for deployment in self.deployments: + tp = deployment.dp == 1 and deployment.moe_ep == 1 and deployment.moe_tp == deployment.tp + dep = deployment.tp == 1 and deployment.moe_tp == 1 and deployment.moe_ep == deployment.dp + tep = deployment.dp == 1 and deployment.moe_tp == 1 and deployment.moe_ep == deployment.tp + dense_tp = deployment.dp == 1 and deployment.moe_ep == 1 and deployment.moe_tp in (1, deployment.tp) + if not (dense_tp if self.num_experts == 0 else tp or dep or tep): + raise ValueError( + "FPM profile requires dense TP or an MoE TP, DEP, or TEP topology with matching world sizes" + ) + identity = ( + deployment.system, + deployment.backend, + deployment.backend_version, + deployment.parallel_tuple, + ) + if identity in identities: + raise ValueError( + "duplicate FPM deployment identity; specify one precision/resource profile per topology" + ) + identities.add(identity) + if deployment.moe_ep > 1 and (self.num_experts == 0 or self.num_experts % deployment.moe_ep): + raise ValueError("num_experts must be positive and divisible by moe_ep for expert parallelism") + return self + + def select( + self, + *, + model: str, + system: str, + backend: str, + backend_version: str, + tp_size: int = 1, + pp_size: int = 1, + attention_dp_size: int = 1, + moe_tp_size: int | None = None, + moe_ep_size: int | None = None, + cp_size: int = 1, + ) -> FpmDeploymentProfile: + if model != self.model: + raise ValueError(f"FPM profile model identity mismatch: requested {model!r}, profile={self.model!r}") + shape = ( + tp_size, + pp_size, + attention_dp_size, + 1 if moe_tp_size is None else moe_tp_size, + 1 if moe_ep_size is None else moe_ep_size, + cp_size, + ) + if any(type(value) is not int or value <= 0 for value in shape): + raise ValueError("FPM deployment dimensions must be positive integers, not booleans") + for deployment in self.deployments: + if ( + deployment.system == system + and deployment.backend == backend + and deployment.backend_version == backend_version + and deployment.parallel_tuple == shape + ): + return deployment + raise ValueError( + f"no matching FPM deployment profile for {model!r}, {system}/{backend}/{backend_version}, " + f"(tp,pp,dp,moe_tp,moe_ep,cp)={shape}; provide explicit resources and precision for that deployment" + ) + + +def load_fpm_profile(value: dict[str, Any] | str | FpmModelProfile) -> FpmModelProfile: + if isinstance(value, FpmModelProfile): + value = value.model_dump(mode="json") + if isinstance(value, str): + value = json.loads(value) + return FpmModelProfile.model_validate(value) diff --git a/python/aisimulate/src/aisimulate/quantization.py b/python/aisimulate/src/aisimulate/quantization.py new file mode 100644 index 0000000000..aa55a5ad01 --- /dev/null +++ b/python/aisimulate/src/aisimulate/quantization.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared quantization identities without performance-model runtime imports.""" + +from collections import namedtuple +from enum import Enum + +# compute_dtype names the tensor-core pipeline the mode's MMA actually executes +# on ("bfloat16" | "int8" | "fp8" | "fp4"), i.e. which _tc_flops entry of +# the system YAML governs its SOL math. Weight-only modes (int8_wo, int4_wo, +# w4a16_*) dequantize to bf16 before the MMA, so they map to "bfloat16". None +# marks memory-only modes (KV cache, comm) that never query FLOPS. +QuantMapping = namedtuple("QuantMapping", ["memory", "compute", "name", "compute_dtype"]) + + +class GEMMQuantMode(Enum): + """ + GEMM quant mode. + """ + + bfloat16 = QuantMapping(2, 1, "bfloat16", "bfloat16") # w16a16 + int8_wo = QuantMapping(1, 1, "int8_wo", "bfloat16") # w8a16 + int4_wo = QuantMapping(0.5, 1, "int4_wo", "bfloat16") # w4a16 + fp8 = QuantMapping(1, 2, "fp8", "fp8") # w8fp8 + fp8_static = QuantMapping( + 1, 2, "fp8_static", "fp8" + ) # fp8 with static quantization (compute_scale/scale_matrix modeled) + sq = QuantMapping(1, 2, "sq", "int8") # w8int8 + fp8_block = QuantMapping(1, 2, "fp8_block", "fp8") # specific for trtllm torch ds fp8 + fp8_ootb = QuantMapping( + 1, 2, "fp8_ootb", "fp8" + ) # in future, should deprecate this mode as it's specific for trtllm trt backend + nvfp4 = QuantMapping(9 / 16, 4, "nvfp4", "fp4") # nvfp4 on blackwell. 1 fp8 scale per 16 nvfp4 weights. + nvfp4_wo = QuantMapping(9 / 16, 1, "nvfp4_wo", "bfloat16") # nvfp4 sw dequant to bf16 (non-Blackwell) + w4a16_nvfp4 = QuantMapping( + 9 / 16, 1, "w4a16_nvfp4", "bfloat16" + ) # NVFP4 weights + 1 fp8 scale per 16 weights, dequantized into the bf16 MMA lane. + + +class MoEQuantMode(Enum): + """ + MoE quant mode. + """ + + bfloat16 = QuantMapping(2, 1, "bfloat16", "bfloat16") # w16a16 + fp8 = QuantMapping(1, 2, "fp8", "fp8") # w8fp8 + int4_wo = QuantMapping(0.5, 1, "int4_wo", "bfloat16") # w4a16 + fp8_block = QuantMapping(1, 2, "fp8_block", "fp8") # specific for trtllm torch ds fp8 + w4afp8 = QuantMapping(0.5, 2, "w4afp8", "fp8") # specific for trtllm torch ds w4a8 + nvfp4 = QuantMapping(9 / 16, 4, "nvfp4", "fp4") # nvfp4 on blackwell. 1 fp8 scale per 16 nvfp4 weights. + nvfp4_wo = QuantMapping(9 / 16, 1, "nvfp4_wo", "bfloat16") # nvfp4 sw dequant to bf16 (non-Blackwell) + w4a16_mxfp4 = QuantMapping(0.5, 1, "w4a16_mxfp4", "bfloat16") # native data format for gpt oss + w4a8_mxfp4_mxfp8 = QuantMapping(0.5, 2, "w4a8_mxfp4_mxfp8", "fp8") + # mxfp4 weights, mxfp8 activations (recommended for Blackwell) + w4a8_mxfp4_mxfp8_trtllm = QuantMapping(0.5, 2, "w4a8_mxfp4_mxfp8_trtllm", "fp8") + # Blackwell trtllm-gen fused MoE: MXFP4 (E2M1, block-32) weights x MXFP8 (E4M3) + # activations -- the kernel DeepSeek-V4-Pro actually runs in prefill on sm100 + # (bmm_MxE4m3_MxE2m1MxE4m3 ... sm100f, flashinfer trtllm_fp4_block_scale_moe). + # Distinct backend from w4a8_mxfp4_mxfp8 above (flashinfer cutedsl). DSV4 MoE + # weights are stored MXFP4 (I8-packed E2M1 + E8M0 scales), so sglang dispatches + # by GPU: sm100 -> this (trtllm-gen); sm90 -> w4a16_mxfp4_cutlass below. + w4a16_mxfp4_cutlass = QuantMapping(0.5, 1, "w4a16_mxfp4_cutlass", "bfloat16") + # Hopper (sm90) DeepSeek-V4-Pro MoE: flashinfer cutlass SM90 mixed GEMM + # (cutlass_fused_moe(use_w4_group_scaling=True)) -- MXFP4 weights x BF16 + # activations (weight-only). Distinct backend from w4a16_mxfp4 above, which is + # GPT-OSS's triton_kernels mxfp4 path. (DSV4 Hopper silicon data pending.) + w4a16_nvfp4 = QuantMapping(9 / 16, 1, "w4a16_nvfp4", "bfloat16") + # Scale-aware NVFP4 weights dequantized into the BF16 MoE compute lane. + + +class FMHAQuantMode(Enum): + """ + FMHA quant mode. + """ + + bfloat16 = QuantMapping(2, 1, "bfloat16", "bfloat16") + float16 = QuantMapping(2, 1, "float16", "bfloat16") # sglang decode attention uses float16 compute + fp8 = QuantMapping(1, 2, "fp8", "fp8") + fp8_block = QuantMapping(1, 2, "fp8_block", "fp8") # FIXME: specific for sglang wideep + + +class KVCacheQuantMode(Enum): + """ + KVCache quant mode. + """ + + bfloat16 = QuantMapping(2, 0, "bfloat16", None) + int8 = QuantMapping(1, 0, "int8", None) + fp8 = QuantMapping(1, 0, "fp8", None) + + +class CommQuantMode(Enum): + """ + Comm quant mode. + """ + + half = QuantMapping(2, 0, "half", None) + int8 = QuantMapping(1, 0, "int8", None) + fp8 = QuantMapping(1, 0, "fp8", None) diff --git a/python/aisimulate/src/aisimulate/support/schema.py b/python/aisimulate/src/aisimulate/support/schema.py index 4c6ed37a78..532a6af58f 100644 --- a/python/aisimulate/src/aisimulate/support/schema.py +++ b/python/aisimulate/src/aisimulate/support/schema.py @@ -12,8 +12,8 @@ from packaging.version import Version from pydantic import Field, field_validator, model_validator -from aiconfigurator_core.sdk.fpm_profile import FpmModelProfile from aisimulate.config.common import PositiveFiniteFloat, PositiveStrictInt, StrictModel, load_yaml +from aisimulate.fpm_profile import FpmModelProfile class SupportIdentity(StrictModel): diff --git a/python/aisimulate/tests/cross_package/test_import_contract.py b/python/aisimulate/tests/cross_package/test_import_contract.py index 416ec3a63a..5473613fca 100644 --- a/python/aisimulate/tests/cross_package/test_import_contract.py +++ b/python/aisimulate/tests/cross_package/test_import_contract.py @@ -10,6 +10,7 @@ import importlib import importlib.resources import json +import pickle import sys from pathlib import Path from unittest.mock import patch @@ -135,11 +136,36 @@ def test_fpm_profile_alias_preserves_module_and_type_identity(namespace: str) -> alias_name = f"{namespace}.fpm_profile" alias = importlib.import_module(alias_name) canonical = importlib.import_module("aiconfigurator_core.sdk.fpm_profile") + lightweight = importlib.import_module("aisimulate.fpm_profile") assert alias is canonical assert sys.modules[alias_name] is canonical for name in ("FpmModelProfile", "FpmDeploymentProfile", "FpmResourceProfile"): assert getattr(alias, name) is getattr(canonical, name) + assert getattr(canonical, name) is getattr(lightweight, name) + legacy_global = f"c{namespace}.fpm_profile\n{name}\n.".encode() + assert pickle.loads(legacy_global) is getattr(lightweight, name) + assert alias.load_fpm_profile is lightweight.load_fpm_profile + + +@pytest.mark.parametrize("namespace", ["aiconfigurator.sdk", "aiconfigurator_core.sdk", "aisimulate_core.sdk"]) +@pytest.mark.parametrize( + "name", ["GEMMQuantMode", "MoEQuantMode", "FMHAQuantMode", "KVCacheQuantMode", "CommQuantMode", "QuantMapping"] +) +def test_quantization_exports_preserve_shared_types_and_pickles(namespace: str, name: str) -> None: + from aisimulate import quantization + + alias = importlib.import_module(f"{namespace}.common") + shared = getattr(quantization, name) + assert getattr(alias, name) is shared + # Historic pickle GLOBAL references resolve through the SDK re-exports. + assert pickle.loads(f"c{namespace}.common\n{name}\n.".encode()) is shared + if name != "QuantMapping": + for member in shared: + assert pickle.loads(pickle.dumps(member)) is member + value = pickle.loads(pickle.dumps(member.value)) + assert type(value) is quantization.QuantMapping + assert value == member.value @pytest.mark.parametrize("namespace", ["aiconfigurator.sdk", "aisimulate_core.sdk"]) diff --git a/python/aisimulate/tests/unit/test_fpm_config_imports.py b/python/aisimulate/tests/unit/test_fpm_config_imports.py new file mode 100644 index 0000000000..ef75e3e5da --- /dev/null +++ b/python/aisimulate/tests/unit/test_fpm_config_imports.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FPM parsing must finish before resource supervision loads the runtime.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + + +def _config() -> dict: + return { + "engine": { + "mode": "aggregated", + "model": "test/unregistered-decoder", + "hardware": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "workers": {"aggregated": {"scheduler": {"max_batched_tokens": 1024, "max_sequences": 4}}}, + "fpm_profile": { + "schema_version": 1, + "model": "test/unregistered-decoder", + "model_revision": "import-boundary-fixture-v1", + "architecture": "UnregisteredDecoderForCausalLM", + "context_length": 4096, + "num_experts": 0, + "provenance": "Synthetic metadata for pre-budget validation; no timing measurements.", + "deployments": [ + { + "system": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "tp": 1, + "dp": 1, + "moe_tp": 1, + "moe_ep": 1, + "gemm_quant_mode": "fp8", + "moe_quant_mode": "fp8", + "fmha_quant_mode": "fp8", + "comm_quant_mode": "half", + "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 100, + "activations_bytes": 20, + "runtime_overhead_bytes": 30, + "comm_overhead_bytes": 50, + "kv_bytes_per_token": 10, + "cache_layout": "linear", + "max_num_tokens": 8192, + "max_batch_size": 256, + "provenance": "Declared bounds for import validation only.", + }, + } + ], + }, + } + } + + +_IMPORT_GUARD = """ +import sys +from pathlib import Path + +blocked = ('aisimulate._runtime', 'aisimulate.sweeper', 'numpy', 'pandas', 'aiconfigurator_core', 'aisimulate_core') + +def assert_lightweight(): + loaded = [name for name in sys.modules if any(name == item or name.startswith(item + '.') for item in blocked)] + assert not loaded, loaded + +class RejectRuntimeImports: + def find_spec(self, fullname, path=None, target=None): + if any(fullname == item or fullname.startswith(item + '.') for item in blocked): + raise AssertionError('pre-budget runtime import: ' + fullname) + +assert_lightweight() +sys.meta_path.insert(0, RejectRuntimeImports()) +""" + +_SUPERVISION_PROBE = ( + _IMPORT_GUARD + + """ +from aisimulate import supervision + +calls = [] +def supervised_child(*args, **kwargs): + assert_lightweight() + calls.append(args) + return {'status': 'completed', 'exit_code': 0} + +supervision.run_process = supervised_child +config, output, command, error = sys.argv[1:] +try: + code = supervision.main([command, '--config', config, '--output-dir', output, '--overwrite']) +except SystemExit as exc: + code = exc.code +assert_lightweight() +if error: + assert code == 2, code + assert not calls, calls + assert (Path(output) / 'existing.txt').read_text() == 'keep existing results' +else: + assert code == 0, code + assert len(calls) == 1, calls + from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig + config_type = CorePredictionConfig if command == 'predict' else CoreRecommendationConfig + parsed = config_type.from_yaml(config) + assert config_type.model_validate_json(parsed.model_dump_json()) == parsed + assert_lightweight() +""" +) + + +@pytest.mark.parametrize("command", ["predict", "recommend"]) +@pytest.mark.parametrize("case", ["no-profile", "profile", "unknown-quant", "boolean-topology", "wrong-model"]) +def test_supervised_profile_validation_is_runtime_free(command: str, case: str, tmp_path: Path) -> None: + """Exercise the real parser and envelope validation before the child starts.""" + raw = _config() + if command == "recommend": + raw["optimization"] = {"constraints": {"max_candidate_gpus": 1}} + error = "" + if case == "no-profile": + del raw["engine"]["fpm_profile"] + elif case == "unknown-quant": + raw["engine"]["fpm_profile"]["deployments"][0]["fmha_quant_mode"] = "unsupported" + error = "unknown FPM fmha_quant_mode" + elif case == "boolean-topology": + raw["engine"]["fpm_profile"]["deployments"][0]["tp"] = True + error = "Input should be a valid integer" + elif case == "wrong-model": + raw["engine"]["model"] = "test/different-decoder" + error = "engine.model must match engine.fpm_profile.model" + config = tmp_path / "config.json" + config.write_text(json.dumps(raw)) + output = tmp_path / "output" + output.mkdir() + (output / "existing.txt").write_text("keep existing results") + + result = subprocess.run( + [sys.executable, "-c", _SUPERVISION_PROBE, str(config), str(output), command, error], + capture_output=True, + text=True, + timeout=15, + ) + assert result.returncode == 0, result.stdout + result.stderr + if error: + assert error in result.stderr + + +def test_onboarding_parser_is_runtime_free() -> None: + result = subprocess.run( + [ + sys.executable, + "-c", + _IMPORT_GUARD + + """ +from aisimulate.cli_args import build_parser +parser = build_parser() +try: + parser.parse_args(['onboard', 'init', '--help']) +except SystemExit as exc: + assert exc.code == 0 +assert_lightweight() +""", + ], + capture_output=True, + text=True, + timeout=15, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "--model" in result.stdout diff --git a/tests/test_unified_traffic_runtime.py b/tests/test_unified_traffic_runtime.py index 92b0564fe2..46719d89f0 100644 --- a/tests/test_unified_traffic_runtime.py +++ b/tests/test_unified_traffic_runtime.py @@ -449,14 +449,38 @@ def test_prediction_spec_lowers_fpm_forward_model_onto_the_rank() -> None: assert deployment.performance_model_metadata["aggregated"]["config"]["forward_model"] == "fpm" -def test_prediction_spec_omits_the_forward_model_rank_field_for_op_level() -> None: +@pytest.mark.parametrize("explicit_mode", [False, True]) +def test_prediction_spec_omits_the_forward_model_rank_field_for_op_level(monkeypatch, explicit_mode: bool) -> None: + from aiconfigurator_core.sdk import RustForwardPassPerfModel + + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") engine = _fpm_engine() engine["workers"]["aggregated"]["timing"] = {"type": "default"} + if explicit_mode: + engine["workers"]["aggregated"]["timing"]["estimation_mode"] = "op_level" parsed = CorePredictionConfig.model_validate({"engine": engine}) deployment = prediction_to_replay_spec(parsed).backend_deployment assert "aic_forward_model" not in deployment.agg_engine_args - assert deployment.performance_model_metadata["aggregated"]["config"]["forward_model"] == "op_level" + config = deployment.performance_model_metadata["aggregated"]["config"] + requested_mode = "op_level" if explicit_mode else "auto" + assert "forward_model" not in config + assert config["estimation_mode"] == requested_mode + assert config["model"] == engine["model"] + assert config["system"] == engine["hardware"] + assert config["backend_version"] == engine["backend_version"] + assert config["worker_type"] == "aggregated" + assert config["tp"] == config["moe_tp_size"] == 4 + model = RustForwardPassPerfModel.best_available(config) + try: + provenance = model.diagnostics()["provenance"] + finally: + model.close() + assert provenance["requested_estimation_mode"] == requested_mode + assert provenance["selected_estimation_mode"] == "op_level" + assert provenance["config"]["estimation_mode"] == "op_level" + for field in ("model", "system", "backend", "backend_version", "worker_type", "tp", "moe_tp_size"): + assert provenance["config"][field] == config[field] def test_prediction_spec_lowers_forward_model_per_role_in_disaggregated_mode() -> None: From ae44d00011adc201b5d2f140194cb2f4b7b07932 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Tue, 22 Sep 2026 14:14:44 +0800 Subject: [PATCH 16/27] fix: verify profile runtime versions in Slurm collection Signed-off-by: Yiming Liu --- .../collector/fpm_forward/runner.py | 76 +++++++------ .../aisimulate/collector/fpm_forward/slurm.py | 32 +++--- .../tests/unit/collector/test_fpm_slurm.py | 100 ++++++++++++++++++ 3 files changed, 161 insertions(+), 47 deletions(-) diff --git a/python/aisimulate/collector/fpm_forward/runner.py b/python/aisimulate/collector/fpm_forward/runner.py index 52e2efc83f..3b5a248d2b 100644 --- a/python/aisimulate/collector/fpm_forward/runner.py +++ b/python/aisimulate/collector/fpm_forward/runner.py @@ -446,6 +446,45 @@ def _run_command( return subprocess.CompletedProcess(args, process.returncode, stdout, stderr) +def _attempt_provenance_command( + *, + cell_id: str, + plan_sha256: str, + attempt_id: str, + expected_backend_version: str | None = None, +) -> list[str]: + """Record the observed runtime before checking a profile's pinned version.""" + + payload = json.dumps( + { + "schema_name": "aic_fpm_collector_provenance", + "schema_version": 1, + "cell_id": cell_id, + "plan_sha256": plan_sha256, + "attempt_id": attempt_id, + }, + sort_keys=True, + separators=(",", ":"), + ) + script = ( + "import importlib.metadata, json, pathlib, sys; " + "payload = json.loads(sys.argv[1]); " + "payload['runtime'] = {'backend': 'vllm', " + "'backend_version': importlib.metadata.version('vllm')}; " + f"path = pathlib.Path('{FPM_RESULTS_DIR}') / sys.argv[2]; " + "path.write_text(json.dumps(payload, sort_keys=True) + '\\n')" + ) + version_args = [] + if expected_backend_version is not None: + script += ( + "\nactual = payload['runtime']['backend_version']\n" + "if actual != sys.argv[3]:\n" + " raise RuntimeError(f'FPM profile runtime mismatch: actual={actual!r}, expected={sys.argv[3]!r}')\n" + ) + version_args.append(expected_backend_version) + return ["python3", "-c", script, payload, COLLECTOR_PROVENANCE_FILENAME, *version_args] + + class KubernetesCellRunner: """Own one generated Pod/LWS/PCS and auxiliary resources through deletion.""" @@ -667,33 +706,12 @@ def prepare_attempt( ) -> None: """Clear stale results and bind every Pod to this Collector attempt.""" - payload = json.dumps( - { - "schema_name": "aic_fpm_collector_provenance", - "schema_version": 1, - "cell_id": cell_id, - "plan_sha256": plan_sha256, - "attempt_id": attempt_id, - }, - sort_keys=True, - separators=(",", ":"), - ) - script = ( - "import importlib.metadata, json, pathlib, sys; " - "payload = json.loads(sys.argv[1]); " - "payload['runtime'] = {'backend': 'vllm', " - "'backend_version': importlib.metadata.version('vllm')}; " - f"path = pathlib.Path('{FPM_RESULTS_DIR}') / sys.argv[2]; " - "path.write_text(json.dumps(payload, sort_keys=True) + '\\n')" + command = _attempt_provenance_command( + cell_id=cell_id, + plan_sha256=plan_sha256, + attempt_id=attempt_id, + expected_backend_version=expected_backend_version, ) - version_args = [] - if expected_backend_version is not None: - script += ( - "\nactual = payload['runtime']['backend_version']\n" - "if actual != sys.argv[3]:\n" - " raise RuntimeError(f'FPM profile runtime mismatch: actual={actual!r}, expected={sys.argv[3]!r}')\n" - ) - version_args.append(expected_backend_version) for pod in pods: self._exec_checked( pod, @@ -713,11 +731,7 @@ def prepare_attempt( ], timeout=60, ) - self._exec_checked( - pod, - ["python3", "-c", script, payload, COLLECTOR_PROVENANCE_FILENAME, *version_args], - timeout=60, - ) + self._exec_checked(pod, command, timeout=60) def _remote_result_manifest(self, pod: str) -> dict[str, dict[str, int | str]]: script = ( diff --git a/python/aisimulate/collector/fpm_forward/slurm.py b/python/aisimulate/collector/fpm_forward/slurm.py index 0fffe4c6b6..aea47d6885 100644 --- a/python/aisimulate/collector/fpm_forward/slurm.py +++ b/python/aisimulate/collector/fpm_forward/slurm.py @@ -171,25 +171,25 @@ def _exec(self, unit: str, command: list[str], *, timeout: int): timeout=timeout, ) - def prepare_attempt(self, pods: list[str], *, cell_id: str, plan_sha256: str, attempt_id: str) -> None: - from .native_artifact import COLLECTOR_PROVENANCE_FILENAME + def prepare_attempt( + self, + pods: list[str], + *, + cell_id: str, + plan_sha256: str, + attempt_id: str, + expected_backend_version: str | None = None, + ) -> None: + from .runner import _attempt_provenance_command - payload = json.dumps( - { - "schema_name": "aic_fpm_collector_provenance", - "schema_version": 1, - "cell_id": cell_id, - "plan_sha256": plan_sha256, - "attempt_id": attempt_id, - } - ) - script = ( - "import importlib.metadata,json,pathlib,sys; p=json.loads(sys.argv[1]); " - "p['runtime']={'backend':'vllm','backend_version':importlib.metadata.version('vllm')}; " - "pathlib.Path('/results',sys.argv[2]).write_text(json.dumps(p,sort_keys=True)+'\\n')" + command = _attempt_provenance_command( + cell_id=cell_id, + plan_sha256=plan_sha256, + attempt_id=attempt_id, + expected_backend_version=expected_backend_version, ) for unit in pods: - self._exec(unit, ["python3", "-c", script, payload, COLLECTOR_PROVENANCE_FILENAME], timeout=300) + self._exec(unit, command, timeout=300) def execute(self, pods: list[str], timeout_seconds: int = 14400) -> None: from .runner import CommandScope, _cancel_preserving_interrupt diff --git a/python/aisimulate/tests/unit/collector/test_fpm_slurm.py b/python/aisimulate/tests/unit/collector/test_fpm_slurm.py index fea09b50e8..5eeacdb83f 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_slurm.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_slurm.py @@ -548,3 +548,103 @@ def run(): assert not failures assert results[0].returncode == -signal.SIGTERM assert denied[:2] == [signal.SIGTERM, 0] + + +@pytest.mark.parametrize("actual_version", ["0.25.1", "0.24.0"]) +def test_profile_campaign_observes_slurm_version_before_native_collection(tmp_path, monkeypatch, actual_version): + """Run the real profile campaign; only external Slurm/container operations are faked.""" + from dataclasses import replace + from pathlib import Path + + from collector.fpm_forward import cli + from collector.fpm_forward import runner as campaign + from collector.fpm_forward.config import FPMCollectionOptions + + from .test_fpm_profile_collection import _argv, _plan, _profile + from .test_fpm_runner import _native_payload + + profile = _profile() + options = FPMCollectionOptions.from_args( + cli._parser().parse_args( + [ + *_argv(profile), + "--fpm-executor", + "slurm", + "--fpm-slurm-container-image", + "image@sha256:synthetic", + ] + ) + ) + plan = _plan(profile, options=replace(options, parallel_presets=("tep",))) + commands = [] + executed = [] + monkeypatch.setenv("SLURM_JOB_ID", "1234") + monkeypatch.setattr("collector.fpm_forward.slurm.shutil.which", lambda name: f"/fake/{name}") + monkeypatch.setattr(campaign.time, "sleep", lambda _seconds: None) + import importlib.metadata + + real_version = importlib.metadata.version + monkeypatch.setattr( + importlib.metadata, "version", lambda name: actual_version if name == "vllm" else real_version(name) + ) + + def command(args, **_kwargs): + commands.append(args) + stdout = "" + if args[0] == "scontrol": + stdout = "JobId=1234 JobState=RUNNING NodeList=test-node" if "job" in args else "test-node" + elif args[0] == "squeue": + pass + else: + assert args[0] == "srun", args + assert "--jobid=1234" in args and "--gpus-per-node=8" in args + assert "--container-image=image@sha256:synthetic" in args + mounts = next(value.split("=", 1)[1] for value in args if value.startswith("--container-mounts=")) + raw = Path( + next(value.removesuffix(":/results") for value in mounts.split(",") if value.endswith(":/results")) + ) + assert raw.name == "node0000" + program = args[args.index("env") + 3 :] + if program[:2] == ["python3", "-c"]: + # Execute the emitted program with the container's result mount mapped to disk. + with monkeypatch.context() as patch: + patch.setattr(sys, "argv", ["-c", *program[3:]]) + try: + exec(program[2].replace("/results", str(raw)), {}) + except RuntimeError as error: + raise subprocess.CalledProcessError(1, args, output="prepared", stderr=str(error)) from error + else: + assert program == ["bash", "/tmp/fpm-bench/fpm_exec.sh"] + cell = next(item for item in plan.cells if item.cell_id == raw.parent.parent.name) + executed.append(cell.cell_id) + (raw / "benchmark.json").write_text(json.dumps(_native_payload(phase=cell.workload_kind, rank=0, dp=1))) + return subprocess.CompletedProcess(args, 0, stdout=stdout, stderr="") + + monkeypatch.setattr(campaign, "_run_command", command) + errors = campaign.run_collection( + plan, + generator_overrides={}, + checkpoint_dir=str(tmp_path / "checkpoints"), + artifact_root=str(tmp_path / "artifacts"), + database_root=str(tmp_path / "data"), + resume=False, + retry_failed=False, + ) + checkpoint = json.loads((tmp_path / "checkpoints/fpm_forward.json").read_text()) + if actual_version == "0.25.1": + assert errors == [] + assert executed == [cell.cell_id for cell in plan.cells] + assert checkpoint["database"]["status"] == "passed" + assert {entry["status"] for entry in checkpoint["cells"].values()} == {"passed"} + else: + assert len(errors) == 2 and all(error["classification"] == "campaign_cell_failed" for error in errors) + assert executed == [] and "database" not in checkpoint + for entry in checkpoint["cells"].values(): + failures = list(Path(entry["artifact_dir"]).glob("logs/transport-failures/*/stderr.log")) + assert len(failures) == 1 and "FPM profile runtime mismatch" in failures[0].read_text() + for cell in plan.cells: + entry = checkpoint["cells"][cell.cell_id] + provenance = json.loads((Path(entry["artifact_dir"]) / "raw/node0000/collector-provenance.json").read_text()) + assert provenance["runtime"] == {"backend": "vllm", "backend_version": actual_version} + assert provenance["plan_sha256"] == plan.sha256 and provenance["attempt_id"] == entry["attempt_id"] + assert all(args[0] in {"srun", "squeue", "scontrol"} for args in commands) From 2e0246786504ccfc11e2780881fd92a755c8e22c Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Tue, 22 Sep 2026 14:37:40 +0800 Subject: [PATCH 17/27] test: permit lightweight FPM metadata before resource admission Signed-off-by: Yiming Liu --- tests/test_resource_supervision.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_resource_supervision.py b/tests/test_resource_supervision.py index 9dea4e6c6f..4dfafc7145 100644 --- a/tests/test_resource_supervision.py +++ b/tests/test_resource_supervision.py @@ -516,11 +516,17 @@ def test_supervisor_argument_and_output_setup_do_not_import_runtime(): [ sys.executable, "-c", - ( - "import sys; import aisimulate.supervision, aisimulate.cli_args, aisimulate.output; " - "assert not any(name in sys.modules for name in " - "('aisimulate._runtime', 'aisimulate.sweeper', 'numpy', 'pandas', 'aisimulate_core'))" - ), + """ +import sys +import aisimulate.supervision, aisimulate.cli_args, aisimulate.output + +runtime = ('aisimulate._runtime', 'aisimulate.sweeper', 'numpy', 'pandas') +assert not any(name == root or name.startswith(root + '.') for name in sys.modules for root in runtime) +# Profile validation needs only lightweight metadata before resource admission. +allowed_core = {'aisimulate_core', 'aisimulate_core.fpm_profile', 'aisimulate_core.quantization'} +loaded_core = {name for name in sys.modules if name == 'aisimulate_core' or name.startswith('aisimulate_core.')} +assert loaded_core <= allowed_core, loaded_core - allowed_core +""", ], check=True, timeout=15, From 45fa2577a00ee24b3071f46c48d66db7bfd13196 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 28 Sep 2026 19:31:45 +0800 Subject: [PATCH 18/27] fix: address FPM onboarding review feedback Signed-off-by: Yiming Liu --- docs/fpm-self-service.md | 8 +- .../src/aisimulate/config/common.py | 11 + .../src/aisimulate/config/engine.py | 14 +- .../aisimulate/src/aisimulate/support/fpm.py | 28 ++- .../aisimulate/src/aisimulate/support/plan.py | 21 +- .../src/aisimulate/sweeper/config.py | 13 +- .../aisimulate/tests/unit/test_support_cli.py | 209 +++++++++++++++++- .../tests/unit/test_support_plan.py | 166 +++++++++++++- .../tests/unit/test_systems_path.py | 91 +++++++- 9 files changed, 519 insertions(+), 42 deletions(-) diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index ef6b335f54..7b6309b36b 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -53,7 +53,7 @@ aisimulate onboard collect-fpm \ --output-dir ./aisimulate-support ``` -The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. The second prints the collector command without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. On an existing plan, `--overwrite` can repair missing generated files for the identical request; it rejects changed inputs and preserves existing collected data. +The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. The second prints the collector command without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. `--overwrite` can repair missing generated files for the identical saved request, including an interrupted write before `support-plan.json` was created. Repair requires an intact matching `request.yaml`, rejects changed inputs, and preserves existing collected data. If a newer draft revision changes generated guidance, recreate the plan in a new output directory: repair compares generated files byte for byte and does not migrate earlier draft plans. Guidance changes do not change the saved-request identity checks used by collection. @@ -69,7 +69,7 @@ aisimulate onboard collect-fpm \ --output-dir ./aisimulate-support --execute ``` -Execution requires a matching saved plan. The request records model and runtime revisions; this setup does not download a pinned checkpoint or verify the installed runtime against them. Keep the actual checkpoint and runtime consistent with the request before collecting or predicting. +Execution requires a matching saved plan. The request records model and runtime revisions; this setup does not download a pinned checkpoint or check the installed runtime before execution. Keep the actual checkpoint and runtime consistent with the request before collecting or predicting. Successful formal collection and resume verify that the published data's pod-reported runtime version exactly matches `framework_version`, including suffixes such as `+cu128`. A mismatch exits 1 and preserves collection artifacts. Use the declared runtime or create a new request and plan for the observed version; generated configs are never silently retargeted. Diagnostic smoke runs do not publish or verify formal data. Set deployment options directly on `onboard collect-fpm`: `--dynamo-version VERSION`, `--image IMAGE`, `--namespace NAME`, `--model-cache NAME[:MOUNT[:SUBPATH]]`, `--transport nvlink|ib|efa`, and `--image-pull-secret NAME`. The mount, when supplied, is an absolute container path. Prefer an immutable image digest. Supply the same options when previewing, executing, and resuming; deployment settings are part of the collector's frozen-plan identity, so changed settings require a new output directory. Arbitrary collector arguments and engine overrides are not accepted by this command. @@ -129,7 +129,9 @@ The loop reports each command's exit status and attempts all commands, including Run either the individual recommendation command or the loop against fresh result directories. The loop also handles a one-candidate plan and does not execute entries from `commands.json`. Results remain separate under `recommend-results/pilot` and, when present, `recommend-results/replicas-N`; compare their objective and latency results for the same workload. This plan does not produce a combined ranking or search additional TP sizes or scheduler settings. -The generated configurations select `engine.workers.aggregated.timing.estimation_mode: fpm_interpolation` with `fallback_policy: deny` and set `engine.systems_paths` to a list containing the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. Existing configurations using the single-root `engine.systems_path` input are normalized to `engine.systems_paths` when saved. +The generated configurations select `engine.workers.aggregated.timing.estimation_mode: fpm_interpolation` with `fallback_policy: deny` and set `engine.systems_paths` to a list containing the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. The single-root `engine.systems_path` input is an alias added by onboarding; saved configurations use the canonical `engine.systems_paths` list. Both spellings resolve relative local paths against the working directory when the configuration is loaded, so saving and reloading from another directory preserves the selected roots. The canonical list's `default` entry continues to select packaged data. + +These configurations target AISimulate's standalone `predict` and `recommend` commands. Dynamo's replay Planner AIC session adapter does not currently forward custom systems roots; using that downstream path requires a separate adapter update. Ordinary `predict` and `recommend` commands retain their existing behavior and defaults. Their generated configs can be loaded and edited through the public configuration schema. In this revision, missing model registration or data may still prevent execution; a successful simulation is not an accuracy result. Compare its output with an independent run of the same model, runtime, topology, and workload to assess accuracy. diff --git a/python/aisimulate/src/aisimulate/config/common.py b/python/aisimulate/src/aisimulate/config/common.py index 8bce052fe4..7ccf3b3a1e 100644 --- a/python/aisimulate/src/aisimulate/config/common.py +++ b/python/aisimulate/src/aisimulate/config/common.py @@ -27,6 +27,17 @@ def _absolute_systems_path(value: str) -> str: SystemsPath = Annotated[str, Field(strict=True, min_length=1), AfterValidator(_absolute_systems_path)] +def _normalize_systems_root(value: str) -> str: + if not value.strip(): + raise ValueError("systems_paths entries must be nonempty") + if value.lower() == "default": + return "default" + return str(Path(value).expanduser().resolve()) + + +SystemsRoot = Annotated[str, Field(strict=True, min_length=1), AfterValidator(_normalize_systems_root)] + + T = TypeVar("T") PositiveFiniteFloat = Annotated[float, Field(strict=True, gt=0, allow_inf_nan=False)] PositiveStrictInt = Annotated[int, Field(strict=True, gt=0)] diff --git a/python/aisimulate/src/aisimulate/config/engine.py b/python/aisimulate/src/aisimulate/config/engine.py index 6921faddab..2c6d35cf4f 100644 --- a/python/aisimulate/src/aisimulate/config/engine.py +++ b/python/aisimulate/src/aisimulate/config/engine.py @@ -16,6 +16,7 @@ NumericRange, StrictModel, SystemsPath, + SystemsRoot, is_active_engine_model_control, ) @@ -217,7 +218,7 @@ class TimingConfig(StrictModel): estimation_mode: Literal["auto", "op_level", "fpm_interpolation", "fpm_regression"] | None = None fallback_policy: Literal["deny", "allow"] | None = None estimator_config: dict[str, Any] | None = None - systems_paths: list[str] | None = Field(default=None, min_length=1) + systems_paths: list[SystemsRoot] | None = Field(default=None, min_length=1) database_mode: Literal["SILICON", "HYBRID", "EMPIRICAL", "SOL"] | None = None transfer_policy: str | list[str] | None = None @@ -352,7 +353,7 @@ def _validate_engine_controls(self): database_mode: Literal["SILICON", "HYBRID", "EMPIRICAL", "SOL"] = "SILICON" transfer_policy: str | list[str] | None = None - systems_paths: list[str] | None = None + systems_paths: list[SystemsRoot] | None = Field(default=None, min_length=1) systems_path: SystemsPath | None = Field(default=None, exclude=True) estimation_mode: Literal["auto", "op_level", "fpm_interpolation", "fpm_regression"] = "auto" fallback_policy: Literal["deny", "allow"] = "deny" @@ -371,15 +372,6 @@ def _migrate_systems_path(self): def _normalize_database_mode(cls, value): return value.upper() if isinstance(value, str) else value - @field_validator("systems_paths") - @classmethod - def _nonempty_system_roots(cls, value): - if value is None: - return value - if not value or any(not path.strip() for path in value): - raise ValueError("systems_paths must contain at least one nonempty root") - return value - @model_validator(mode="after") def _supported_estimator_policies(self): mode = getattr(self, "mode", "aggregated") diff --git a/python/aisimulate/src/aisimulate/support/fpm.py b/python/aisimulate/src/aisimulate/support/fpm.py index b13e4cd3ef..af53c8fdad 100644 --- a/python/aisimulate/src/aisimulate/support/fpm.py +++ b/python/aisimulate/src/aisimulate/support/fpm.py @@ -150,12 +150,32 @@ def run_fpm( with plan_lock(root): check_plan(request, root) _check_campaign_outputs(root, smoke=smoke, resume=resume, checkpoint_dir=checkpoint_dir) - from collector.fpm_forward.cli import main as fpm_main - # The collector reports input/plan failures through argparse before - # entering run_resolved; only execution failures escape this call. + # entering run_resolved; imports and execution failures return exit 1. try: - return fpm_main(command[3:]) + from collector.fpm_forward.cli import main as fpm_main + + result = fpm_main(command[3:]) + if result == 0 and not smoke: + selected = Path(checkpoint_dir).expanduser().resolve() if checkpoint_dir else root / "fpm-checkpoint" + checkpoint = json.loads((selected / "fpm_forward.json").read_text(encoding="utf-8")) + publication = checkpoint.get("database") + if not isinstance(publication, dict) or publication.get("status") != "passed": + raise RuntimeError("collector did not record a completed FPM database publication") + # The collector validates this commit record, including on resume + # after raw-artifact reclamation. Check its actual runtime version, + # not an unrelated directory left by an earlier collection. + metadata = json.loads(Path(publication["metadata"]).read_text(encoding="utf-8")) + observed = metadata.get("backend_version") + expected = request.identity.framework_version + if observed != expected: + raise RuntimeError( + f"pod-reported {request.identity.framework} version {observed!r} does not match " + f"framework_version {expected!r}; generated configs cannot use this publication. " + "Collected artifacts are preserved. Create a new plan in a new output directory " + "using a matching runtime or the observed framework_version." + ) + return result except Exception as exc: print(f"aisimulate onboard collect-fpm failed: {exc}", file=sys.stderr) return 1 diff --git a/python/aisimulate/src/aisimulate/support/plan.py b/python/aisimulate/src/aisimulate/support/plan.py index 14d6e0c48b..46b08c406b 100644 --- a/python/aisimulate/src/aisimulate/support/plan.py +++ b/python/aisimulate/src/aisimulate/support/plan.py @@ -12,6 +12,8 @@ import stat from collections.abc import Iterator from contextlib import contextmanager +from importlib.resources import files +from importlib.util import find_spec, module_from_spec from pathlib import Path from typing import Any @@ -140,7 +142,11 @@ def _commands(request: SupportRequest, root: Path, recommendation_names: list[st def _plan_documents(request: SupportRequest, root: Path) -> tuple[dict[str, Any], dict[Path, bytes]]: - source_spec = Path(__file__).resolve().parents[2] / "aisimulate_core/systems" / f"{request.identity.gpu}.yaml" + package_spec = find_spec("aisimulate_core") + if package_spec is None: + raise ValueError("aisimulate_core package resources are required for packaged system specifications") + # Consult the resource loader without executing the package's native imports. + source_spec = files(module_from_spec(package_spec)) / "systems" / f"{request.identity.gpu}.yaml" if not source_spec.is_file(): raise ValueError( f"GPU {request.identity.gpu!r} has no packaged system specification; " @@ -284,6 +290,10 @@ def check_plan(request: SupportRequest, root: Path) -> None: ) from exc if not isinstance(prior, dict) or prior.get("request_id") != request_id(request): raise ValueError(f"refusing to mix a different request identity in {root}; choose a new output directory") + _check_saved_request(request, root) + + +def _check_saved_request(request: SupportRequest, root: Path) -> None: try: saved = SupportRequest.from_yaml(root / "request.yaml") except ValueError as exc: @@ -304,7 +314,12 @@ def create_plan(request: SupportRequest, output_dir: str | Path, *, overwrite: b if nonempty: if not overwrite: raise ValueError(f"output directory {root} is nonempty; use overwrite only for the same request") - check_plan(request, root) + if (root / "support-plan.json").exists(): + check_plan(request, root) + else: + # request.yaml is written first, so it can establish identity + # after an interruption before the completed plan was saved. + _check_saved_request(request, root) # Inspect every generated file before writing any file. Existing data, # checkpoints, results, and edited inputs are never replaced or deleted. pending: dict[Path, bytes] = {} @@ -316,10 +331,10 @@ def create_plan(request: SupportRequest, output_dir: str | Path, *, overwrite: b pending[relative] = content elif destination.read_bytes() != content: raise ValueError(f"generated plan input {destination} was modified; choose a new output directory") - (root / "systems/data").mkdir(parents=True, exist_ok=True) for relative, content in pending.items(): destination = root / relative destination.parent.mkdir(parents=True, exist_ok=True) with destination.open("xb") as handle: handle.write(content) + (root / "systems/data").mkdir(parents=True, exist_ok=True) return plan diff --git a/python/aisimulate/src/aisimulate/sweeper/config.py b/python/aisimulate/src/aisimulate/sweeper/config.py index dccc8e3c87..fe34bd42c6 100644 --- a/python/aisimulate/src/aisimulate/sweeper/config.py +++ b/python/aisimulate/src/aisimulate/sweeper/config.py @@ -29,7 +29,7 @@ from aisimulate.config.traffic import AgenticSnapshotOptions -from ..config.common import ENGINE_MODEL_CONTROL_FIELDS, SystemsPath, is_active_engine_model_control +from ..config.common import ENGINE_MODEL_CONTROL_FIELDS, SystemsPath, SystemsRoot, is_active_engine_model_control from ..config.engine import NgramSpeculationConfig @@ -525,7 +525,7 @@ class SearchSpace(BaseModel): hardware_sku: str # e.g. "h200_sxm" database_mode: Literal["SILICON", "HYBRID", "EMPIRICAL", "SOL"] = "SILICON" transfer_policy: str | list[str] | None = None - systems_paths: list[str] | None = Field(default=None, min_length=1) + systems_paths: list[SystemsRoot] | None = Field(default=None, min_length=1) systems_path: SystemsPath | None = Field(default=None, exclude=True) estimation_mode: Literal["auto", "op_level", "fpm_interpolation", "fpm_regression"] = "auto" fallback_policy: Literal["deny", "allow"] = "deny" @@ -828,15 +828,6 @@ def _validate_afd_contract(self) -> SearchSpace: def _normalize_estimator_database_mode(cls, value): return value.upper() if isinstance(value, str) else value - @field_validator("systems_paths") - @classmethod - def _validate_estimator_roots(cls, value): - if value is None: - return value - if any(not path.strip() for path in value): - raise ValueError("systems_paths entries must be nonempty") - return value - @model_validator(mode="after") def _validate_estimator_controls(self): from aisimulate_core.sdk.common import resolve_transfer_policy diff --git a/python/aisimulate/tests/unit/test_support_cli.py b/python/aisimulate/tests/unit/test_support_cli.py index c8d8ea3070..4333cb7121 100644 --- a/python/aisimulate/tests/unit/test_support_cli.py +++ b/python/aisimulate/tests/unit/test_support_cli.py @@ -5,9 +5,11 @@ from __future__ import annotations +import builtins import json import os import shlex +import shutil import subprocess import sys from pathlib import Path @@ -429,7 +431,7 @@ def collector_main(argv): assert calls[0][calls[0].index("--limit") + 1] == "1" -def _local_collection_command(tmp_path): +def _local_collection_command(tmp_path, **changes): model = tmp_path / "model with spaces" model.mkdir() (model / "config.json").write_text( @@ -450,11 +452,189 @@ def _local_collection_command(tmp_path): ) request_path = tmp_path / "request.yaml" root = tmp_path / "plan" - assert cli.main(_init_args(request_path, model=model)) == 0 + assert cli.main(_init_args(request_path, model=model, **changes)) == 0 assert cli.main(["onboard", "plan", "-c", str(request_path), "--output-dir", str(root)]) == 0 return ["onboard", "collect-fpm", "-c", str(request_path), "--output-dir", str(root), "--execute"] +def _synthetic_collector(monkeypatch, version): + """Exercise real collection, provenance validation, and publication without GPUs.""" + from collector.fpm_forward import planner, runner + + executions = [] + + def render_cell(_plan, _cell, cell_dir, _overrides, **_kwargs): + (cell_dir / "k8s_deploy.yaml").write_text("apiVersion: v1\nkind: Pod\nmetadata:\n name: test-cell\n") + for name in ("run.sh", "fpm_env.sh", "collector-runtime-env.sh"): + (cell_dir / name).write_text("#!/bin/sh\n") + + class LocalResource: + def __init__(self, _manifest, cell_dir): + self.cell = json.loads((cell_dir / "cell.json").read_text()) + self.raw = cell_dir / "raw/pod-0" + + def apply(self): + self.raw.mkdir(parents=True) + + def wait_ready(self, _expected_nodes): + return ["pod-0"] + + def stage(self, _pods, _files): + pass + + def prepare_attempt(self, _pods, **identity): + (self.raw / "collector-provenance.json").write_text( + json.dumps( + { + "schema_name": "aic_fpm_collector_provenance", + "schema_version": 1, + **identity, + "runtime": {"backend": "vllm", "backend_version": version}, + } + ) + ) + + def execute(self, _pods): + executions.append(self.cell["cell_id"]) + phase = self.cell["workload_kind"] + prefill = int(phase == "prefill") + point = { + "point_type": phase, + "benchmark_id": 1, + "batch_size": 1, + "total_prefill_tokens": prefill, + "total_kv_read_tokens": 1, + } + fpm = { + "counter_id": 1, + "dp_rank": 0, + "wall_time": 0.01, + "scheduled_requests": { + "num_prefill_requests": prefill, + "sum_prefill_tokens": prefill, + "sum_prefill_kv_tokens": prefill, + "num_decode_requests": 1 - prefill, + "sum_decode_kv_tokens": 1 - prefill, + }, + } + (self.raw / "benchmark.json").write_text( + json.dumps( + { + "schema_version": 2, + "artifact_type": "rank", + "status": "complete", + "valid": True, + "usable": True, + "timing_valid": True, + "run_id": "synthetic-run", + "grid_digest": "synthetic-grid", + "config": {"mode": phase}, + "coverage": {"expected_points": 1, "completed_points": 1, "skipped_points": 0}, + "dp": {"rank": 0, "size": 1}, + "results": [{"point": point, "fpms": [fpm]}], + "iteration_groups": [ + { + "benchmark_id": 1, + "point": point, + "expected_dp_ranks": [0], + "complete": True, + "wall_time": 0.01, + "rank_results": [{"dp_rank": 0, "fpms": [fpm]}], + } + ], + "skipped_points": [], + "missing_phases": [], + "timing": {"benchmark_elapsed_seconds": 1.0, "measured_iteration_seconds": 0.01}, + "kvwarm": {"enabled": True, "warm_eligible": True, "skip_reason": None}, + } + ) + ) + + def collect(self, _pods, *, require_benchmark=True): + pass + + def cleanup(self): + pass + + monkeypatch.setattr(planner, "_git_revision", lambda: "test-source-revision") + monkeypatch.setattr(runner, "_render_cell", render_cell) + monkeypatch.setattr(runner, "KubernetesCellRunner", LocalResource) + return executions + + +@pytest.mark.parametrize("smoke", [False, True]) +@pytest.mark.parametrize( + "declared,observed", + [ + ("0.25.1", "0.25.1"), + ("0.25.1", "0.25.1+cu128"), + ("0.25.1", "0.26.0"), + ("0.25.1+cu128", "0.25.1+cu128"), + ], +) +def test_formal_collection_checks_published_version_and_preserves_evidence_on_resume( + tmp_path, monkeypatch, capsys, smoke, declared, observed +): + executions = _synthetic_collector(monkeypatch, observed) + command = _local_collection_command(tmp_path, framework_version=declared) + root = tmp_path / "plan" + if declared != observed: + command += ["--checkpoint-dir", str(root / "fpm-checkpoint/custom")] + if smoke: + command += ["--smoke", "--limit", "1"] + capsys.readouterr() + expected = int(not smoke and declared != observed) + + assert cli.main(command) == expected + + stderr = capsys.readouterr().err + if expected: + assert f"pod-reported vllm version {observed!r}" in stderr + assert f"framework_version {declared!r}" in stderr + assert "preserved" in stderr + assert "new plan" in stderr + assert "Traceback" not in stderr + provenance = list((root / "fpm-artifacts").rglob("collector-provenance.json")) + assert provenance + assert {json.loads(path.read_text())["runtime"]["backend_version"] for path in provenance} == {observed} + metadata = root / "systems/data/h200_sxm/vllm" / observed / "fpm_forward_perf.metadata.json" + if smoke: + assert not list((root / "systems/data").iterdir()) + else: + assert json.loads(metadata.read_text())["backend_version"] == observed + if declared != observed: + # A stale matching directory must not hide this campaign's different runtime. + (root / "systems/data/h200_sxm/vllm" / declared).mkdir() + preserved = { + path: path.read_bytes() for path in root.rglob("*") if path.is_file() and path.name not in {"run-manifest.json"} + } + initial_executions = executions.copy() + + assert cli.main([*command, "--resume"]) == expected + assert executions == initial_executions + assert all(path.read_bytes() == contents for path, contents in preserved.items()) + if not smoke: + # A completed formal checkpoint remains resumable after raw-artifact reclamation. + for path in provenance: + shutil.rmtree(path.parent.parent) + assert cli.main([*command, "--resume"]) == expected + assert executions == initial_executions + assert metadata.read_bytes() == preserved[metadata] + + +def test_formal_collection_requires_publication_evidence(tmp_path, monkeypatch, capsys): + from collector.fpm_forward import runner + + command = _local_collection_command(tmp_path) + monkeypatch.setattr(runner, "run_collection", lambda *args, **kwargs: []) + capsys.readouterr() + + assert cli.main(command) == 1 + stderr = capsys.readouterr().err + assert "aisimulate onboard collect-fpm failed:" in stderr + assert "Traceback" not in stderr + + def test_deployment_options_reach_frozen_collector_plan(tmp_path, monkeypatch): from collector.fpm_forward import runner @@ -462,6 +642,7 @@ def test_deployment_options_reach_frozen_collector_plan(tmp_path, monkeypatch): monkeypatch.setattr(runner, "run_collection", lambda plan, **kwargs: calls.append((plan, kwargs)) or []) command = _local_collection_command(tmp_path) command += [ + "--smoke", "--dynamo-version", "1.2.0", "--image", @@ -539,8 +720,8 @@ def test_deployment_options_reject_invalid_or_unrestricted_inputs(tmp_path, caps def test_deployment_resume_requires_the_same_frozen_identity(tmp_path, monkeypatch, capsys, option, first, changed): from collector.fpm_forward import planner, runner - command = [*_local_collection_command(tmp_path), option, first] - checkpoint = tmp_path / "plan/fpm-checkpoint/fpm_forward.json" + command = [*_local_collection_command(tmp_path), "--smoke", option, first] + checkpoint = tmp_path / "plan/fpm-checkpoint/fpm_forward_smoke.json" plans = [] actual_run = runner.run_collection @@ -606,6 +787,26 @@ def fail(*args, **kwargs): assert "aisimulate onboard collect-fpm failed: test collection failure" in stderr +@pytest.mark.parametrize("error_type", [ImportError, KeyboardInterrupt]) +def test_collector_import_failures_keep_public_cli_exit_codes(tmp_path, monkeypatch, capsys, error_type): + command = _local_collection_command(tmp_path) + real_import = builtins.__import__ + + def fail_collector_import(name, *args, **kwargs): + if name == "collector.fpm_forward.cli": + raise error_type("collector dependency unavailable") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fail_collector_import) + capsys.readouterr() + + assert cli.main(command) == (130 if error_type is KeyboardInterrupt else 1) + stderr = capsys.readouterr().err + assert "Traceback" not in stderr + if error_type is ImportError: + assert "aisimulate onboard collect-fpm failed: collector dependency unavailable" in stderr + + def test_malformed_resumed_checkpoint_fails_cleanly_without_launching_collection(tmp_path, capsys): from collector.fpm_forward.runner import CHECKPOINT_SCHEMA diff --git a/python/aisimulate/tests/unit/test_support_plan.py b/python/aisimulate/tests/unit/test_support_plan.py index 0409d6c53b..26b4747fb6 100644 --- a/python/aisimulate/tests/unit/test_support_plan.py +++ b/python/aisimulate/tests/unit/test_support_plan.py @@ -14,7 +14,10 @@ import shlex import subprocess import sys +import zipfile from contextlib import contextmanager +from importlib.machinery import PathFinder +from importlib.resources import files from pathlib import Path from typing import ClassVar @@ -209,6 +212,8 @@ def test_onboarding_plan_and_preview_do_not_import_estimator_runtime(tmp_path): class NoEstimatorRuntime(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path=None, target=None): + if fullname == "aisimulate_core": + return None # Resource lookup may inspect the package spec without importing it. assert not any(fullname == name or fullname.startswith(name + ".") for name in blocked), fullname sys.meta_path.insert(0, NoEstimatorRuntime()) @@ -474,9 +479,12 @@ def test_prefill_bounds_follow_concurrent_workload_and_collector_minimum(tmp_pat {"identity": {"tokenizer_revision": "tokenizer-v2"}}, ], ) -def test_overwrite_rejects_different_requests_without_touching_data(tmp_path, updates): +@pytest.mark.parametrize("complete", [False, True]) +def test_overwrite_rejects_different_requests_without_touching_data(tmp_path, updates, complete): request = _request() create_plan(request, tmp_path) + if not complete: + (tmp_path / "support-plan.json").unlink() data = tmp_path / "systems/data/collected.parquet" data.write_bytes(b"timings") before = {str(p.relative_to(tmp_path)): p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()} @@ -504,9 +512,12 @@ def test_same_request_overwrite_preserves_timings_and_rejects_modified_inputs(tm @pytest.mark.parametrize("modification", ["retired_command", "missing_execute", "formatting"]) -def test_overwrite_rejects_modified_commands_before_repairing_missing_files(tmp_path, modification): +@pytest.mark.parametrize("complete", [False, True]) +def test_overwrite_rejects_modified_commands_before_repairing_missing_files(tmp_path, modification, complete): request = _request() create_plan(request, tmp_path) + if not complete: + (tmp_path / "support-plan.json").unlink() command_file = tmp_path / "commands.json" commands = json.loads(command_file.read_text()) if modification == "retired_command": @@ -526,6 +537,51 @@ def test_overwrite_rejects_modified_commands_before_repairing_missing_files(tmp_ assert _file_contents(tmp_path) == before_repair +@pytest.mark.parametrize("layout", ["directory", "zip"]) +def test_plan_copies_system_specification_from_core_package_resources(tmp_path, monkeypatch, layout): + from aisimulate.support import plan + + system_spec = files("aisimulate_core").joinpath("systems", "h200_sxm.yaml").read_bytes() + system_spec += b"\n# Packaged separately from the onboarding module.\n" + package_init = "raise AssertionError('reading system resources must not execute the core package')\n" + if layout == "directory": + distribution = tmp_path / "separate-core-package" + package = distribution / "aisimulate_core" + (package / "systems").mkdir(parents=True) + (package / "__init__.py").write_text(package_init) + (package / "systems/h200_sxm.yaml").write_bytes(system_spec) + else: + distribution = tmp_path / "core.zip" + with zipfile.ZipFile(distribution, "w") as handle: + handle.writestr("aisimulate_core/__init__.py", package_init) + handle.writestr("aisimulate_core/systems/h200_sxm.yaml", system_spec) + package_spec = PathFinder.find_spec("aisimulate_core", [str(distribution)]) + assert package_spec is not None + + def core_spec(name): + assert name == "aisimulate_core" + return package_spec + + monkeypatch.setattr(plan, "find_spec", core_spec) + root = tmp_path / "plan" + + create_plan(_request(), root) + + assert (root / "systems/h200_sxm.yaml").read_bytes() == system_spec + + +def test_missing_core_package_is_rejected_before_creating_outputs(tmp_path, monkeypatch): + from aisimulate.support import plan + + monkeypatch.setattr(plan, "find_spec", lambda name: None) + root = tmp_path / "plan" + + with pytest.raises(ValueError, match="aisimulate_core package resources"): + create_plan(_request(), root) + + assert not root.exists() + + def test_new_gpu_is_rejected_before_creating_outputs(tmp_path): root = tmp_path / "new-plan" with pytest.raises(ValueError, match="packaged.*system|system specification"): @@ -753,13 +809,16 @@ def test_invalid_limited_collection_fails_before_preview_or_execution(tmp_path, run_fpm(_request(), output_dir=tmp_path, smoke=smoke, limit=limit) -def test_plan_rejects_symlinked_output_without_touching_target(tmp_path): +@pytest.mark.parametrize("complete", [False, True]) +def test_plan_rejects_symlinked_output_without_touching_target(tmp_path, complete): external = tmp_path / "external" external.mkdir() data = external / "collected.parquet" data.write_bytes(b"timings") root = tmp_path / "plan" create_plan(_request(), root) + if not complete: + (root / "support-plan.json").unlink() (root / "systems/data").rmdir() (root / "systems/data").symlink_to(external, target_is_directory=True) with pytest.raises(ValueError, match="symlink"): @@ -898,3 +957,104 @@ def racing_open(path, mode="r", *args, **kwargs): assert not (tmp_path / "support-plan.json").exists() with plan_lock(tmp_path): pass + + before_repair = _file_contents(tmp_path) + with pytest.raises(ValueError, match="generated plan input .*pilot.yaml was modified"): + create_plan(_request(), tmp_path, overwrite=True) + assert _file_contents(tmp_path) == before_repair + + monkeypatch.setattr(Path, "open", original_open) + destination.unlink() + create_plan(_request(), tmp_path, overwrite=True) + assert CorePredictionConfig.from_yaml(destination).engine.model == _request().identity.model + + +@pytest.mark.parametrize( + "interrupted_file", + ["predict/pilot.yaml", "recommend/pilot.yaml", "systems/h200_sxm.yaml", "commands.json", "support-plan.json"], +) +def test_overwrite_repairs_interrupted_plan_and_preserves_existing_outputs(tmp_path, monkeypatch, interrupted_file): + request = _request() + original_open = Path.open + + def interrupted_open(path, mode="r", *args, **kwargs): + if path == tmp_path / interrupted_file and mode == "xb": + raise InterruptedError("interrupted plan generation") + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", interrupted_open) + with pytest.raises(InterruptedError): + create_plan(request, tmp_path) + monkeypatch.setattr(Path, "open", original_open) + assert SupportRequest.from_yaml(tmp_path / "request.yaml") == request + assert not (tmp_path / "support-plan.json").exists() + (tmp_path / "systems/data").mkdir(parents=True, exist_ok=True) + _seed_campaign(tmp_path) + before_repair = _file_contents(tmp_path) + + with pytest.raises(ValueError, match="nonempty"): + create_plan(request, tmp_path) + _reject_collector_import(monkeypatch) + with pytest.raises(ValueError, match="readable onboarding plan"): + run_fpm(request, execute=True, resume=True, output_dir=tmp_path) + assert _file_contents(tmp_path) == before_repair + + repaired = create_plan(request, tmp_path, overwrite=True) + + assert json.loads((tmp_path / "support-plan.json").read_text()) == repaired + assert repaired["request_id"] == request_id(request) + assert CorePredictionConfig.from_yaml(tmp_path / "predict/pilot.yaml").engine.model == request.identity.model + assert CoreRecommendationConfig.from_yaml(tmp_path / "recommend/pilot.yaml").engine.model == request.identity.model + assert json.loads((tmp_path / "commands.json").read_text())["fpm_run_local"][1:3] == ["onboard", "collect-fpm"] + assert (tmp_path / "systems/h200_sxm.yaml").is_file() + assert {name: (tmp_path / name).read_bytes() for name in before_repair} == before_repair + assert create_plan(request, tmp_path, overwrite=True) == repaired + + +def test_plan_can_restart_after_interruption_before_saved_request(tmp_path, monkeypatch): + original_open = Path.open + + def interrupted_open(path, mode="r", *args, **kwargs): + if path == tmp_path / "request.yaml" and mode == "xb": + raise InterruptedError("interrupted plan generation") + return original_open(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", interrupted_open) + with pytest.raises(InterruptedError): + create_plan(_request(), tmp_path) + monkeypatch.setattr(Path, "open", original_open) + + assert [path.name for path in tmp_path.iterdir()] == [".support.lock"] + assert create_plan(_request(), tmp_path)["request_id"] == request_id(_request()) + + +@pytest.mark.parametrize("metadata", ["invalid JSON", "{}", '{"request_id": "another-request"}']) +def test_overwrite_rejects_existing_invalid_metadata_before_repair(tmp_path, metadata): + request = _request() + create_plan(request, tmp_path) + (tmp_path / "support-plan.json").write_text(metadata) + (tmp_path / "predict/pilot.yaml").unlink() + before_repair = _file_contents(tmp_path) + + with pytest.raises(ValueError, match="readable onboarding plan|request identity"): + create_plan(request, tmp_path, overwrite=True) + + assert _file_contents(tmp_path) == before_repair + + +@pytest.mark.parametrize("saved_request", [None, "identity: broken\n"]) +def test_overwrite_requires_frozen_request_for_interrupted_plan(tmp_path, saved_request): + request = _request() + create_plan(request, tmp_path) + (tmp_path / "support-plan.json").unlink() + request_path = tmp_path / "request.yaml" + if saved_request is None: + request_path.unlink() + else: + request_path.write_text(saved_request) + before_repair = _file_contents(tmp_path) + + with pytest.raises(ValueError, match="saved request"): + create_plan(request, tmp_path, overwrite=True) + + assert _file_contents(tmp_path) == before_repair diff --git a/python/aisimulate/tests/unit/test_systems_path.py b/python/aisimulate/tests/unit/test_systems_path.py index 4d89190321..b1956713bf 100644 --- a/python/aisimulate/tests/unit/test_systems_path.py +++ b/python/aisimulate/tests/unit/test_systems_path.py @@ -11,6 +11,7 @@ import sys from copy import deepcopy from importlib.resources import files +from pathlib import Path import pyarrow as pa import pyarrow.parquet as pq @@ -21,8 +22,9 @@ from aisimulate.compiler import prediction_to_replay_spec from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig from aisimulate.output import write_recommendations -from aisimulate.recommend import run_recommendation +from aisimulate.recommend import recommendation_to_sweeper, run_recommendation from aisimulate.runner import EngineReplayRunnerFactory +from aisimulate.sweeper.config import SearchSpace from aisimulate_core.sdk import ForwardPassPerfModelConfig, RustForwardPassPerfModel, models, perf_database from aisimulate_core.sdk.config import ModelConfig from aisimulate_core.sdk.errors import PerfDataNotAvailableError @@ -48,10 +50,11 @@ def _request() -> dict: @pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) -def test_systems_root_is_absolute_and_reloadable_before_collection(config_type, tmp_path, monkeypatch): +@pytest.mark.parametrize("root_key", ["systems_path", "systems_paths"]) +def test_systems_root_is_absolute_and_reloadable_before_collection(config_type, root_key, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) raw = _request() - raw["engine"]["systems_path"] = "local profiles" + raw["engine"][root_key] = "local profiles" if root_key == "systems_path" else ["local profiles"] if config_type is CoreRecommendationConfig: raw["optimization"] = {"target": "throughput"} @@ -64,6 +67,66 @@ def test_systems_root_is_absolute_and_reloadable_before_collection(config_type, assert config_type.model_validate(saved).engine.systems_paths == config.engine.systems_paths +@pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) +@pytest.mark.parametrize("scope", ["engine", "worker"]) +def test_canonical_systems_roots_preserve_order_and_default_on_reload(config_type, scope, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + raw = _request() + controls = raw["engine"] if scope == "engine" else raw["engine"]["workers"]["aggregated"].setdefault("timing", {}) + controls["systems_paths"] = ["second root", "DEFAULT", "~/first root", "with,comma", "second root"] + if config_type is CoreRecommendationConfig: + raw["optimization"] = {"target": "throughput"} + expected = [ + str(tmp_path / "second root"), + "default", + str(Path.home() / "first root"), + str(tmp_path / "with,comma"), + str(tmp_path / "second root"), + ] + config = config_type.model_validate(raw) + saved = config.model_dump(mode="json", exclude_none=True) + monkeypatch.chdir(tmp_path.parent) + + reloaded = config_type.model_validate(saved) + controls = reloaded.engine if scope == "engine" else reloaded.engine.workers.aggregated.timing + assert controls.systems_paths == expected + if config_type is CoreRecommendationConfig: + assert recommendation_to_sweeper(reloaded).search_space.systems_paths_for("agg") == expected + + +@pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) +def test_systems_root_alias_accepts_equivalent_canonical_roots(config_type, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + raw = _request() + raw["engine"].update(systems_path="local profiles", systems_paths=["./local profiles"]) + if config_type is CoreRecommendationConfig: + raw["optimization"] = {"target": "throughput"} + + assert config_type.model_validate(raw).engine.systems_paths == [str(tmp_path / "local profiles")] + + +@pytest.mark.parametrize("root_key", ["systems_path", "systems_paths"]) +def test_search_space_roots_are_normalized_before_reload(root_key, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + roots = {root_key: "shared" if root_key == "systems_path" else ["shared", "default", "other"]} + space = SearchSpace( + model_name="model", + hardware_sku="system", + role_estimator_controls={"prefill": {"systems_paths": ["worker", "default"]}}, + **roots, + ) + saved = space.model_dump(mode="json", exclude_none=True) + monkeypatch.chdir(tmp_path.parent) + + reloaded = SearchSpace.model_validate(saved) + expected = [str(tmp_path / "shared")] + if root_key == "systems_paths": + expected.extend(["default", str(tmp_path / "other")]) + assert "systems_path" not in saved + assert reloaded.systems_paths_for("decode") == expected + assert reloaded.systems_paths_for("prefill") == [str(tmp_path / "worker"), "default"] + + @pytest.mark.parametrize("config_type", [CorePredictionConfig, CoreRecommendationConfig]) @pytest.mark.parametrize("value", ["", " ", 7, ["one"], {"choices": ["one"]}, "one,two"]) def test_systems_root_rejects_empty_or_multiple_roots(config_type, value): @@ -218,6 +281,28 @@ def _predict(raw): runner.close() +@pytest.mark.parametrize("root_key", ["systems_path", "systems_paths"]) +def test_reloaded_prediction_consumes_relative_systems_roots(local_profiles, root_key, tmp_path, monkeypatch): + fast, slow = local_profiles + monkeypatch.chdir(fast.parent) + raw = _local_request(fast) + raw["engine"].pop("systems_path") + raw["engine"][root_key] = fast.name if root_key == "systems_path" else [fast.name, slow.name] + saved = CorePredictionConfig.model_validate(raw).model_dump(mode="json", exclude_none=True) + other_directory = tmp_path / "other" + other_directory.mkdir() + monkeypatch.chdir(other_directory) + + report = _predict(saved) + + assert report.metrics["completed_requests"] == 1 + assert report.metrics["mean_ttft_ms"] == pytest.approx(40.0) + assert report.metrics["mean_itl_ms"] == pytest.approx(20.0) + deployment = prediction_to_replay_spec(CorePredictionConfig.model_validate(saved)).backend_deployment + expected = [str(fast)] if root_key == "systems_path" else [str(fast), str(slow)] + assert deployment.performance_model_metadata["aggregated"]["config"]["systems_paths"] == expected + + @pytest.mark.parametrize("mode", ["aggregated", "disaggregated"]) def test_prediction_consumes_local_fpm_timings_without_changing_default_roots(local_profiles, mode): default_roots = perf_database.get_systems_paths() From 8f31886665878e69addc1c819e6e19d3542e3bd5 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Tue, 29 Sep 2026 02:28:12 +0800 Subject: [PATCH 19/27] fix: address FPM profile review feedback Signed-off-by: Yiming Liu --- crates/core/src/engine/timing.rs | 14 + crates/core/src/lib.rs | 5 + .../core/src/perfmodel/engine/diagnostics.rs | 3 + crates/core/src/perfmodel/engine/readiness.rs | 68 ++- crates/core/src/perfmodel/engine/runtime.rs | 290 +++++++++-- crates/core/src/perfmodel/fpm/estimator.rs | 294 ++++++++++++ crates/core/src/perfmodel/fpm/evidence.rs | 97 ++++ crates/core/src/perfmodel/fpm/mod.rs | 3 + crates/core/src/perfmodel/fpm/model.rs | 57 +++ crates/core/src/perfmodel/mod.rs | 5 + crates/core/src/perfmodel/operators/base.rs | 14 + .../src/perfmodel/operators/fpm_forward.rs | 328 +++++++++++-- crates/core/src/perfmodel/py.rs | 90 ++-- crates/core/src/python.rs | 232 ++++++++- docs/cli/prediction-details.schema.json | 237 +++++++++ docs/core-api.md | 56 ++- docs/fpm-self-service.md | 28 +- .../collector/fpm_forward/runner.py | 5 +- python/aisimulate/src/aisimulate/detail.py | 76 ++- python/aisimulate/src/aisimulate/recommend.py | 6 +- python/aisimulate/src/aisimulate/runner.py | 1 + .../src/aisimulate/sweeper/search.py | 28 +- .../src/aisimulate_core/sdk/fpm_profile.py | 61 ++- .../aisimulate_core/sdk/rust_engine_step.py | 37 +- .../src/aisimulate_core/sdk/utils.py | 5 +- .../cross_package/test_import_contract.py | 5 +- .../collector/test_fpm_profile_collection.py | 45 +- .../tests/unit/sdk/test_fpm_forward.py | 10 +- .../unit/sdk/test_fpm_legacy_migration.py | 252 ++++++++++ .../unit/sdk/test_fpm_phase_readiness.py | 177 +++++++ .../tests/unit/sdk/test_fpm_profile.py | 227 ++++++++- .../aisimulate/tests/unit/sdk/test_utils.py | 24 +- .../tests/unit/test_fpm_config_imports.py | 3 +- tests/test_fpm_profile_workflow.py | 6 +- tests/test_fpm_query_evidence.py | 454 ++++++++++++++++++ 35 files changed, 3057 insertions(+), 186 deletions(-) create mode 100644 crates/core/src/perfmodel/fpm/evidence.rs create mode 100644 python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py create mode 100644 python/aisimulate/tests/unit/sdk/test_fpm_phase_readiness.py create mode 100644 tests/test_fpm_query_evidence.py diff --git a/crates/core/src/engine/timing.rs b/crates/core/src/engine/timing.rs index 34c7a1a070..00c4105a51 100644 --- a/crates/core/src/engine/timing.rs +++ b/crates/core/src/engine/timing.rs @@ -199,6 +199,11 @@ impl TimingOperationEvidence { left.fallbacks.push(fallback); } } + crate::perfmodel::fpm::accumulate_fpm_estimates( + &mut left.fpm_estimates, + right.fpm_estimates, + ) + .map_err(anyhow::Error::msg)?; Some(left) } (Some(mut details), None) | (None, Some(mut details)) => { @@ -488,6 +493,12 @@ pub trait TimingModel: Send + Sync { None } + /// Latency-only whole-model FPM evidence is separate from energy evidence. + /// It must not change the provider's existing power publication contract. + fn fpm_evidence_summary(&self) -> Option { + None + } + /// Start a new measurement epoch without changing timing predictions or /// provider caches. Called only after preparation work has settled. /// @@ -708,6 +719,7 @@ mod tests { TimingOperationEvidence::new("test", 2.0, None, TimingEvidenceSource::Estimated) .unwrap(); op.details = Some(OperationDetails { + fpm_estimates: Vec::new(), sol: has_sol.then_some(SolDiagnostics { latency_ms: 1.0, math_ms: 0.0, @@ -766,6 +778,7 @@ mod tests { ) .unwrap(); operation.details = Some(OperationDetails { + fpm_estimates: Vec::new(), sol: None, sol_unavailable_reason: Some("unsupported operation".into()), fallbacks: vec![valid.clone(), fallback], @@ -787,6 +800,7 @@ mod tests { .unwrap(); let mut detailed = plain.clone(); detailed.details = Some(OperationDetails { + fpm_estimates: Vec::new(), sol: Some(SolDiagnostics { latency_ms: 1.0, math_ms: 0.0, diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 99025ae6dc..256b4f7033 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -75,3 +75,8 @@ pub(crate) use perfmodel::{ #[cfg(feature = "python")] #[allow(unused_imports)] pub(crate) use perfmodel::{py, py_ops}; + +pub use perfmodel::{ + DirectFpmQueryEvidence, ForwardPassEstimate, FpmCoordinates, FpmEstimateEvidence, + FpmMeasurementSupport, FpmQueryResolution, FpmRankEstimate, +}; diff --git a/crates/core/src/perfmodel/engine/diagnostics.rs b/crates/core/src/perfmodel/engine/diagnostics.rs index 8200370678..afcf51acee 100644 --- a/crates/core/src/perfmodel/engine/diagnostics.rs +++ b/crates/core/src/perfmodel/engine/diagnostics.rs @@ -27,6 +27,9 @@ pub struct OperationDetails { pub sol_unavailable_reason: Option, /// Executed measurement substitutions; empty means no recorded substitution. pub fallbacks: Vec, + /// Whole-forward estimates, counted by identical result and replay scale. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub fpm_estimates: Vec, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/crates/core/src/perfmodel/engine/readiness.rs b/crates/core/src/perfmodel/engine/readiness.rs index cfa40c0b51..022e07d005 100644 --- a/crates/core/src/perfmodel/engine/readiness.rs +++ b/crates/core/src/perfmodel/engine/readiness.rs @@ -10,7 +10,7 @@ use crate::AicError; use crate::common::enums::{DatabaseMode, GemmQuantMode, TransferKind}; use crate::config::PerfSource; use crate::operators::Op; -use crate::operators::fpm_forward::{FpmForwardOp, FpmInterpolation}; +use crate::operators::fpm_forward::{FpmForwardOp, FpmInterpolation, FpmPhase}; use crate::perf_database::{PerfDatabase, kernel_source_ok, parquet_loader::PerfReader}; pub(super) fn validate<'a>( @@ -32,11 +32,16 @@ pub(super) fn validate_fpm(db: &PerfDatabase, fpm: &FpmForwardOp) -> Result<(), .fpm_forward .select_cell(&fpm.match_identity, &fpm.model_path)?; if fpm.interpolation == FpmInterpolation::Direct - && cell.direct_prefill.is_empty() - && cell.direct_decode.is_empty() + && match fpm.phase { + FpmPhase::Prefill => cell.direct_prefill.is_empty(), + FpmPhase::Decode => cell.direct_decode.is_empty(), + } { + let phase = fpm.phase.as_str(); return Err(AicError::PerfDatabase(format!( - "direct FPM interpolation has no genuine measurements for {:?} at {}", + "direct FPM {phase} interpolation has no genuine measurements for {:?} at {}. \ + Collect genuine {phase} FPM rows for this model and identity, or provide a \ + systems root containing them.", fpm.model_path, db.data_root.display(), ))); @@ -364,6 +369,61 @@ mod tests { write_parquet(path, &[Col::Str("kernel_source", vec!["fixture"])]); } + #[test] + fn direct_fpm_readiness_requires_genuine_rows_only_for_requested_phases() { + use crate::perf_database::fpm_forward::tests::{RowSpec, default_identity, write_pair}; + + for missing_phase in [FpmPhase::Prefill, FpmPhase::Decode] { + for fake_only in [false, true] { + let root = systems(); + let data = root.path().join("data/b200_sxm/vllm/0.25.1"); + std::fs::create_dir_all(&data).unwrap(); + let present_phase = match missing_phase { + FpmPhase::Prefill => FpmPhase::Decode, + FpmPhase::Decode => FpmPhase::Prefill, + }; + let row = |phase: FpmPhase, regime| RowSpec { + workload_kind: phase.as_str(), + total_prefill_tokens: if phase == FpmPhase::Prefill { 8 } else { 0 }, + kv_seed_regime: Some(regime), + ..RowSpec::default() + }; + let mut rows = vec![row(present_phase, "real_kv")]; + if fake_only { + rows.push(row(missing_phase, "fake_fallback")); + } + write_pair(&data, &rows); + let db = PerfDatabase::load(root.path(), "b200_sxm", "vllm", "0.25.1").unwrap(); + let op = |phase: FpmPhase| { + Op::FpmForward(FpmForwardOp { + name: format!("fpm_forward_{}", phase.as_str()), + phase, + model_path: "org/model-a".into(), + match_identity: default_identity(4), + weight_bytes: 0.0, + verify_width: 1, + interpolation: FpmInterpolation::Direct, + sol_ops: vec![], + original_fmha_quant_mode: None, + }) + }; + let present = op(present_phase); + let missing = op(missing_phase); + // A single-phase operation requires only its own measurements. + validate(&db, [&present].into_iter()).unwrap(); + for requested in [vec![&missing], vec![&present, &missing]] { + let error = validate(&db, requested.into_iter()) + .unwrap_err() + .to_string(); + assert!(error.contains(missing_phase.as_str()), "{error}"); + assert!(error.contains("no genuine measurements"), "{error}"); + assert!(error.contains("Collect genuine"), "{error}"); + assert!(error.contains(data.to_str().unwrap()), "{error}"); + } + } + } + } + #[test] fn dsv41_stage_requires_measured_module_for_silicon() { use crate::operators::dsv41::{Dsv41LinearOp, Dsv41StageOp}; diff --git a/crates/core/src/perfmodel/engine/runtime.rs b/crates/core/src/perfmodel/engine/runtime.rs index 30418adf8d..8ad07d7cf3 100644 --- a/crates/core/src/perfmodel/engine/runtime.rs +++ b/crates/core/src/perfmodel/engine/runtime.rs @@ -612,6 +612,14 @@ impl Engine { /// Python `_run_context_phase` (`base_backend.py:144`): `effective_isl = /// isl - prefix`, validate `> 0`, then one full pass over `context_ops`. fn run_context_phase(&self, runtime: &RuntimeConfig) -> Result { + self.run_context_phase_with(runtime, |_, _| {}) + } + + fn run_context_phase_with( + &self, + runtime: &RuntimeConfig, + on_op: impl FnMut(&Op, PerformanceResult), + ) -> Result { // Python raises `ValueError` when `effective_isl <= 0`; mirror that. if runtime.prefix >= runtime.isl { return Err(AicError::InvalidEngineConfig(format!( @@ -620,7 +628,7 @@ impl Engine { ))); } let effective_isl = runtime.isl - runtime.prefix; - run_context_ops( + run_context_ops_with( &self.context_ops, &self.db, runtime.batch_size, @@ -628,6 +636,7 @@ impl Engine { runtime.prefix, runtime.seq_imbalance_correction_scale, ContextOpFilter::All, + on_op, ) } @@ -727,6 +736,42 @@ impl Engine { .total_ms) } + /// Preserve static prefill geometry, including explicitly equal per-request + /// lengths for bounded decoder replay. Aggregate telemetry cannot express + /// that information and retains its separate validation. + pub(crate) fn predict_prefill_detailed( + &self, + bs: u32, + isl: u32, + prefix: u32, + ) -> Result { + let runtime = RuntimeConfig { + batch_size: bs, + isl, + prefix, + osl: 1, + ..Default::default() + }; + let mut queries = Vec::new(); + let prefill_ms = self.run_context_phase_with(&runtime, |_, result| { + queries.extend(result.fpm_queries.into_iter().flat_map(|queries| *queries)); + })?; + let latency_ms = prefill_ms + 0.0; // same context + generation sum as run_static + Ok(crate::ForwardPassEstimate { + latency_ms: Some(latency_ms), + native_latency_ms: Some(latency_ms), + correction_factor: Some(1.0), + max_rank: (latency_ms > 0.0).then_some(0), + ranks: vec![crate::FpmRankEstimate { + rank: 0, + latency_ms, + prefill_ms: Some(prefill_ms), + queries, + ..Default::default() + }], + }) + } + /// Mocker H2: decode-step latency in ms. Pure-Rust inherent method (no /// PyO3 `py` token). Thin shim over [`Self::run_static`] with /// `mode=Generation`. Mocker passes `osl=2` (one decode step at @@ -1501,6 +1546,7 @@ impl Engine { energy_wms, source: source.into(), details: OperationDetails { + fpm_estimates: Vec::new(), sol, sol_unavailable_reason, fallbacks, @@ -1999,6 +2045,15 @@ impl Engine { &self, metrics_by_rank: &[ForwardPassMetrics], ) -> Result { + self.forward_pass_estimate(metrics_by_rank, false) + .map(|estimate| estimate.native_latency_ms.unwrap()) + } + + pub(crate) fn forward_pass_estimate( + &self, + metrics_by_rank: &[ForwardPassMetrics], + capture: bool, + ) -> Result { if metrics_by_rank.is_empty() { return Err(AicError::InvalidForwardPassMetrics( "at least one attention-DP rank metric required".to_string(), @@ -2008,13 +2063,30 @@ impl Engine { validate_forward_pass_metrics(metrics)?; } let mut max_latency = 0.0_f64; - for metrics in metrics_by_rank { - let rank_latency = self.rank_latency_ms(metrics)?; + let mut max_rank = None; + let mut ranks = Vec::new(); + for (rank, metrics) in metrics_by_rank.iter().enumerate() { + let mut evidence = crate::FpmRankEstimate { + rank, + ..Default::default() + }; + let rank_latency = self.rank_latency_ms(metrics, capture.then_some(&mut evidence))?; if rank_latency > max_latency { max_latency = rank_latency; + max_rank = Some(rank); + } + if capture && self.fpm_ops().is_some() { + evidence.latency_ms = rank_latency; + ranks.push(evidence); } } - Ok(max_latency) + Ok(crate::ForwardPassEstimate { + latency_ms: Some(max_latency), + native_latency_ms: Some(max_latency), + correction_factor: Some(1.0), + max_rank, + ranks, + }) } /// Dispatch one rank's FPM on its scheduled workload. Literal port of @@ -2023,7 +2095,11 @@ impl Engine { /// [`run_context_ops`]; decode-only -> [`run_generation_ops_step`]. The FPM /// counts pass through unscaled (no `nextn` multiplier — see /// [`Self::forward_pass_time_ms`]). - fn rank_latency_ms(&self, metrics: &ForwardPassMetrics) -> Result { + fn rank_latency_ms( + &self, + metrics: &ForwardPassMetrics, + mut evidence: Option<&mut crate::FpmRankEstimate>, + ) -> Result { let sched = &metrics.scheduled_requests; // Token-based dispatch, aligned with `IterationFeatures` (fpm/model.rs): // a fully prefix-cached payload can retain prefill request/KV metadata @@ -2088,39 +2164,58 @@ impl Engine { // division on each axis. let mut total = 0.0_f64; if has_prefill { - total += prefill_op - .query_totals( - &self.db, - &[ - sched.num_prefill_requests as f64, - sched.sum_prefill_tokens as f64, - sched.sum_prefill_kv_tokens as f64, - ], - )? - .latency_ms; + let result = prefill_op.query_totals( + &self.db, + &[ + sched.num_prefill_requests as f64, + sched.sum_prefill_tokens as f64, + sched.sum_prefill_kv_tokens as f64, + ], + )?; + total += result.latency_ms; + if let Some(evidence) = evidence.as_deref_mut() { + evidence.prefill_ms = Some(result.latency_ms); + evidence + .queries + .extend(result.fpm_queries.into_iter().flat_map(|queries| *queries)); + } } if has_decode { - let decode_ms = decode_op - .query_totals( - &self.db, - &[ - sched.num_decode_requests as f64, - sched.sum_decode_kv_tokens as f64, - ], - )? - .latency_ms; + let result = decode_op.query_totals( + &self.db, + &[ + sched.num_decode_requests as f64, + sched.sum_decode_kv_tokens as f64, + ], + )?; + let decode_ms = result.latency_ms; + if let Some(evidence) = evidence.as_deref_mut() { + evidence.decode_ms = Some(decode_ms); + evidence + .queries + .extend(result.fpm_queries.into_iter().flat_map(|queries| *queries)); + } if has_prefill { // Mixed rank: marginal-decode composition, mirroring // `_get_fpm_mix_step_latency` (counts already packed, no // `(nextn + 1)` — speculative FPM was rejected above). - let baseline_ms = decode_op - .query_pass_baseline( - &self.db, - sched.num_decode_requests, - sched.sum_decode_kv_tokens as f64, - )? - .latency_ms; + let baseline = decode_op.query_pass_baseline( + &self.db, + sched.num_decode_requests, + sched.sum_decode_kv_tokens as f64, + )?; + let baseline_ms = baseline.latency_ms; total += (decode_ms - baseline_ms).max(0.0); + if let Some(evidence) = evidence.as_deref_mut() { + evidence.decode_baseline_ms = Some(baseline_ms); + evidence.marginal_decode_ms = Some((decode_ms - baseline_ms).max(0.0)); + evidence.queries.extend( + baseline + .fpm_queries + .into_iter() + .flat_map(|queries| *queries), + ); + } } else { total += decode_ms; } @@ -3620,6 +3715,27 @@ mod tests { .context_ms, 43.0 ); + let detailed = engine + .predict_prefill_detailed(input.batch_size, input.isl, input.prefix) + .unwrap(); + assert_eq!(detailed.latency_ms, Some(43.0)); + assert_eq!( + engine + .predict_prefill_latency(input.batch_size, input.isl, input.prefix) + .unwrap(), + 43.0 + ); + // Capturing the known static geometry does not weaken the aggregate + // telemetry guard for multi-request bounded decoder replay. + assert!( + engine + .forward_pass_estimate(&[fpm_replay_guard_metrics(0.0)], true) + .is_err() + ); + for (isl, prefix) in [(0, 0), (512, 512)] { + assert!(engine.predict_prefill_latency(2, isl, prefix).is_err()); + assert!(engine.predict_prefill_detailed(2, isl, prefix).is_err()); + } } #[test] @@ -3786,6 +3902,118 @@ mod tests { /// Mixed telemetry may request a synthetic decode baseline below the KV /// floor of its padded bracket rows. Only that baseline holds each row at /// its measured floor; the actual decode query remains in-range and strict. + #[test] + fn direct_fpm_detailed_result_explains_mixed_rank_max_and_correction() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = vec![ + RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: 4, + total_kv_read_tokens: 0, + latency_ms: 10.0, + ..Default::default() + }, + RowSpec { + batch_size: 2, + total_kv_read_tokens: 4, + latency_ms: 10.0, + ..Default::default() + }, + RowSpec { + batch_size: 2, + total_kv_read_tokens: 20, + latency_ms: 26.0, + ..Default::default() + }, + RowSpec { + batch_size: 4, + total_kv_read_tokens: 8, + latency_ms: 20.0, + ..Default::default() + }, + RowSpec { + batch_size: 4, + total_kv_read_tokens: 24, + latency_ms: 52.0, + ..Default::default() + }, + ]; + let tmp = tempfile::tempdir().unwrap(); + let mut engine = build_fpm_engine_with_rows(tmp.path(), &rows).unwrap(); + for op in engine + .context_ops + .iter_mut() + .chain(engine.generation_ops.iter_mut()) + { + if let Op::FpmForward(op) = op { + op.interpolation = + crate::perfmodel::operators::fpm_forward::FpmInterpolation::Direct; + } + } + let mut model = crate::ForwardPassPerfModel::from_engine( + Arc::new(engine), + crate::ForwardPassPerfOptions { + min_observations: 1, + ..Default::default() + }, + ); + let mixed = ForwardPassMetrics { + scheduled_requests: crate::ScheduledRequestMetrics { + num_prefill_requests: 1, + sum_prefill_tokens: 4, + num_decode_requests: 3, + sum_decode_kv_tokens: 12, + ..Default::default() + }, + ..Default::default() + }; + let decode = ForwardPassMetrics { + scheduled_requests: crate::ScheduledRequestMetrics { + num_decode_requests: 4, + sum_decode_kv_tokens: 24, + ..Default::default() + }, + ..Default::default() + }; + let metrics = [mixed, decode]; + let result = model.estimate_forward_pass_detailed(&metrics).unwrap(); + assert_eq!( + result.latency_ms, + model.estimate_forward_pass_time_ms(&metrics).unwrap() + ); + assert_eq!(result.latency_ms, Some(52.0)); + assert_eq!(result.max_rank, Some(1)); + let rank = &result.ranks[0]; + assert_eq!( + ( + rank.prefill_ms, + rank.decode_ms, + rank.decode_baseline_ms, + rank.marginal_decode_ms, + rank.latency_ms + ), + (Some(10.0), Some(23.0), Some(15.0), Some(8.0), 18.0) + ); + assert_eq!(rank.queries.len(), 3); + let mut observed = metrics.clone(); + observed[1].wall_time = 0.104; + model.tune_with_fpms(&[observed.to_vec()]).unwrap(); + let corrected = model.estimate_forward_pass_detailed(&metrics).unwrap(); + assert_eq!(corrected.correction_factor, Some(2.0)); + assert_eq!(corrected.latency_ms, Some(104.0)); + assert_eq!(corrected.ranks, result.ranks); + assert_eq!( + corrected.latency_ms, + model.estimate_forward_pass_time_ms(&metrics).unwrap() + ); + let empty = model + .estimate_forward_pass_detailed(&[ForwardPassMetrics::default()]) + .unwrap(); + assert_eq!(empty.latency_ms, Some(0.0)); + assert!(empty.ranks.iter().all(|rank| rank.queries.is_empty())); + } + #[test] fn fpm_rank_mixed_baseline_holds_bracket_curve_floors() { use crate::fpm::{ForwardPassMetrics, ScheduledRequestMetrics}; diff --git a/crates/core/src/perfmodel/fpm/estimator.rs b/crates/core/src/perfmodel/fpm/estimator.rs index 7e1c8a0f2a..eeb7f4ee6b 100644 --- a/crates/core/src/perfmodel/fpm/estimator.rs +++ b/crates/core/src/perfmodel/fpm/estimator.rs @@ -201,6 +201,128 @@ impl Default for CorrectionConfig { } impl EstimatorConfig { + /// Merge explicit migration inputs before applying canonical defaults. + /// Serialized defaults from parsing one input must not become choices that + /// conflict with a setting supplied by another input. + #[cfg(any(feature = "python", test))] + pub(crate) fn migrate_legacy_inputs( + estimator_json: Option<&str>, + options_json: Option<&str>, + interpolation_method: Option<&str>, + fpm_parquet_path: Option<&str>, + ) -> Result { + let mut explicit = serde_json::json!({}); + if let Some(json) = estimator_json { + let supplied: serde_json::Value = serde_json::from_str(json) + .map_err(|error| invalid_migration("estimator_config", error))?; + let typed: Self = serde_json::from_str(json) + .map_err(|error| invalid_migration("estimator_config", error))?; + // Normalize scalar representations (for example 1 and 1.0) while + // retaining only fields that were actually supplied. + let normalized = serde_json::to_value(typed) + .map_err(|error| invalid_migration("estimator_config", error))?; + explicit = explicit_fields(&supplied, &normalized); + } + if let Some(json) = options_json { + let supplied: serde_json::Value = serde_json::from_str(json) + .map_err(|error| invalid_migration("legacy options", error))?; + let options: ForwardPassPerfOptions = serde_json::from_str(json) + .map_err(|error| invalid_migration("legacy options", error))?; + let normalized = serde_json::to_value(&options) + .map_err(|error| invalid_migration("legacy options", error))?; + let mut migrated = serde_json::json!({}); + for (legacy, paths) in [ + ( + "max_observations", + &[ + "fpm_regression.sampling.max_observations", + "correction.sampling.max_observations", + ][..], + ), + ( + "min_observations", + &[ + "fpm_regression.min_observations", + "correction.min_observations", + ], + ), + ( + "min_faster_correction_factor", + &["correction.factor_bounds.min"], + ), + ( + "max_slower_correction_factor", + &["correction.factor_bounds.max"], + ), + ("max_num_tokens", &["correction.max_num_tokens"]), + ("max_batch_size", &["correction.max_batch_size"]), + ("max_kv_tokens", &["correction.max_kv_tokens"]), + ( + "regression_attention_kv_weight", + &["features.attention_kv_weight"], + ), + ( + "regression_prefill_attention_pair_weight", + &["features.prefill_attention_pair_weight"], + ), + ( + "regression_ffn_token_weight", + &["features.ffn_token_weight"], + ), + ( + "regression_ridge_scale", + &["fpm_regression.fit.singular_ridge_scale"], + ), + ] { + if supplied.get(legacy).is_some() { + for path in paths { + insert_explicit(&mut migrated, path, normalized[legacy].clone()); + } + } + } + if supplied.get("bucket_count").is_some() || supplied.get("bucket_shape").is_some() { + // Bucket shape overrides count in the legacy schema. Validate + // those controls here because count is absent from the new schema. + validate_options(&ForwardPassPerfOptions { + bucket_count: options.bucket_count, + bucket_shape: options.bucket_shape, + ..ForwardPassPerfOptions::default() + })?; + let bins = options.bucket_shape.unwrap_or_else(|| { + let n = super::samples::integer_sqrt(options.bucket_count); + [n, n] + }); + for path in [ + "fpm_regression.sampling.bins_per_axis", + "correction.sampling.bins_per_axis", + ] { + insert_explicit(&mut migrated, path, serde_json::json!(bins)); + } + } + merge_explicit(&mut explicit, migrated, "estimator_config")?; + } + let mut interpolation = serde_json::json!({}); + if let Some(method) = interpolation_method { + insert_explicit( + &mut interpolation, + "fpm_interpolation.method", + serde_json::json!(method), + ); + } + if let Some(path) = fpm_parquet_path { + insert_explicit( + &mut interpolation, + "fpm_interpolation.fpm_parquet_path", + serde_json::json!(path), + ); + } + merge_explicit(&mut explicit, interpolation, "estimator_config")?; + let config: Self = serde_json::from_value(explicit) + .map_err(|error| invalid_migration("estimator_config", error))?; + config.validate()?; + Ok(config) + } + pub(crate) fn correction_options(&self) -> ForwardPassPerfOptions { let config = &self.correction; ForwardPassPerfOptions { @@ -288,3 +410,175 @@ impl EstimatorConfig { }) } } + +#[cfg(any(feature = "python", test))] +fn invalid_migration(field: &str, error: serde_json::Error) -> AicError { + AicError::InvalidEngineConfig(format!("invalid {field}: {error}")) +} + +#[cfg(any(feature = "python", test))] +fn explicit_fields( + supplied: &serde_json::Value, + normalized: &serde_json::Value, +) -> serde_json::Value { + match supplied { + serde_json::Value::Object(fields) => serde_json::Value::Object( + fields + .iter() + .map(|(key, value)| (key.clone(), explicit_fields(value, &normalized[key]))) + .collect(), + ), + _ => normalized.clone(), + } +} + +#[cfg(any(feature = "python", test))] +fn insert_explicit(root: &mut serde_json::Value, path: &str, value: serde_json::Value) { + if let Some((field, rest)) = path.split_once('.') { + if root.get(field).is_none() { + root[field] = serde_json::json!({}); + } + insert_explicit(&mut root[field], rest, value); + } else { + root[path] = value; + } +} + +#[cfg(any(feature = "python", test))] +fn merge_explicit( + target: &mut serde_json::Value, + incoming: serde_json::Value, + path: &str, +) -> Result<(), AicError> { + if let (Some(target_fields), Some(incoming_fields)) = + (target.as_object_mut(), incoming.as_object()) + { + for (key, value) in incoming_fields { + if let Some(existing) = target_fields.get_mut(key) { + merge_explicit(existing, value.clone(), &format!("{path}.{key}"))?; + } else { + target_fields.insert(key.clone(), value.clone()); + } + } + } else if *target != incoming { + if path == "estimator_config.fpm_interpolation.fpm_parquet_path" + && target + .as_str() + .zip(incoming.as_str()) + .is_some_and(|(saved, supplied)| { + std::path::Path::new(saved) == std::path::Path::new(supplied) + }) + { + return Ok(()); + } + return Err(AicError::InvalidEngineConfig(format!( + "conflicting explicit values for {path}" + ))); + } + Ok(()) +} + +#[cfg(test)] +mod migration_tests { + use super::{EstimatorConfig, FpmInterpolationMethod}; + + #[test] + fn migration_defaults_are_applied_after_explicit_inputs_merge() { + let saved = r#"{ + "fpm_interpolation": {"method": "direct"}, + "correction": {"enabled": false, "sampling": {"max_observations": 128}}, + "fpm_regression": {"sampling": {"max_observations": 128}} + }"#; + let options = r#"{"min_observations": 80, "regression_ridge_scale": 0}"#; + let config = + EstimatorConfig::migrate_legacy_inputs(Some(saved), Some(options), None, None).unwrap(); + assert_eq!( + config.fpm_interpolation.method, + FpmInterpolationMethod::Direct + ); + assert!(!config.correction.enabled); + assert_eq!(config.correction.min_observations, 80); + assert_eq!(config.fpm_regression.min_observations, 80); + assert_eq!(config.fpm_regression.fit.singular_ridge_scale, 0.0); + assert_eq!(config.features.attention_kv_weight, 1.0); + let reloaded: EstimatorConfig = + serde_json::from_str(&serde_json::to_string(&config).unwrap()).unwrap(); + assert_eq!(reloaded, config); + } + + #[test] + fn migration_accepts_equal_numeric_array_and_null_controls() { + let saved = r#"{ + "features": {"attention_kv_weight": 1}, + "correction": {"factor_bounds": {"min": null}}, + "fpm_regression": {"sampling": {"bins_per_axis": [4, 16]}} + }"#; + let options = r#"{ + "regression_attention_kv_weight": 1.0, + "min_faster_correction_factor": null, + "bucket_shape": [4, 16] + }"#; + let config = + EstimatorConfig::migrate_legacy_inputs(Some(saved), Some(options), None, None).unwrap(); + assert_eq!(config.features.attention_kv_weight, 1.0); + assert_eq!(config.correction.factor_bounds.min, None); + assert_eq!(config.correction.sampling.bins_per_axis, [4, 16]); + assert_eq!(config.fpm_regression.sampling.bins_per_axis, [4, 16]); + } + + #[test] + fn migration_compares_parquet_paths_without_filesystem_resolution() { + let saved = r#"{"fpm_interpolation":{"fpm_parquet_path":"/tmp/fpm.parquet"}}"#; + for legacy in [ + "/tmp/fpm.parquet", + "/tmp/./fpm.parquet", + "/tmp//fpm.parquet", + ] { + let config = + EstimatorConfig::migrate_legacy_inputs(Some(saved), None, None, Some(legacy)) + .unwrap(); + assert_eq!( + config.fpm_interpolation.fpm_parquet_path.unwrap().to_str(), + Some("/tmp/fpm.parquet") + ); + } + for legacy in ["/tmp/different.parquet", "/tmp/subdir/../fpm.parquet"] { + let error = + EstimatorConfig::migrate_legacy_inputs(Some(saved), None, None, Some(legacy)) + .unwrap_err(); + assert!( + error.to_string().contains( + "conflicting explicit values for estimator_config.fpm_interpolation.fpm_parquet_path" + ), + "{error}" + ); + } + } + + #[test] + fn interpolation_conflicts_include_explicit_auto_but_not_omission() { + let config = EstimatorConfig::migrate_legacy_inputs( + Some(r#"{"fpm_interpolation": {}}"#), + None, + Some("direct"), + None, + ) + .unwrap(); + assert_eq!( + config.fpm_interpolation.method, + FpmInterpolationMethod::Direct + ); + for (canonical, legacy) in [("auto", "direct"), ("direct", "auto"), ("sol", "direct")] { + let saved = format!(r#"{{"fpm_interpolation": {{"method": "{canonical}"}}}}"#); + let error = + EstimatorConfig::migrate_legacy_inputs(Some(&saved), None, Some(legacy), None) + .unwrap_err(); + assert!( + error.to_string().contains( + "conflicting explicit values for estimator_config.fpm_interpolation.method" + ), + "{error}" + ); + } + } +} diff --git a/crates/core/src/perfmodel/fpm/evidence.rs b/crates/core/src/perfmodel/fpm/evidence.rs new file mode 100644 index 0000000000..6f8cce55b7 --- /dev/null +++ b/crates/core/src/perfmodel/fpm/evidence.rs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Executed direct-FPM lookups and their forward-pass composition. +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct FpmCoordinates { + pub batch_size: f64, + pub total_prefill_tokens: Option, + pub total_kv_read_tokens: f64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct FpmMeasurementSupport { + pub coordinates: FpmCoordinates, + pub latency_ms: f64, + /// Product of the linear interpolation weights at this measured point. + pub weight: f64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FpmQueryResolution { + ExactLookup, + WithinCurveInterpolation, + CrossKvInterpolation, + CrossBatchInterpolation, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct DirectFpmQueryEvidence { + pub phase: String, + pub model_path: String, + /// True for the mixed-pass baseline: support uses each selected row's + /// measured KV floor, while `query` retains the paired decode request. + pub decode_baseline: bool, + pub query: FpmCoordinates, + pub resolution: FpmQueryResolution, + /// Raw lookup latency, before composition, correction or replay speedup. + pub latency_ms: f64, + pub support: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct FpmRankEstimate { + pub rank: usize, + pub latency_ms: f64, + pub prefill_ms: Option, + pub decode_ms: Option, + pub decode_baseline_ms: Option, + /// For a mixed pass, max(decode_ms - decode_baseline_ms, 0). + pub marginal_decode_ms: Option, + pub queries: Vec, +} + +/// Additive detailed result from the canonical returned model. Native timing +/// reduces ranks by maximum, then applies `correction_factor`. Rank lookup +/// evidence is supplied for whole-model FPM; other estimators leave it empty. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ForwardPassEstimate { + pub latency_ms: Option, + pub native_latency_ms: Option, + pub correction_factor: Option, + /// First rank attaining the positive native maximum; None for empty work. + pub max_rank: Option, + pub ranks: Vec, +} + +/// Identical complete estimates are retained once, with the number of timing +/// invocations that used them. No records are truncated. `latency_scale` is +/// replay's synthetic speedup multiplier and does not alter measured support. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct FpmEstimateEvidence { + pub estimate: ForwardPassEstimate, + pub count: u64, + pub latency_scale: f64, +} + +pub(crate) fn accumulate_fpm_estimates( + target: &mut Vec, + incoming: Vec, +) -> Result<(), &'static str> { + for record in incoming { + if let Some(existing) = target.iter_mut().find(|existing| { + existing.estimate == record.estimate && existing.latency_scale == record.latency_scale + }) { + existing.count = existing + .count + .checked_add(record.count) + .ok_or("FPM evidence invocation count overflow")?; + } else { + target.push(record); + } + } + Ok(()) +} diff --git a/crates/core/src/perfmodel/fpm/mod.rs b/crates/core/src/perfmodel/fpm/mod.rs index 65144f25ba..945862f9cb 100644 --- a/crates/core/src/perfmodel/fpm/mod.rs +++ b/crates/core/src/perfmodel/fpm/mod.rs @@ -25,6 +25,7 @@ mod config; mod correction; mod estimator; +mod evidence; mod metrics; mod model; mod options; @@ -39,6 +40,8 @@ pub use config::{ ForwardPassSpeculationConfig, }; pub use estimator::*; +pub(crate) use evidence::accumulate_fpm_estimates; +pub use evidence::*; pub(crate) use metrics::validate_forward_pass_metrics; pub use metrics::{FPM_VERSION, ForwardPassMetrics, QueuedRequestMetrics, ScheduledRequestMetrics}; pub use model::{ diff --git a/crates/core/src/perfmodel/fpm/model.rs b/crates/core/src/perfmodel/fpm/model.rs index 725918d76f..097665250d 100644 --- a/crates/core/src/perfmodel/fpm/model.rs +++ b/crates/core/src/perfmodel/fpm/model.rs @@ -456,6 +456,50 @@ impl ForwardPassPerfModel { } } + /// Estimate the same iteration with executed direct-FPM support. The + /// native rank maximum and online correction are explicit; measurement + /// weights describe raw lookup values only. No query state is retained. + pub fn estimate_forward_pass_detailed( + &self, + metrics_by_rank: &[ForwardPassMetrics], + ) -> Result { + if let ForwardPassPerfMode::Native { + engine, + corrections, + } = &self.mode + { + let Some(feature) = IterationFeatures::from_metrics(metrics_by_rank)? else { + return Ok(crate::ForwardPassEstimate { + latency_ms: Some(0.0), + native_latency_ms: Some(0.0), + correction_factor: Some(1.0), + max_rank: None, + ranks: Vec::new(), + }); + }; + let mut estimate = engine.forward_pass_estimate(metrics_by_rank, true)?; + let factor = + Some(feature) + .filter(|_| self.is_correction_enabled) + .map_or(1.0, |feature| { + corrections + .store(feature.workload_kind) + .correction_factor_for(&feature.x) + }); + estimate.correction_factor = Some(factor); + estimate.latency_ms = estimate.native_latency_ms.map(|native| native * factor); + Ok(estimate) + } else { + Ok(crate::ForwardPassEstimate { + latency_ms: self.estimate_forward_pass_time_ms(metrics_by_rank)?, + native_latency_ms: None, + correction_factor: None, + max_rank: None, + ranks: Vec::new(), + }) + } + } + /// API: /// `model.tune_with_fpms(iterations) -> Result<(), AicError>` /// @@ -659,6 +703,19 @@ impl ForwardPassPerfModel { } } + pub(crate) fn static_prefill_detailed( + &self, + batch_size: u32, + input_tokens: u32, + prefix: u32, + ) -> Result { + self.native_engine() + .ok_or_else(|| { + AicError::InvalidEngineConfig("static prefill requires a native estimator".into()) + })? + .predict_prefill_detailed(batch_size, input_tokens, prefix) + } + /// Native operation evidence for one static prefill or decode step. Values /// precede learned online correction; SOL is a comparison only. Whole-model /// estimators cannot provide an operation decomposition and fail explicitly. diff --git a/crates/core/src/perfmodel/mod.rs b/crates/core/src/perfmodel/mod.rs index 86b98291f3..e05b422273 100644 --- a/crates/core/src/perfmodel/mod.rs +++ b/crates/core/src/perfmodel/mod.rs @@ -103,3 +103,8 @@ pub(crate) fn repo_relative(rel: &str) -> Option { pub fn register_python(module: &pyo3::Bound<'_, pyo3::types::PyModule>) -> pyo3::PyResult<()> { py::register(module) } + +pub use fpm::{ + DirectFpmQueryEvidence, ForwardPassEstimate, FpmCoordinates, FpmEstimateEvidence, + FpmMeasurementSupport, FpmQueryResolution, FpmRankEstimate, +}; diff --git a/crates/core/src/perfmodel/operators/base.rs b/crates/core/src/perfmodel/operators/base.rs index 6e7214376d..585a5c3e25 100644 --- a/crates/core/src/perfmodel/operators/base.rs +++ b/crates/core/src/perfmodel/operators/base.rs @@ -183,6 +183,8 @@ pub struct PerformanceResult { pub source: Source, pub sol: Option, pub moe_comm_fallbacks: MoeCommFallbacks, + /// Raw direct-FPM leaves; composition never rewrites their measured support. + pub fpm_queries: Option>>, } impl PerformanceResult { @@ -193,6 +195,7 @@ impl PerformanceResult { source, sol: None, moe_comm_fallbacks: MoeCommFallbacks::default(), + fpm_queries: None, } } @@ -203,6 +206,7 @@ impl PerformanceResult { source, sol: None, moe_comm_fallbacks: MoeCommFallbacks::default(), + fpm_queries: None, } } @@ -215,6 +219,7 @@ impl PerformanceResult { source: Source::Sol, sol: Some(components), moe_comm_fallbacks: MoeCommFallbacks::default(), + fpm_queries: None, } } @@ -261,6 +266,7 @@ impl PerformanceResult { mem_ms: c.mem_ms * factor, }), moe_comm_fallbacks: self.moe_comm_fallbacks, + fpm_queries: self.fpm_queries, } } @@ -269,6 +275,12 @@ impl PerformanceResult { /// AND energy both 0.0) is a source-neutral identity — the other /// side's tag survives, mirroring Python's zero-identity rule. pub fn plus(self, other: PerformanceResult) -> Self { + let mut fpm_queries = self.fpm_queries; + if let Some(other_queries) = other.fpm_queries { + fpm_queries + .get_or_insert_with(Default::default) + .extend(*other_queries); + } let mut moe_comm_fallbacks = self.moe_comm_fallbacks; moe_comm_fallbacks.extend(other.moe_comm_fallbacks); let (source, sol) = if self.latency_ms == 0.0 && self.energy_wms == 0.0 { @@ -294,6 +306,7 @@ impl PerformanceResult { source, sol, moe_comm_fallbacks, + fpm_queries, } } @@ -310,6 +323,7 @@ impl PerformanceResult { mem_ms: c.mem_ms.max(0.0), }), moe_comm_fallbacks: self.moe_comm_fallbacks, + fpm_queries: self.fpm_queries, } } } diff --git a/crates/core/src/perfmodel/operators/fpm_forward.rs b/crates/core/src/perfmodel/operators/fpm_forward.rs index dcf5e1926c..41f9897ce7 100644 --- a/crates/core/src/perfmodel/operators/fpm_forward.rs +++ b/crates/core/src/perfmodel/operators/fpm_forward.rs @@ -15,19 +15,20 @@ //! - prefill coords `(B, B*s, B*prefix)` — `s` is NEW prefill tokens per //! request, `prefix` the past-KV per request; //! - decode coords `(B, B*s)` — one new token per request, `s` the per-request -//! KV length at this decode step; -//! - hard per-axis domain gate BEFORE interpolation (FPM never extrapolates); -//! - ScatteredSites resolution with `own_curve_coverage_fallback=true` and -//! `max_site_distance=2.0`; prefill additionally admits sites within 32 raw -//! KV tokens while retaining the normal log2 gate on batch, so small -//! block-aligned coordinates around zero remain connected. +//! KV length at this decode step. //! //! With `interpolation="sol"` (the default), the roofline anchoring transfer //! is the model's ORIGINAL op-level list (`sol_ops` on the wire) queried in //! SOL mode with Python's coordinate back-mapping — see [`sol_total`]. +//! SOL applies a hard per-axis domain gate before ScatteredSites resolution +//! with `own_curve_coverage_fallback=true` and `max_site_distance=2.0`; +//! prefill additionally admits sites within 32 raw KV tokens while retaining +//! the log2 gate on batch, so small block-aligned coordinates remain connected. +//! //! `interpolation="direct"` instead uses raw linear interpolation between -//! genuine measured curves. Prefill stays at one batch with two-sided KV -//! brackets; decode stays inside its inferred capture regime. It never +//! genuine measured curves. Prefill uses the nearest same-batch lower and +//! upper KV sites whose token curves cover the query, without a KV distance +//! limit; decode stays inside its inferred capture regime. It never //! evaluates `sol_ops`, clamps an unsupported batch, or uses fabricated KV. //! //! The canonical `ForwardPassPerfModel` constructor selects this native @@ -37,6 +38,8 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +use crate::{DirectFpmQueryEvidence, FpmCoordinates, FpmMeasurementSupport, FpmQueryResolution}; + use crate::common::error::AicError; use crate::operators::op::{Op, RuntimeContext}; use crate::operators::{PerformanceResult, Source}; @@ -507,7 +510,17 @@ impl FpmForwardOp { .filter(|_| kv == (kv as u32) as f64) .and_then(|curve| direct_curve_value(curve, tokens)) { - return self.direct_result(cell, coords, value); + let support = direct_curve_support( + &cell.direct_prefill[&(batch as u32, kv as u32)], + tokens, + |token| fpm_coordinates(self.phase, &[batch, token, kv]), + ); + let resolution = if support.len() == 1 { + FpmQueryResolution::ExactLookup + } else { + FpmQueryResolution::WithinCurveInterpolation + }; + return self.direct_result(cell, coords, value, resolution, support, false); } let covered: Vec<(u32, f64)> = cell .direct_prefill @@ -516,25 +529,14 @@ impl FpmForwardOp { direct_curve_value(curve, tokens).map(|value| (site_kv, value)) }) .collect(); - // Preserve the original raw-KV method's supported values. Only when - // its two-sided bracket is missing, widen to the nearest covered - // lower and upper KV sites at this exact batch. - let bracket = |guarded: bool| { - let admitted = |site_kv: u32| { - !guarded - || ((site_kv as f64).max(1e-12).log2() - kv.max(1e-12).log2()).abs() <= 2.0 - || (site_kv as f64 - kv).abs() <= 32.0 - }; - let lower = covered - .iter() - .rev() - .find(|&&(site_kv, _)| (site_kv as f64) < kv && admitted(site_kv)); - let upper = covered - .iter() - .find(|&&(site_kv, _)| (site_kv as f64) > kv && admitted(site_kv)); - lower.zip(upper) - }; - let Some((&(lo, lo_ms), &(hi, hi_ms))) = bracket(true).or_else(|| bracket(false)) else { + // The map orders sites by KV. Direct interpolation uses the nearest + // token-covered site on each side, with no SOL site-distance guard. + let lower = covered + .iter() + .rev() + .find(|&&(site_kv, _)| (site_kv as f64) < kv); + let upper = covered.iter().find(|&&(site_kv, _)| (site_kv as f64) > kv); + let Some((&(lo, lo_ms), &(hi, hi_ms))) = lower.zip(upper) else { return Err(self.direct_coverage_err( cell, coords, @@ -543,7 +545,29 @@ impl FpmForwardOp { }; let weight = (kv - lo as f64) / (hi as f64 - lo as f64); let latency = lo_ms + (hi_ms - lo_ms) * weight; - self.direct_result(cell, coords, latency) + let mut support = Vec::new(); + for (site_kv, site_weight) in [(lo, 1.0 - weight), (hi, weight)] { + support.extend( + direct_curve_support( + &cell.direct_prefill[&(batch as u32, site_kv)], + tokens, + |token| fpm_coordinates(self.phase, &[batch, token, site_kv as f64]), + ) + .into_iter() + .map(|mut point| { + point.weight *= site_weight; + point + }), + ); + } + self.direct_result( + cell, + coords, + latency, + FpmQueryResolution::CrossKvInterpolation, + support, + false, + ) } fn direct_result( @@ -551,6 +575,9 @@ impl FpmForwardOp { cell: &FpmForwardCell, coords: &[f64], latency: f64, + resolution: FpmQueryResolution, + support: Vec, + decode_baseline: bool, ) -> Result { if !latency.is_finite() || latency <= 0.0 { return Err(self.direct_coverage_err( @@ -559,7 +586,17 @@ impl FpmForwardOp { &format!("interpolation produced an invalid latency ({latency})"), )); } - Ok(PerformanceResult::new(latency, Source::Silicon)) + let mut result = PerformanceResult::new(latency, Source::Silicon); + result.fpm_queries = Some(Box::new(vec![DirectFpmQueryEvidence { + phase: self.phase.as_str().into(), + model_path: cell.model_path.clone(), + decode_baseline, + query: fpm_coordinates(self.phase, coords), + resolution, + latency_ms: latency, + support, + }])); + Ok(result) } fn direct_coverage_err(&self, cell: &FpmForwardCell, coords: &[f64], reason: &str) -> AicError { @@ -603,7 +640,39 @@ impl FpmForwardOp { let weight = (coords[0] - lo_row as f64) / (hi_row as f64 - lo_row as f64); lo_ms + (row_value(hi_row) - lo_ms) * weight }; - self.direct_result(cell, coords, latency) + let rows = if lo_row == hi_row { + vec![(lo_row, 1.0)] + } else { + let weight = (coords[0] - lo_row as f64) / (hi_row as f64 - lo_row as f64); + vec![(lo_row, 1.0 - weight), (hi_row, weight)] + }; + let mut support = Vec::new(); + for (row, row_weight) in rows { + let curve = &cell.direct_decode[&row]; + let kv = if baseline { + *curve.keys().next().expect("nonempty measured curve") as f64 + } else { + coords[1] + }; + support.extend( + direct_curve_support(curve, kv, |kv| { + fpm_coordinates(self.phase, &[row as f64, kv]) + }) + .into_iter() + .map(|mut point| { + point.weight *= row_weight; + point + }), + ); + } + let resolution = if lo_row != hi_row { + FpmQueryResolution::CrossBatchInterpolation + } else if support.len() == 1 { + FpmQueryResolution::ExactLookup + } else { + FpmQueryResolution::WithinCurveInterpolation + }; + self.direct_result(cell, coords, latency, resolution, support, baseline) } /// One shared coverage decision for decode queries and their mixed-pass @@ -765,6 +834,51 @@ impl FpmForwardOp { /// Exact lookup or raw linear interpolation strictly inside one measured /// curve. There is deliberately no callback, boundary hold, or SOL input. +fn fpm_coordinates(phase: FpmPhase, coords: &[f64]) -> FpmCoordinates { + FpmCoordinates { + batch_size: coords[0], + total_prefill_tokens: (phase == FpmPhase::Prefill).then(|| coords[1]), + total_kv_read_tokens: coords[coords.len() - 1], + } +} + +/// Read support only after coverage and latency have been resolved. Keep this +/// separate from the latency arithmetic so existing evaluation order is intact. +fn direct_curve_support( + curve: &BTreeMap, + coordinate: f64, + coordinates: impl Fn(f64) -> FpmCoordinates, +) -> Vec { + let (&lo, &lo_ms) = curve + .range(..=(coordinate as u32)) + .next_back() + .expect("covered curve"); + if coordinate == lo as f64 { + return vec![FpmMeasurementSupport { + coordinates: coordinates(lo as f64), + latency_ms: lo_ms, + weight: 1.0, + }]; + } + let (&hi, &hi_ms) = curve + .range((std::ops::Bound::Excluded(lo), std::ops::Bound::Unbounded)) + .next() + .expect("covered curve"); + let weight = (coordinate - lo as f64) / (hi as f64 - lo as f64); + vec![ + FpmMeasurementSupport { + coordinates: coordinates(lo as f64), + latency_ms: lo_ms, + weight: 1.0 - weight, + }, + FpmMeasurementSupport { + coordinates: coordinates(hi as f64), + latency_ms: hi_ms, + weight, + }, + ] +} + fn direct_curve_value(curve: &BTreeMap, coordinate: f64) -> Option { if !coordinate.is_finite() || coordinate < 0.0 || coordinate > u32::MAX as f64 { return None; @@ -917,6 +1031,156 @@ mod tests { } } + #[test] + fn direct_query_evidence_preserves_support_and_nested_weights() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let mut rows = Vec::new(); + for (tokens, kv, latency_ms) in [(4, 0, 10.0), (12, 0, 18.0), (4, 8, 30.0), (12, 8, 46.0)] { + rows.push(RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: tokens, + total_kv_read_tokens: kv, + latency_ms, + ..Default::default() + }); + } + for (batch_size, kv, latency_ms) in + [(2, 4, 10.0), (2, 20, 26.0), (4, 8, 20.0), (4, 24, 52.0)] + { + rows.push(RowSpec { + batch_size, + total_kv_read_tokens: kv, + latency_ms, + ..Default::default() + }); + } + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut prefill = op(FpmPhase::Prefill); + prefill.interpolation = FpmInterpolation::Direct; + // Exact leaf, quarter along the token curve, then one-quarter along KV: + // 12 + (34 - 12) / 4 = 17.5 ms. Combined weights are 9/16,3/16,3/16,1/16. + for (query, latency, resolution, weights) in [ + ( + [1.0, 4.0, 0.0], + 10.0, + FpmQueryResolution::ExactLookup, + vec![1.0], + ), + ( + [1.0, 6.0, 0.0], + 12.0, + FpmQueryResolution::WithinCurveInterpolation, + vec![0.75, 0.25], + ), + ( + [1.0, 6.0, 2.0], + 17.5, + FpmQueryResolution::CrossKvInterpolation, + vec![0.5625, 0.1875, 0.1875, 0.0625], + ), + ] { + let result = prefill.query_totals(&db, &query).unwrap(); + assert_eq!(result.latency_ms, latency); + assert_eq!(result.source, Source::Silicon); + let evidence = &result.fpm_queries.as_ref().unwrap()[0]; + assert_eq!(evidence.resolution, resolution); + assert_eq!( + evidence + .support + .iter() + .map(|point| point.weight) + .collect::>(), + weights + ); + assert_eq!(evidence.query, fpm_coordinates(FpmPhase::Prefill, &query)); + } + let mut decode = op(FpmPhase::Decode); + decode.interpolation = FpmInterpolation::Direct; + let result = decode.query_totals(&db, &[3.0, 12.0]).unwrap(); + assert_eq!(result.latency_ms, 23.0); // midpoint of 18 and 28 ms + let evidence = &result.fpm_queries.as_ref().unwrap()[0]; + assert_eq!( + evidence.resolution, + FpmQueryResolution::CrossBatchInterpolation + ); + assert_eq!( + evidence + .support + .iter() + .map(|p| ( + p.coordinates.batch_size, + p.coordinates.total_kv_read_tokens, + p.latency_ms, + p.weight + )) + .collect::>(), + vec![ + (2.0, 4.0, 10.0, 0.25), + (2.0, 20.0, 26.0, 0.25), + (4.0, 8.0, 20.0, 0.375), + (4.0, 24.0, 52.0, 0.125) + ] + ); + let baseline = decode.query_pass_baseline(&db, 3, 12.0).unwrap(); + assert_eq!(baseline.latency_ms, 15.0); + let evidence = &baseline.fpm_queries.as_ref().unwrap()[0]; + assert!(evidence.decode_baseline); + assert_eq!(evidence.query.total_kv_read_tokens, 12.0); + assert_eq!( + evidence + .support + .iter() + .map(|p| (p.coordinates.total_kv_read_tokens, p.weight)) + .collect::>(), + vec![(4.0, 0.5), (8.0, 0.5)] + ); + assert!(decode.query_totals(&db, &[3.0, 30.0]).is_err()); + } + + #[test] + fn direct_prefill_uses_nearest_token_covered_kv_sites() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = [ + (100, 0, 2.0), + (200, 0, 4.0), + (100, 64, 10.0), + (200, 64, 20.0), + (100, 80, 999.0), + (100, 112, 999.0), + (100, 128, 30.0), + (200, 128, 50.0), + (100, 256, 100.0), + (200, 256, 200.0), + ] + .into_iter() + .map(|(tokens, kv, latency)| RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: tokens, + total_kv_read_tokens: kv, + latency_ms: latency, + ..RowSpec::default() + }) + .collect::>(); + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut direct = op(FpmPhase::Prefill); + direct.interpolation = FpmInterpolation::Direct; + // T=150 gives 15 ms at KV=64 and 40 ms at KV=128. KV=96 is + // their midpoint; closer sites at KV=80/112 do not cover T=150. + assert_eq!( + direct + .query_totals(&db, &[1.0, 150.0, 96.0]) + .unwrap() + .latency_ms, + 27.5 + ); + } + #[test] fn direct_prefill_uses_wider_two_sided_kv_brackets_without_sol() { use crate::perf_database::fpm_forward::tests::RowSpec; @@ -950,7 +1214,7 @@ mod tests { is_pre: true, quant_mode: crate::common::enums::GemmQuantMode::Bfloat16, })]; - // KV=0 is outside the original two-octave/32-token admission gate. + // KV=0 is outside the SOL two-octave/32-token site-distance guard. // First evaluate the two token curves at T=150, then blend raw KV. let result = direct.query_totals(&db, &[1.0, 150.0, 2048.0]).unwrap(); assert_eq!(result.latency_ms, 25.0); diff --git a/crates/core/src/perfmodel/py.rs b/crates/core/src/perfmodel/py.rs index 8048e14ed3..cbacddd4ae 100644 --- a/crates/core/src/perfmodel/py.rs +++ b/crates/core/src/perfmodel/py.rs @@ -1670,11 +1670,18 @@ pub(crate) fn compile_engine_to_engine( )); } let mut request = engine_build_request(config, systems_path)?; - if let Some(value) = config.extra.get("estimator_config") { - let controls: crate::EstimatorConfig = serde_json::from_str(value) - .map_err(|error| AicError::InvalidEngineConfig(format!("estimator_config: {error}")))?; - controls.validate()?; + if config.extra.contains_key("estimator_config") || request.fpm_interpolation.is_some() { + let controls = crate::EstimatorConfig::migrate_legacy_inputs( + config.extra.get("estimator_config").map(String::as_str), + None, + request.fpm_interpolation.as_deref(), + request.fpm_parquet_path.as_deref(), + )?; request.fpm_interpolation = Some(controls.fpm_interpolation.method.as_str().to_owned()); + request.fpm_parquet_path = controls + .fpm_interpolation + .fpm_parquet_path + .map(|path| path.to_str().expect("validated UTF-8 path").to_owned()); } if config.extra.contains_key("fpm_profile") { let mut modes = Python::with_gil(|py| fpm_profile_quantization(py, config)) @@ -1894,11 +1901,6 @@ impl PyForwardPassPerfModel { ) -> PyResult { let legacy: EngineConfig = serde_json::from_str(config_json).map_err(|e| PyValueError::new_err(e.to_string()))?; - let options = options_json - .map(serde_json::from_str::) - .transpose() - .map_err(|e| PyValueError::new_err(e.to_string()))? - .unwrap_or_default(); let mut request = engine_build_request( &legacy, legacy.systems_path.as_ref().and_then(|path| path.to_str()), @@ -1917,38 +1919,13 @@ impl PyForwardPassPerfModel { "cp_size must be the integer 1; this SDK entry point does not support context parallelism", )); } - let mut estimator_config = - crate::EstimatorConfig::from_legacy(options).map_err(aic_to_py)?; - if let Some(value) = legacy.extra.get("estimator_config") { - estimator_config = serde_json::from_str(value) - .map_err(|e| PyValueError::new_err(format!("invalid estimator_config: {e}")))?; - } - if let Some(value) = request.fpm_interpolation.as_ref() { - let method = - serde_json::from_value(serde_json::Value::String(value.clone())).map_err(|e| { - PyValueError::new_err(format!("invalid FPM interpolation method: {e}")) - })?; - if estimator_config.fpm_interpolation.method != crate::FpmInterpolationMethod::Auto - && estimator_config.fpm_interpolation.method != method - { - return Err(PyValueError::new_err( - "fpm_interpolation conflicts with estimator_config.fpm_interpolation.method", - )); - } - estimator_config.fpm_interpolation.method = method; - } - if let Some(path) = request.fpm_parquet_path.as_ref() { - let configured = &mut estimator_config.fpm_interpolation.fpm_parquet_path; - if configured - .as_ref() - .is_some_and(|existing| existing != Path::new(path)) - { - return Err(PyValueError::new_err( - "conflicting fpm_parquet_path and estimator_config.fpm_interpolation.fpm_parquet_path", - )); - } - *configured = Some(PathBuf::from(path)); - } + let estimator_config = crate::EstimatorConfig::migrate_legacy_inputs( + legacy.extra.get("estimator_config").map(String::as_str), + options_json, + request.fpm_interpolation.as_deref(), + request.fpm_parquet_path.as_deref(), + ) + .map_err(aic_to_py)?; let worker_type = serde_json::from_value(serde_json::Value::String(worker_type.to_owned())) .map_err(|e| PyValueError::new_err(format!("invalid worker_type: {e}")))?; let estimation_mode = match request.forward_model.as_deref().unwrap_or("op_level") { @@ -2020,6 +1997,14 @@ impl PyForwardPassPerfModel { .map_err(aic_to_py) } + fn estimate_forward_pass_detailed(&self, py: Python<'_>, fpm_json: &str) -> PyResult { + let metrics = parse_fpm_iteration(fpm_json)?; + let result = py + .allow_threads(|| self.inner.estimate_forward_pass_detailed(&metrics)) + .map_err(aic_to_py)?; + serde_json::to_string(&result).map_err(|error| PyValueError::new_err(error.to_string())) + } + /// Tune from observed FPM iterations. `iterations_json` is the nested list /// `[[iter0_rank0, ...], [iter1_rank0, ...]]` (the wrapper normalizes the /// convenience forms before calling). Pure-Rust compute (GIL freed). @@ -2284,6 +2269,29 @@ mod tests { } } + #[test] + fn legacy_compilation_rejects_conflicting_interpolation_before_entering_python() { + for (canonical, legacy) in [("auto", "direct"), ("direct", "auto"), ("sol", "direct")] { + let mut config = fixture_engine_config(); + config.extra.insert( + "estimator_config".into(), + format!(r#"{{"fpm_interpolation": {{"method": "{canonical}"}}}}"#), + ); + config + .extra + .insert("fpm_interpolation".into(), legacy.into()); + let error = compile_engine_to_engine(&config, None) + .err() + .expect("conflicting interpolation methods"); + assert!( + error.to_string().contains( + "conflicting explicit values for estimator_config.fpm_interpolation.method" + ), + "{error}" + ); + } + } + #[cfg(unix)] #[test] fn native_model_rejects_non_utf8_fpm_path() { diff --git a/crates/core/src/python.rs b/crates/core/src/python.rs index bbfd41ba78..9d4e400711 100644 --- a/crates/core/src/python.rs +++ b/crates/core/src/python.rs @@ -218,7 +218,7 @@ struct AicTimingConfig { #[serde(default)] fallback_policy: ForwardPassFallbackPolicy, #[serde(default)] - estimator_config: EstimatorConfig, + estimator_config: serde_json::Map, #[serde(default)] database_mode: crate::DatabaseMode, #[serde(default)] @@ -278,29 +278,18 @@ impl AicTimingConfig { } else { self.systems_paths.clone() }; - let mut estimator_config = self.estimator_config.clone(); - if let Some(legacy) = self.fpm_interpolation { - let method = &mut estimator_config.fpm_interpolation.method; - ensure!( - *method == crate::FpmInterpolationMethod::Auto || *method == legacy, - "fpm_interpolation conflicts with estimator_config.fpm_interpolation.method" - ); - *method = legacy; - } if let Some(path) = &self.fpm_parquet_path { crate::config::validate_fpm_parquet_path( Some(std::path::Path::new(path)), mode == EstimationMode::FpmInterpolation, )?; - let configured = &mut estimator_config.fpm_interpolation.fpm_parquet_path; - ensure!( - configured - .as_ref() - .is_none_or(|existing| existing == std::path::Path::new(path)), - "conflicting fpm_parquet_path and estimator_config.fpm_interpolation.fpm_parquet_path" - ); - *configured = Some(path.into()); } + let estimator_config = EstimatorConfig::migrate_legacy_inputs( + Some(&serde_json::to_string(&self.estimator_config)?), + None, + self.fpm_interpolation.map(|method| method.as_str()), + self.fpm_parquet_path.as_deref(), + )?; Ok(ForwardPassPerfModelConfig { model: self.model.clone(), system: self.system.clone(), @@ -434,6 +423,10 @@ struct AicTimingModel { use_fpm_decode_totals: bool, fpm_decode_kv_ceiling: Option, evidence: Mutex, + // Capture-only history. Keyed by phase and exact iteration totals, so + // repeated invocations increment a count without cloning prior history. + fpm_evidence: + Mutex>, phase_cache: quick_cache::sync::Cache, } @@ -459,7 +452,8 @@ impl AicTimingModel { config.forward_model = None; config.fpm_parquet_path = None; config.fallback_policy = ForwardPassFallbackPolicy::Deny; - config.estimator_config = provenance.config.estimator_config.clone(); + config.estimator_config = + serde_json::from_value(serde_json::to_value(&provenance.config.estimator_config)?)?; config.fpm_profile = provenance.config.fpm_profile.clone(); config.fpm_interpolation = None; config.gemm_dtype = provenance.config.gemm_quant_mode.clone(); @@ -492,6 +486,7 @@ impl AicTimingModel { use_fpm_decode_totals, fpm_decode_kv_ceiling, evidence: Mutex::new(TimingEvidenceSummary::default()), + fpm_evidence: Mutex::default(), phase_cache: quick_cache::sync::Cache::new(128), }) } @@ -577,6 +572,32 @@ impl AicTimingModel { } Ok(()) } + + fn record_fpm_estimate( + &self, + key: (bool, u32, u32, u32), + estimate: crate::ForwardPassEstimate, + ) -> Result { + let latency_ms = estimate + .latency_ms + .context("native FPM returned no estimate")?; + if latency_ms > 0.0 { + let mut evidence = self + .fpm_evidence + .lock() + .map_err(|_| anyhow!("FPM evidence accumulator was poisoned"))?; + let entry = evidence.entry(key).or_insert(crate::FpmEstimateEvidence { + estimate, + count: 0, + latency_scale: 1.0, + }); + entry.count = entry + .count + .checked_add(1) + .context("FPM evidence invocation count overflow")?; + } + Ok(latency_ms) + } } fn phase_evidence_from_python( @@ -632,6 +653,12 @@ impl TimingModel for AicTimingModel { self.record_evidence(evidence, true)?; return Ok(latency_ms); } + if let Some(model) = &self.diagnostic_model { + return self.record_fpm_estimate( + (true, batch_size, mean_isl, mean_prefix), + model.static_prefill_detailed(batch_size, mean_isl, mean_prefix)?, + ); + } Python::with_gil(|py| { self.engine .bind(py) @@ -658,6 +685,19 @@ impl TimingModel for AicTimingModel { .min(total_kv_tokens); let batch_size = checked_u32(batch_size, "decode batch size")?; let total_past_kv_tokens = checked_u32(total_past_kv_tokens, "total past KV tokens")?; + if let Some(model) = &self.diagnostic_model { + let estimate = + model.estimate_forward_pass_detailed(&[crate::ForwardPassMetrics { + scheduled_requests: crate::ScheduledRequestMetrics { + num_decode_requests: batch_size, + sum_decode_kv_tokens: total_past_kv_tokens, + ..Default::default() + }, + ..Default::default() + }])?; + return self + .record_fpm_estimate((false, batch_size, 0, total_past_kv_tokens), estimate); + } return Python::with_gil(|py| { self.engine .bind(py) @@ -691,7 +731,52 @@ impl TimingModel for AicTimingModel { self.evidence.lock().ok().map(|evidence| evidence.clone()) } + fn fpm_evidence_summary(&self) -> Option { + if !self.use_fpm_decode_totals || self.diagnostic_model.is_none() { + return None; + } + let history = self.fpm_evidence.lock().ok()?; + let phase = |prefill: bool| { + let fpm_estimates: Vec<_> = history + .iter() + .filter(|(key, _)| key.0 == prefill) + .map(|(_, evidence)| evidence.clone()) + .collect(); + if fpm_estimates.is_empty() { + return TimingPhaseEvidence::default(); + } + let latency_ms = fpm_estimates + .iter() + .map(|record| record.estimate.latency_ms.unwrap_or(0.0) * record.count as f64) + .sum(); + let mut operation = TimingOperationEvidence::new( + if prefill { "fpm_prefill" } else { "fpm_decode" }, + latency_ms, + None, + TimingEvidenceSource::Silicon, + ) + .expect("finite native FPM evidence"); + operation.details = Some(crate::perfmodel::engine::diagnostics::OperationDetails { + sol: None, + sol_unavailable_reason: Some( + "whole-model FPM does not export an operation SOL decomposition".into(), + ), + fallbacks: Vec::new(), + fpm_estimates, + }); + TimingPhaseEvidence::from_operations(vec![operation]) + }; + Some(TimingEvidenceSummary { + prefill: phase(true), + decode: phase(false), + }) + } + fn reset_evidence(&self) -> Result<()> { + self.fpm_evidence + .lock() + .map_err(|_| anyhow!("FPM evidence accumulator was poisoned"))? + .clear(); *self .evidence .lock() @@ -1326,9 +1411,20 @@ impl TimingPowerSource { } fn replay_timing_evidence(sources: &[TimingPowerSource]) -> Result> { + replay_timing_evidence_with_fpm(sources, false) +} + +fn replay_timing_evidence_with_fpm( + sources: &[TimingPowerSource], + include_fpm: bool, +) -> Result> { let mut combined = TimingEvidenceSummary::default(); for source in sources { - let Some(summary) = source.timing.evidence_summary() else { + let Some(summary) = source.timing.evidence_summary().or_else(|| { + include_fpm + .then(|| source.timing.fpm_evidence_summary()) + .flatten() + }) else { return Ok(None); }; combined.prefill.try_accumulate(scale_power_phase( @@ -1343,6 +1439,42 @@ fn replay_timing_evidence(sources: &[TimingPowerSource]) -> Result Result> { + let mut providers = Vec::new(); + for (provider_index, source) in sources.iter().enumerate() { + let Some(summary) = source.timing.fpm_evidence_summary() else { + continue; + }; + let mut phases = Vec::new(); + for (name, phase, speedup) in [ + ("prefill", summary.prefill, source.prefill_speedup_ratio), + ("decode", summary.decode, source.decode_speedup_ratio), + ] { + let phase = scale_power_phase(phase, speedup)?; + let operations = phase + .operations + .into_iter() + .filter_map(|operation| { + let details = operation.details?; + (!details.fpm_estimates.is_empty()).then(|| { + serde_json::json!({ + "name": operation.name, "fpm_estimates": details.fpm_estimates, + }) + }) + }) + .collect::>(); + phases.push(serde_json::json!({"name": name, "operations": operations})); + } + providers.push(serde_json::json!({"provider_index": provider_index, "phases": phases})); + } + Ok((!providers.is_empty()).then(|| { + serde_json::json!({ + "schema_version": "1.0", "aggregation": "identical_estimates_with_invocation_counts", + "providers": providers, + }) + })) +} + fn replay_power_stats(summary: &TimingEvidenceSummary) -> Result { let mut combined = summary.prefill.clone(); combined.try_accumulate(summary.decode.clone())?; @@ -1388,12 +1520,12 @@ fn replay_performance_diagnostics(summary: Option<&TimingEvidenceSummary>) -> se let scope = "accumulated_active_forward_pass_per_gpu"; let Some(summary) = summary else { return serde_json::json!({"status": "unavailable", "scope": scope, "latency_unit": "ms", "phases": [], - "unavailable_reason": "selected timing provider does not export operation diagnostics (whole-model FPM and latency-only providers are unsupported)"}); + "unavailable_reason": "selected timing provider does not export operation diagnostics"}); }; let phases = [("prefill", &summary.prefill), ("decode", &summary.decode)].into_iter().map(|(name, phase)| { let mut operations = phase.operations.iter().map(|op| { let sol = op.details.as_ref().and_then(|d| d.sol.as_ref()); - serde_json::json!({"name": op.name, "latency_ms": op.latency_ms, + let mut record = serde_json::json!({"name": op.name, "latency_ms": op.latency_ms, "source": op.source.as_str(), "sol": sol, "sol_unavailable_reason": if sol.is_some() { None } else { @@ -1401,7 +1533,11 @@ fn replay_performance_diagnostics(summary: Option<&TimingEvidenceSummary>) -> se }, "latency_to_sol_ratio": sol.filter(|s| s.latency_ms > 0.0).map(|s| op.latency_ms / s.latency_ms).filter(|r| r.is_finite()), "fallbacks": op.details.as_ref().map(|d| &d.fallbacks), - }) + }); + if let Some(details) = &op.details && !details.fpm_estimates.is_empty() { + record["fpm_estimates"] = serde_json::to_value(&details.fpm_estimates).expect("finite native FPM evidence"); + } + record }).collect::>(); operations.sort_by(|a,b| a["name"].as_str().cmp(&b["name"].as_str())); let sol = phase.operations.iter().map(|op| op.details.as_ref()?.sol.as_ref()).collect::>>() @@ -1597,6 +1733,11 @@ fn scale_power_phase( operation.energy_wms = operation.energy_wms.map(|energy| energy * scale); operation.latency_ms *= scale; operation.covered_latency_ms *= scale; + if let Some(details) = &mut operation.details { + for evidence in &mut details.fpm_estimates { + evidence.latency_scale *= scale; + } + } // SOL compares the same scheduled work, before synthetic speedup; it // remains an unscaled physical baseline. } @@ -1881,8 +2022,12 @@ fn execute_json(payload: &str, capture_artifacts: bool) -> Result { let mut report_json = serde_json::to_value(&report).context("serializing AISimulate replay report summary")?; if capture_performance_diagnostics { + let performance_evidence = replay_timing_evidence_with_fpm(&power_sources, true)?; report_json["performance_diagnostics"] = - replay_performance_diagnostics(timing_evidence.as_ref()); + replay_performance_diagnostics(performance_evidence.as_ref()); + } + if let Some(evidence) = replay_fpm_query_evidence(&power_sources)? { + report_json["fpm_query_evidence"] = evidence; } if report.agentic_graph.is_some() && let Some((input_format, agentic_lanes, execution_model)) = agentic_input @@ -2452,6 +2597,7 @@ mod tests { use_fpm_decode_totals, fpm_decode_kv_ceiling: None, evidence: Mutex::new(TimingEvidenceSummary::default()), + fpm_evidence: Mutex::default(), phase_cache: quick_cache::sync::Cache::new(128), } } @@ -2492,6 +2638,34 @@ mod tests { ); } + #[test] + fn replay_interpolation_migration_preserves_explicit_method_presence() { + for canonical in [None, Some("direct"), Some("auto"), Some("sol")] { + let mut config = aic_config(); + config.fpm_interpolation = Some(crate::FpmInterpolationMethod::Direct); + config.estimator_config = serde_json::from_value(serde_json::json!({ + "correction": {"enabled": false}, + "fpm_interpolation": canonical + .map(|method| serde_json::json!({"method": method})) + .unwrap_or_else(|| serde_json::json!({})) + })) + .unwrap(); + let request = config.estimator_request(ForwardPassWorkerType::Aggregated); + if canonical.is_none() || canonical == Some("direct") { + let controls = request.unwrap().estimator_config; + assert_eq!( + controls.fpm_interpolation.method, + crate::FpmInterpolationMethod::Direct + ); + assert!(!controls.correction.enabled); + } else { + assert!(request.unwrap_err().to_string().contains( + "conflicting explicit values for estimator_config.fpm_interpolation.method" + )); + } + } + } + fn aic_config() -> AicTimingConfig { AicTimingConfig { model: "test-model".into(), @@ -2519,7 +2693,7 @@ mod tests { worker_type: None, estimation_mode: None, fallback_policy: ForwardPassFallbackPolicy::Deny, - estimator_config: EstimatorConfig::default(), + estimator_config: serde_json::Map::new(), database_mode: crate::DatabaseMode::default(), transfer_policy: None, systems_paths: Vec::new(), @@ -2992,6 +3166,7 @@ mod tests { TimingOperationEvidence::new("dispatch", 10.0, None, TimingEvidenceSource::Estimated) .unwrap(); op.details = Some(OperationDetails { + fpm_estimates: Vec::new(), sol: Some(SolDiagnostics { latency_ms: 2.0, math_ms: 0.0, @@ -3156,6 +3331,13 @@ mod tests { assert_eq!(latency, 546_046.0); assert_eq!(timing.evidence_summary(), None); + assert_eq!(timing.fpm_evidence_summary(), None); + // Capture is disabled: even a long sequence of distinct total-KV + // queries cannot allocate a growing evidence history. + for kv in 546_100..547_100 { + timing.predict_decode_ms(35, kv, 15_602, 1_000_000).unwrap(); + } + assert!(timing.fpm_evidence.lock().unwrap().is_empty()); } #[test] diff --git a/docs/cli/prediction-details.schema.json b/docs/cli/prediction-details.schema.json index 75f237e680..bf3005a7c4 100644 --- a/docs/cli/prediction-details.schema.json +++ b/docs/cli/prediction-details.schema.json @@ -1031,6 +1031,12 @@ } } ] + }, + "fpm_estimates": { + "type": "array", + "items": { + "$ref": "#/$defs/fpmEstimateRecord" + } } }, "required": [ @@ -1094,6 +1100,12 @@ "null" ], "minimum": 0 + }, + "fpm_estimates": { + "type": "array", + "items": { + "$ref": "#/$defs/fpmEstimateRecord" + } } }, "required": [ @@ -1323,6 +1335,231 @@ } } ] + }, + "fpmCoordinates": { + "type": "object", + "properties": { + "batch_size": { + "type": "number", + "minimum": 0 + }, + "total_prefill_tokens": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "total_kv_read_tokens": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "batch_size", + "total_prefill_tokens", + "total_kv_read_tokens" + ], + "additionalProperties": false + }, + "fpmMeasurementSupport": { + "type": "object", + "properties": { + "coordinates": { + "$ref": "#/$defs/fpmCoordinates" + }, + "latency_ms": { + "type": "number", + "minimum": 0 + }, + "weight": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "coordinates", + "latency_ms", + "weight" + ], + "additionalProperties": false + }, + "fpmQuery": { + "type": "object", + "properties": { + "phase": { + "enum": [ + "prefill", + "decode" + ] + }, + "model_path": { + "type": "string", + "minLength": 1 + }, + "decode_baseline": { + "type": "boolean" + }, + "query": { + "$ref": "#/$defs/fpmCoordinates" + }, + "resolution": { + "enum": [ + "exact_lookup", + "within_curve_interpolation", + "cross_kv_interpolation", + "cross_batch_interpolation" + ] + }, + "latency_ms": { + "type": "number", + "minimum": 0 + }, + "support": { + "type": "array", + "items": { + "$ref": "#/$defs/fpmMeasurementSupport" + }, + "minItems": 1 + } + }, + "required": [ + "phase", + "model_path", + "decode_baseline", + "query", + "resolution", + "latency_ms", + "support" + ], + "additionalProperties": false + }, + "fpmRankEstimate": { + "type": "object", + "properties": { + "rank": { + "type": "integer", + "minimum": 0 + }, + "latency_ms": { + "type": "number", + "minimum": 0 + }, + "prefill_ms": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "decode_ms": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "decode_baseline_ms": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "marginal_decode_ms": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "queries": { + "type": "array", + "items": { + "$ref": "#/$defs/fpmQuery" + } + } + }, + "required": [ + "rank", + "latency_ms", + "prefill_ms", + "decode_ms", + "decode_baseline_ms", + "marginal_decode_ms", + "queries" + ], + "additionalProperties": false + }, + "fpmEstimate": { + "type": "object", + "properties": { + "latency_ms": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "native_latency_ms": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "correction_factor": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "max_rank": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "ranks": { + "type": "array", + "items": { + "$ref": "#/$defs/fpmRankEstimate" + } + } + }, + "required": [ + "latency_ms", + "native_latency_ms", + "correction_factor", + "max_rank", + "ranks" + ], + "additionalProperties": false + }, + "fpmEstimateRecord": { + "type": "object", + "properties": { + "estimate": { + "$ref": "#/$defs/fpmEstimate" + }, + "count": { + "type": "integer", + "minimum": 1 + }, + "latency_scale": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "estimate", + "count", + "latency_scale" + ], + "additionalProperties": false } }, "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0", diff --git a/docs/core-api.md b/docs/core-api.md index b057ccc339..03178cd12e 100644 --- a/docs/core-api.md +++ b/docs/core-api.md @@ -283,9 +283,10 @@ nested paths. The supported namespaces are: (5), `factor_bounds` (min 0.5, max 2.0), and the existing `max_num_tokens` (8192), `max_batch_size` (512), and `max_kv_tokens` (2000000) ranges. - `fpm_interpolation.method`: `auto` (default), `sol`, or `direct`. Rust selects - SOL for a registered architecture, or direct interpolation for an unknown - architecture with a valid profile. Without a profile, auto retains SOL. - Explicit SOL requires a registered analytical model; direct requires a profile. + SOL for a registered architecture, or direct interpolation for a verified + architecture without a registered class when a profile is supplied. Without + a profile, auto retains SOL. Explicit SOL requires a registered analytical + model; direct requires a profile and also supports registered architectures. - `op_level`: a reserved typed namespace; unknown fields are rejected. The top-level `fpm_profile` contains the complete profile dictionary: pinned @@ -296,6 +297,14 @@ slot aliases and omitted versions are rejected. Profile/schema and precision conflicts fail before estimator fallback. Omitted precision fields are filled from the profile and preserved in the resolved canonical configuration. +Runtime normalization and construction verify the profile architecture against +checkpoint `config.json` from the local model path or pinned remote revision +before interpolation selection. Missing or malformed architecture metadata and +mismatched declarations fail explicitly. This check reads configuration only: +no weights or analytical graph are required. An architecture with verified +metadata remains valid for direct interpolation without a registered analytical +class. Schema-only profile and application configuration parsing remain lightweight. + For measured-only timing, pass `estimation_mode="fpm_interpolation"`, `fallback_policy="deny"`, and `estimator_config={"fpm_interpolation": {"method": "direct"}}` together with @@ -304,6 +313,14 @@ whole-forward native operations without SOL operations, and never constructs an analytical graph. Profile resource estimates and memory planning do not require timing data or a native timing model. +Direct readiness requires genuine measurements for each operation's phase; +prefill-only operations do not require decode rows, and vice versa. Construction +continues to later systems roots when a required phase is unavailable. Query +coverage still needs an exact point or supported interpolation. Cross-KV prefill +uses the nearest same-batch lower and upper KV curves that both cover the +requested token count, without a KV distance limit. SOL's site-distance guard +does not apply to this direct bracket. See the [self-service coverage rules](fpm-self-service.md#choose-the-model-execution-route). + The returned provenance pins both the selected estimation mode and interpolation method, alongside the complete normalized profile. Reusing its `config` keeps that selection across serialization and replay. Later timing coverage errors @@ -343,6 +360,14 @@ Legacy `extra.fpm_profile` and `extra.fpm_interpolation` migrate to the full canonical profile and nested interpolation method; newly exported configuration uses only the canonical fields. +Migration merges saved estimator controls with explicitly supplied legacy +options before applying ordinary defaults. It preserves disjoint settings and +accepts agreeing overlaps; contradictory explicit values report the canonical +setting's path. The same rule applies to legacy and canonical interpolation +methods, including an explicit `auto`. An omitted field does not override a +saved value. `ForwardPassPerfOptions.to_dict()` serializes only arguments +explicitly supplied to that legacy options object, including explicit defaults. + Previously saved CLI timing with `forward_model` retains explicit selection and deny. Newly authored requests without a selection use auto. `ForwardPassPerfOptions` is retained as a legacy migration value type; new construction has one complete @@ -397,6 +422,31 @@ missing-data sentinel, not evidence of a zero-power operation. See the aggregation rules, and public output boundary. Typed per-op energy alone does not make unified replay power available. +## Direct FPM query evidence + +The same model returned by `RustForwardPassPerfModel.best_available(config)` exposes `estimate_forward_pass_detailed(metrics)` (also available on the Rust `ForwardPassPerfModel`). Its `latency_ms` equals `estimate_forward_pass_time_ms(metrics)`; the scalar API and four-element per-operation tuple APIs are unchanged. + +```python +result = model.estimate_forward_pass_detailed({ + "scheduled_requests": { + "num_prefill_requests": 1, + "sum_prefill_tokens": 512, + "sum_prefill_kv_tokens": 256, + }, +}) +for rank in result["ranks"]: + for query in rank["queries"]: + print(query["resolution"], query["query"], query["support"]) +``` + +Each direct lookup records `exact_lookup`, `within_curve_interpolation`, `cross_kv_interpolation`, or `cross_batch_interpolation`, its phase and model, requested coordinates, raw latency, and supporting measured coordinates, latencies and weights. Coordinates are iteration totals: `batch_size`, `total_prefill_tokens` (null for decode), and `total_kv_read_tokens`. Nested interpolation weights are multiplied for each supporting measurement. This evidence preserves `source="silicon"`, whose meaning includes interpolation; it does not establish accuracy or observed CUDA graph dispatch. + +Weights describe a raw lookup, before later composition. Each rank records prefill, decode and mixed-pass baseline latencies, plus `marginal_decode_ms = max(decode_ms - decode_baseline_ms, 0)`. A baseline lookup has `decode_baseline=true`: its query is the paired decode request, but its support contains each selected row's actual measured minimum-KV point. `max_rank` is the first zero-based input index attaining the positive native maximum. The result separately records `native_latency_ms`, `correction_factor`, and final `latency_ms`. Empty work has no query evidence. Other estimators also have no direct lookup evidence; an unready regression retains `latency_ms=null`. + +Replay and prediction capture this evidence with `ReplayOutputRequirements(capture_performance_diagnostics=True)`; CLI prediction enables it with `--detail source` or `--detail time`. Use `--format json` or inspect the saved `prediction.json` for the query records and measured support; the default source text output shows operation sources and fallbacks. Public `run_recommendation(..., output_requirements=ReplayOutputRequirements(capture_performance_diagnostics=True))` and `Sweeper(..., output_requirements=...)` forward the same request to each candidate's replay. Evidence survives saved results under `ReplayReport.metadata["fpm_query_evidence"]` and each candidate's `provenance.runner_metadata["fpm_query_evidence"]`. The native replay report carries the same top-level key; source details include `fpm_estimates` on whole-model operations. FPM power remains unavailable and SOL comparison has an explicit unavailable reason. + +Captured replay records are grouped by timing provider and phase. Each `fpm_estimates` entry holds an `estimate`, an invocation `count`, and `latency_scale` for any synthetic replay speedup; raw support is never rescaled. Identical query coordinates within an immutable replay provider share one counted record. Counts describe timing invocations in the measurement epoch, not independent silicon observations or served requests. Distinct queries are retained without truncation, so capture memory grows with distinct coordinates; ordinary runs do not retain query histories. Aggregated topology has provider index 0; disaggregated topology orders prefill before decode. + ## Static phase diagnostics The canonical `ForwardPassPerfModel` returned by `best_available` exposes diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index 32adaddd22..17a4cc8fa8 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 Planning works before the model has an AISimulate model class or measured FPM timings. With a supplied FPM profile, planning validates the declared deployment identity and estimates memory admission from your resource bounds. Runtime compatibility and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not provision GPUs or run target preflight checks. -Ordinary FPM `predict` and `recommend` accept an inline `engine.fpm_profile` with model identity and rank-local resource bounds. Direct interpolation uses measured timings without constructing an op-level model. Registered models retain SOL interpolation. See [Choose the model execution route](#choose-the-model-execution-route) for selection and coverage rules. +Ordinary FPM `predict` and `recommend` accept an inline `engine.fpm_profile` with model identity and rank-local resource bounds. Direct interpolation uses measured timings without constructing an op-level model. Automatic interpolation selection retains SOL for registered architectures. See [Choose the model execution route](#choose-the-model-execution-route) for selection and coverage rules. The Inkling pilot targets NVIDIA GB200 with TP, DEP, and TEP through the class-independent route. Its checkpoint, runtime, allocation, profile resource bounds, and strategy degrees must be pinned before collection. This guide does not establish Inkling readiness or GB200 accuracy. @@ -59,7 +59,7 @@ For a model without an analytical class, create a JSON or YAML FPM profile and p | Profile fields | Required meaning | | --- | --- | | `schema_version` | Integer `1`. | -| `model`, `model_revision`, `architecture` | Exact timing model identity, pinned checkpoint revision, and architecture identifier. An unknown architecture is valid for direct interpolation. | +| `model`, `model_revision`, `architecture` | Exact timing model identity, pinned checkpoint revision, and architecture identifier matching the checkpoint configuration. An analytical model class is optional for direct interpolation. | | `context_length`, `num_experts`, `provenance` | Declared context limit, routed expert count (`0` for dense), and how the metadata was obtained. | | `deployments` | One or more exact deployment records, with one precision/resource identity per hardware/runtime/full parallel tuple. | | Deployment `system`, `backend`, `backend_version` | GPU system, `vllm`, and literal runtime version. | @@ -76,6 +76,8 @@ Automatic KV admission subtracts these non-KV bounds and CUDA graph reservation The declared checkpoint revision is preserved for review and replay. Existing FPM tables may not pin a checkpoint revision; supplying one in a profile does not prove that the historical measurements came from that exact revision. Record that limitation in `provenance` when validating existing data. +Ordinary FPM model construction reads `config.json` from the local checkpoint or the declared remote `model_revision` and verifies the profile's architecture against the checkpoint's text-model architecture before selecting interpolation. Missing or malformed architecture metadata and mismatched names fail explicitly. This verification loads no weights and constructs no analytical model graph. A verified architecture without a registered analytical class remains valid for direct interpolation. Schema-only parsing and onboarding request creation do not require this runtime metadata check. + ## Plan, preview, and explicitly execute ```bash @@ -103,7 +105,15 @@ aisimulate onboard collect-fpm \ --output-dir ./aisimulate-support --execute ``` -Execution requires a matching saved plan. Creating the initial plan records model and runtime revisions without downloading a pinned checkpoint or inspecting the running runtime. During profile-based collection execution, the collector checks the observed Pod's vLLM version against the profile's literal backend version before benchmarking. Keep the actual checkpoint consistent with the declared model revision. +Execution requires a matching saved plan. Creating the initial plan records model and runtime revisions without downloading a pinned checkpoint or inspecting the running runtime. During profile-based collection execution, the collector records the observed container's vLLM version before checking it against the profile's literal backend version and before benchmarking. Keep the actual checkpoint consistent with the declared model revision. + +Custom and development builds are supported with an exact full-version pin. Local and development suffixes are part of the identity: `0.25.1+custom` must match `0.25.1+custom`, and `0.25.2.dev3` must match `0.25.2.dev3`. Inspect the version **inside the collection container**: + +```bash +python3 -c "import importlib.metadata; print(importlib.metadata.version('vllm'))" +``` + +Use that complete string for both the request's `--framework-version` and the collection profile's deployment `backend_version`. On a mismatch, the error reports the expected and observed strings, and the observed version remains in `collector-provenance.json`. Recover by using the pinned runtime, or by creating a new collection profile pinned to the observed full version and regenerating the request and collection plan in a new output directory. For a profile-based campaign, the collector validates its resolved topology and precision against the supplied profile before execution. It does not relabel an FP8 cell as BF16 or change checkpoint quantization to satisfy the profile. A mismatch reports the conflicting field and requires a matching profile/runtime or a supported collector configuration. Resource admission uses the supplied bounds without constructing an analytical model. Profile contents participate in the collector's frozen-plan identity. @@ -188,8 +198,16 @@ timing: method: direct ``` -Within `estimator_config.fpm_interpolation`, `method: auto` retains SOL for a registered architecture and chooses direct for an unregistered architecture with a profile. Explicit `sol` requires a registered class; explicit `direct` requires a profile. Rust validates and selects the method through `RustForwardPassPerfModel.best_available(ForwardPassPerfModelConfig)`, then pins it for queries and exported configurations. A model-construction error does not trigger a silent change of method. The engine's top-level `estimation_mode: auto` retains the normal priority of operation-level estimation, FPM interpolation, then FPM regression; providing a profile does not change that priority. +Within `estimator_config.fpm_interpolation`, `method: auto` retains SOL for a registered architecture and chooses direct for a verified architecture without a registered class when a profile is supplied. Explicit `sol` requires a registered class; explicit `direct` requires a profile and is also valid for registered architectures. Rust validates and selects the method through `RustForwardPassPerfModel.best_available(ForwardPassPerfModelConfig)`, then pins it for queries and exported configurations. A model-construction error does not trigger a silent change of method. The engine's top-level `estimation_mode: auto` retains the normal priority of operation-level estimation, FPM interpolation, then FPM regression; providing a profile does not change that priority. + +Direct readiness requires genuine measurements for each operation's phase. Prefill rows cannot make an operation with no genuine decode rows ready, and vice versa. Construction tries later configured systems roots when a root lacks a required phase; if none is usable, it reports the missing phase. An operation that needs only prefill does not require decode measurements, and vice versa. Readiness does not guarantee coverage of every subsequent query. + +Direct timing first uses an exact point or interpolation within a measured curve. For cross-KV prefill interpolation, it selects the nearest measured KV curve on each side at the same batch size, considering only curves that cover the requested total new-token count. This direct bracket has no KV distance limit; SOL's site-distance guard is separate. Decode interpolation respects the measured batch/capture domain. Both phases exclude synthetic `fake_fallback` rows, including healed/extrapolated values. Missing two-sided support, unmeasured prefill batches and out-of-domain queries fail explicitly. The direct route does not apply SOL-dependent prefill batch clamping or general extrapolation; 2D interpolation remains experimental. -Direct timing first uses an exact point or interpolation within a measured curve. Prefill interpolation stays at the same batch size, with two measured KV neighbors whose prompt curves both cover the requested token count. Wider KV bracketing removes the old distance limit only when the narrower direct bracket is unavailable. Decode interpolation respects the measured batch/capture domain. Both phases exclude synthetic `fake_fallback` rows, including healed/extrapolated values. Missing two-sided support, unmeasured batches and out-of-domain queries fail explicitly. The direct route does not apply SOL-dependent prefill batch clamping or general extrapolation; 2D interpolation remains experimental. +Use `predict --detail source --format json` to inspect executed direct lookups and their measured support, or inspect the saved `prediction.json` after requesting source details. The default text output shows operation sources and fallbacks. The canonical model's `estimate_forward_pass_detailed(metrics)` provides the same lookup evidence with rank reduction, mixed-pass baseline and online-correction context. Replay and recommendation APIs accept `ReplayOutputRequirements(capture_performance_diagnostics=True)` to retain counted query records in their returned metadata. Ordinary runs keep no query history; requested capture retains every distinct query without truncation. See [Direct FPM query evidence](core-api.md#direct-fpm-query-evidence) for the result fields and recommendation API. Exact and interpolated values keep the existing `silicon` source tag. Provenance alone does not qualify interpolation accuracy or prove which CUDA graph a measured forward used; accuracy gates and graph-aware interpolation remain deferred until new measurements establish the relevant behavior. + +Saved legacy interpolation controls merge with explicitly supplied correction, regression, and sampling controls. Disjoint settings and agreeing overlapping values are retained; conflicting explicit values fail with the setting's name. Defaults apply after these explicit inputs merge. See [Migrating saved configuration](core-api.md#migrating-saved-configuration) for the public migration API. Per-operation silicon profiling described in the model guide is not required by either FPM route. The workflow collects whole-forward timings, then verifies prediction and recommendation for the exact target deployment. Report timing coverage and interpolation error separately; successful simulation alone does not establish measured accuracy. + +Accuracy gates and interpolation corrections remain deferred until new measurements are collected with the provenance from [Dynamo #15110](https://github.com/ai-dynamo/dynamo/pull/15110). Those records support investigation; they do not by themselves establish the cause of earlier interpolation errors. If a reproducible CUDA graph boundary causes the discrepancy, interpolation must respect that boundary. No graph-aware partitioning or accuracy policy is introduced here. diff --git a/python/aisimulate/collector/fpm_forward/runner.py b/python/aisimulate/collector/fpm_forward/runner.py index 3b5a248d2b..4c3e1f2732 100644 --- a/python/aisimulate/collector/fpm_forward/runner.py +++ b/python/aisimulate/collector/fpm_forward/runner.py @@ -479,7 +479,10 @@ def _attempt_provenance_command( script += ( "\nactual = payload['runtime']['backend_version']\n" "if actual != sys.argv[3]:\n" - " raise RuntimeError(f'FPM profile runtime mismatch: actual={actual!r}, expected={sys.argv[3]!r}')\n" + " raise RuntimeError(" + "f'FPM profile runtime mismatch: actual={actual!r}, expected={sys.argv[3]!r}. '" + "'Use the pinned runtime, or create a new collection profile with '" + "f'backend_version={actual!r} and regenerate the collection plan in a new output directory.')\n" ) version_args.append(expected_backend_version) return ["python3", "-c", script, payload, COLLECTOR_PROVENANCE_FILENAME, *version_args] diff --git a/python/aisimulate/src/aisimulate/detail.py b/python/aisimulate/src/aisimulate/detail.py index b5be9d1727..3ec5727058 100644 --- a/python/aisimulate/src/aisimulate/detail.py +++ b/python/aisimulate/src/aisimulate/detail.py @@ -115,6 +115,65 @@ def number(value: Any, path: str) -> None: def array(value: Any, path: str) -> None: require(isinstance(value, list), path, "a list") + def fpm_coordinates(value: Any, path: str) -> None: + fields(value, path, {"batch_size", "total_prefill_tokens", "total_kv_read_tokens"}) + number(value["batch_size"], f"{path}.batch_size") + if value["total_prefill_tokens"] is not None: + number(value["total_prefill_tokens"], f"{path}.total_prefill_tokens") + number(value["total_kv_read_tokens"], f"{path}.total_kv_read_tokens") + + def fpm_query(value: Any, path: str) -> None: + fields(value, path, {"phase", "model_path", "decode_baseline", "query", "resolution", "latency_ms", "support"}) + require(value["phase"] in ("prefill", "decode"), f"{path}.phase", "prefill or decode") + require( + isinstance(value["model_path"], str) and bool(value["model_path"]), + f"{path}.model_path", + "a nonempty string", + ) + require(type(value["decode_baseline"]) is bool, f"{path}.decode_baseline", "a boolean") + fpm_coordinates(value["query"], f"{path}.query") + require( + value["resolution"] + in ("exact_lookup", "within_curve_interpolation", "cross_kv_interpolation", "cross_batch_interpolation"), + f"{path}.resolution", + "a supported direct resolution", + ) + number(value["latency_ms"], f"{path}.latency_ms") + array(value["support"], f"{path}.support") + require(bool(value["support"]), f"{path}.support", "nonempty") + for i, support in enumerate(value["support"]): + sp = f"{path}.support[{i}]" + fields(support, sp, {"coordinates", "latency_ms", "weight"}) + fpm_coordinates(support["coordinates"], f"{sp}.coordinates") + number(support["latency_ms"], f"{sp}.latency_ms") + number(support["weight"], f"{sp}.weight") + require(support["weight"] <= 1, f"{sp}.weight", "at most 1") + + def fpm_estimate(value: Any, path: str) -> None: + fields(value, path, {"latency_ms", "native_latency_ms", "correction_factor", "max_rank", "ranks"}) + for key in ("latency_ms", "native_latency_ms", "correction_factor"): + if value[key] is not None: + number(value[key], f"{path}.{key}") + if value["max_rank"] is not None: + require( + type(value["max_rank"]) is int and value["max_rank"] >= 0, + f"{path}.max_rank", + "a nonnegative integer", + ) + array(value["ranks"], f"{path}.ranks") + for i, rank in enumerate(value["ranks"]): + rp = f"{path}.ranks[{i}]" + durations = {"prefill_ms", "decode_ms", "decode_baseline_ms", "marginal_decode_ms"} + fields(rank, rp, {"rank", "latency_ms", "queries"} | durations) + require(type(rank["rank"]) is int and rank["rank"] >= 0, f"{rp}.rank", "a nonnegative integer") + number(rank["latency_ms"], f"{rp}.latency_ms") + for key in durations: + if rank[key] is not None: + number(rank[key], f"{rp}.{key}") + array(rank["queries"], f"{rp}.queries") + for j, query in enumerate(rank["queries"]): + fpm_query(query, f"{rp}.queries[{j}]") + def sol(value: dict[str, Any], path: str) -> None: if value["sol"] is None: string(value["sol_unavailable_reason"], f"{path}.sol_unavailable_reason") @@ -150,10 +209,21 @@ def sol(value: dict[str, Any], path: str) -> None: operation, op, {"name", "latency_ms", "source", "fallbacks", "sol", "sol_unavailable_reason", "latency_to_sol_ratio"}, + {"fpm_estimates"}, ) string(operation["name"], f"{op}.name") string(operation["source"], f"{op}.source") number(operation["latency_ms"], f"{op}.latency_ms") + if "fpm_estimates" in operation: + array(operation["fpm_estimates"], f"{op}.fpm_estimates") + for k, evidence in enumerate(operation["fpm_estimates"]): + ep = f"{op}.fpm_estimates[{k}]" + fields(evidence, ep, {"estimate", "count", "latency_scale"}) + require( + type(evidence["count"]) is int and evidence["count"] > 0, f"{ep}.count", "a positive integer" + ) + number(evidence["latency_scale"], f"{ep}.latency_scale") + fpm_estimate(evidence["estimate"], f"{ep}.estimate") sol(operation, op) if operation["latency_to_sol_ratio"] is not None: number(operation["latency_to_sol_ratio"], f"{op}.latency_to_sol_ratio") @@ -244,7 +314,11 @@ def build_prediction_details(native: dict[str, Any], sections: tuple[str, ...]) { "name": phase["name"], "operations": [ - {key: op[key] for key in ("name", "latency_ms", "source", "fallbacks")} + { + key: op[key] + for key in ("name", "latency_ms", "source", "fallbacks", "fpm_estimates") + if key in op + } for op in phase["operations"] ], } diff --git a/python/aisimulate/src/aisimulate/recommend.py b/python/aisimulate/src/aisimulate/recommend.py index 24739e428c..f7af7fe1de 100644 --- a/python/aisimulate/src/aisimulate/recommend.py +++ b/python/aisimulate/src/aisimulate/recommend.py @@ -24,7 +24,7 @@ from .sweeper.afd_perfmodel import AFDPerformanceModel from .sweeper.config import SmartSearchConfig from .sweeper.provider import InfeasibleCandidate, SweepContext -from .sweeper.replay import ReplaySpec, RunnerFactory +from .sweeper.replay import ReplayOutputRequirements, ReplaySpec, RunnerFactory from .sweeper.result import SweepResult @@ -37,6 +37,7 @@ def run_recommendation( providers: Mapping[str, SimulationConfigAdapter] | None = None, afd_performance_model: AFDPerformanceModel | None = None, show_progress: bool = True, + output_requirements: ReplayOutputRequirements | None = None, ) -> SweepResult: """Run a public recommendation through the existing Sweeper core.""" @@ -49,6 +50,7 @@ def run_recommendation( providers=providers, afd_performance_model=afd_performance_model, show_progress=show_progress, + output_requirements=output_requirements, ) if not in_supervised_process(): return supervised_recommendation(config, kwargs) @@ -64,6 +66,7 @@ def _run_recommendation( providers: Mapping[str, SimulationConfigAdapter] | None = None, afd_performance_model: AFDPerformanceModel | None = None, show_progress: bool = True, + output_requirements: ReplayOutputRequirements | None = None, ) -> SweepResult: from .supervision import checkpoint @@ -102,6 +105,7 @@ def _run_recommendation( adapter_sections = {name: provider.section for name, provider in (providers or {}).items()} sweeper = Sweeper( runner_factory=runner_factory, + output_requirements=output_requirements, providers=compiled_providers, show_progress=show_progress, prediction_config_factory=lambda sample, spec: _candidate_prediction( diff --git a/python/aisimulate/src/aisimulate/runner.py b/python/aisimulate/src/aisimulate/runner.py index dc6559360b..a589a8d969 100644 --- a/python/aisimulate/src/aisimulate/runner.py +++ b/python/aisimulate/src/aisimulate/runner.py @@ -1956,6 +1956,7 @@ def add(name: str, value: object) -> None: "agentic_phases", "agentic_model_projection", "weka_nested_timestamp_basis", + "fpm_query_evidence", ) if key in payload } diff --git a/python/aisimulate/src/aisimulate/sweeper/search.py b/python/aisimulate/src/aisimulate/sweeper/search.py index e726bd3068..4b30bb630d 100644 --- a/python/aisimulate/src/aisimulate/sweeper/search.py +++ b/python/aisimulate/src/aisimulate/sweeper/search.py @@ -66,6 +66,7 @@ ) from .replay import ( REPLAY_SPEC_API_VERSION, + ReplayOutputRequirements, ReplayReport, ReplaySpec, Runner, @@ -777,12 +778,18 @@ def _materialize_one( ), None -def _run_replay_detailed(spec: ReplaySpec, runner: Runner) -> _ReplayEvaluation: +def _run_replay_detailed( + spec: ReplaySpec, runner: Runner, output_requirements: ReplayOutputRequirements | None = None +) -> _ReplayEvaluation: """Run one replay while retaining validated runner provenance metadata.""" try: try: - report = runner.run(spec) + report = ( + runner.run(spec) + if output_requirements is None + else runner.run(spec, output_requirements=output_requirements) + ) except ResourceLimitError: raise except Exception as exc: @@ -1066,6 +1073,8 @@ def _score_prepared( def _init_worker( runner_factory: RunnerFactory, + *, + output_requirements: ReplayOutputRequirements | None = None, ) -> None: identity = getattr(mp.current_process(), "_identity", ()) worker_id = int(identity[0]) if identity else 0 @@ -1074,11 +1083,11 @@ def _init_worker( # shutdown. Unlike a plain atexit handler, this matches the ProcessPool # worker lifecycle and lets a runtime release worker-local resources. Finalize(None, runner.close, exitpriority=0) - _WORKER_CTX.update(runner=runner) + _WORKER_CTX.update(runner=runner, output_requirements=output_requirements) def _worker_eval(spec: ReplaySpec) -> _ReplayEvaluation: - return _run_replay_detailed(spec, _WORKER_CTX["runner"]) + return _run_replay_detailed(spec, _WORKER_CTX["runner"], _WORKER_CTX.get("output_requirements")) class Sweeper: @@ -1098,8 +1107,10 @@ def __init__( show_progress: bool = True, prediction_config_factory: Callable[[dict[str, Any], ReplaySpec], dict[str, Any]] | None = None, afd_performance_model: AFDPerformanceModel | None = None, + output_requirements: ReplayOutputRequirements | None = None, ) -> None: self._runner_factory = runner_factory + self._output_requirements = output_requirements self._providers = dict(providers or {}) self._sampler_factory = sampler_factory self._show_progress = show_progress @@ -1131,6 +1142,9 @@ def run( raise ValueError(f"top_n must be positive or None, got {top_n}") retention = CandidateRetention(candidate_retention) runner_factory = self._runner_factory + from functools import partial + + worker_initializer = partial(_init_worker, output_requirements=self._output_requirements) providers = self._providers sampler_factory = self._sampler_factory show_progress = self._show_progress @@ -1233,7 +1247,7 @@ def _new_pool() -> ProcessPoolExecutor: return ProcessPoolExecutor( max_workers=worker_count, mp_context=mp.get_context("spawn"), - initializer=_init_worker, + initializer=worker_initializer, initargs=(runner_factory,), ) @@ -1296,7 +1310,7 @@ def _eval_batch(todo: list[tuple[Suggestion, _PreparedCandidate]]): for index, result in evaluate_waves( [prepared.replay_spec for _, prepared in todo], factory=runner_factory, - initializer=_init_worker, + initializer=worker_initializer, evaluate=_worker_eval, workers=worker_count, timeout=max_eval_seconds, @@ -1327,7 +1341,7 @@ def _eval_batch(todo: list[tuple[Suggestion, _PreparedCandidate]]): suggestion, _score_prepared( prepared, - _run_replay_detailed(prepared.replay_spec, sequential_runner), + _run_replay_detailed(prepared.replay_spec, sequential_runner, self._output_requirements), config=config, goal=goal, ), diff --git a/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py b/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py index 83c67100a3..cdbc4779a6 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py +++ b/python/aisimulate/src/aisimulate_core/sdk/fpm_profile.py @@ -6,6 +6,8 @@ from __future__ import annotations import json +from functools import cache +from pathlib import Path from aisimulate_core.fpm_profile import _MUTABLE_REFERENCES from aisimulate_core.fpm_profile import FpmDeploymentProfile as FpmDeploymentProfile @@ -56,15 +58,61 @@ def _quantization_from_engine_config(config_json: str) -> dict[str, str]: return modes +@cache +def _load_pinned_checkpoint_config(model: str, revision: str) -> dict | None: + """Reuse immutable remote metadata across candidate normalization and replay.""" + from aisimulate_core.sdk.utils import _download_hf_json + + return _download_hf_json(model, "config.json", revision=revision) + + +def _validate_checkpoint_architecture(profile: FpmModelProfile) -> None: + """Verify raw checkpoint metadata without loading weights or remote code.""" + from aisimulate_core.sdk.utils import HuggingFaceDownloadError, _load_local_config + + checkpoint = Path(profile.model).expanduser() + identity = f"{profile.model!r} at model_revision={profile.model_revision!r}" + try: + if checkpoint.exists() or checkpoint.is_absolute() or profile.model.startswith(("./", "../", "~")): + metadata = _load_local_config(str(checkpoint)) + else: + # Bundled configs do not record their revision and cannot establish + # the architecture of this pinned checkpoint. + metadata = _load_pinned_checkpoint_config(profile.model, profile.model_revision) + except (HuggingFaceDownloadError, OSError, ValueError) as error: + raise ValueError( + f"cannot verify FPM profile architecture: config.json metadata for {identity} " + f"could not be resolved; make the pinned checkpoint configuration available: {error}" + ) from error + if not isinstance(metadata, dict): + raise ValueError(f"FPM profile checkpoint config.json must be a mapping for {identity}") + # Match the collector's text-model view of checkpoint metadata. + text_config = metadata.get("text_config") + if isinstance(text_config, dict): + metadata = {**metadata, **text_config} + architectures = metadata.get("architectures") + if ( + not isinstance(architectures, list) + or not architectures + or not isinstance(architectures[0], str) + or not architectures[0].strip() + ): + raise ValueError( + f"FPM profile checkpoint config.json must declare a nonempty architectures list for {identity}" + ) + if profile.architecture != architectures[0]: + raise ValueError( + f"FPM profile architecture mismatch for {identity}: " + f"checkpoint={architectures[0]!r}, profile={profile.architecture!r}" + ) + + def _validate_forward_pass_profile(config_json: str) -> str: - """Validate schema/identity and return registration facts to the Rust owner. + """Validate identity against checkpoint metadata and return registration facts. This path reads no timing data and does not construct an analytical model. Estimator defaults and interpolation selection belong to Rust. """ - from aisimulate_core.sdk import common - from aisimulate_core.sdk.models.base import _MODEL_REGISTRY - config = json.loads(config_json) profile = load_fpm_profile(config["fpm_profile"]) version = config.get("backend_version") @@ -82,6 +130,11 @@ def _validate_forward_pass_profile(config_json: str) -> str: moe_ep_size=config.get("moe_ep_size"), ) deployment.validate_overrides(**{key: config.get(key) for key in deployment.quantization_kwargs()}) + _validate_checkpoint_architecture(profile) + + from aisimulate_core.sdk import common + from aisimulate_core.sdk.models.base import _MODEL_REGISTRY + family = common.ARCHITECTURE_TO_MODEL_FAMILY.get(profile.architecture, profile.architecture) return json.dumps( { diff --git a/python/aisimulate/src/aisimulate_core/sdk/rust_engine_step.py b/python/aisimulate/src/aisimulate_core/sdk/rust_engine_step.py index acf18944d7..bd4fe058da 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/rust_engine_step.py +++ b/python/aisimulate/src/aisimulate_core/sdk/rust_engine_step.py @@ -22,7 +22,7 @@ import threading from collections import OrderedDict from collections.abc import Mapping -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, fields from dataclasses import field as dataclass_field from importlib import resources as pkg_resources from pathlib import Path @@ -167,7 +167,12 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class ForwardPassPerfOptions: - """Runtime observation, regression, correction, and capacity controls.""" + """Legacy controls; serialization retains only explicit constructor arguments. + + Use ``ForwardPassPerfOptions(**(options.to_dict() | changes))`` to retain + omitted fields when modifying options. ``dataclasses.replace`` supplies + every field to the constructor, making all of its values explicit. + """ max_observations: int = 64 min_observations: int = 5 @@ -183,8 +188,25 @@ class ForwardPassPerfOptions: bucket_shape: tuple[int, int] | None = None regression_ridge_scale: float = 1e-9 + def __new__(cls, *args: Any, **kwargs: Any) -> ForwardPassPerfOptions: + instance = super().__new__(cls) + # Capture presence before the dataclass initializer supplies defaults. + # Comparing values against defaults would discard explicit user choices. + positional = tuple(field.name for field in fields(cls))[: len(args)] + object.__setattr__(instance, "_explicit_fields", frozenset((*positional, *kwargs))) + return instance + + def __setstate__(self, state: dict[str, Any]) -> None: + for name, value in state.items(): + object.__setattr__(self, name, value) + if "_explicit_fields" not in state: + # Older pickles have saved values but no constructor-presence metadata. + object.__setattr__( + self, "_explicit_fields", frozenset(field.name for field in fields(self) if field.name in state) + ) + def to_dict(self) -> dict[str, Any]: - return asdict(self) + return {name: value for name, value in asdict(self).items() if name in self._explicit_fields} class RustForwardPassPerfModel: @@ -292,6 +314,15 @@ def estimate_forward_pass_time_ms(self, metrics: dict[str, Any] | list[dict[str, """ return self._inner.estimate_forward_pass_time_ms(_json_dumps(metrics)) + def estimate_forward_pass_detailed(self, metrics: dict[str, Any] | list[dict[str, Any]]) -> dict[str, Any]: + """Return timing with direct-FPM support, rank composition and correction. + + Measurement weights explain raw lookups. ``ranks`` records mixed-pass + subtraction and ``max_rank`` identifies the native rank maximum, before + ``correction_factor``. Other estimators have no direct lookup evidence. + """ + return json.loads(self._inner.estimate_forward_pass_detailed(_json_dumps(metrics))) + def tune_with_fpms(self, iterations: dict[str, Any] | list[Any]) -> None: """API: ``model.tune_with_fpms(iterations) -> None``. diff --git a/python/aisimulate/src/aisimulate_core/sdk/utils.py b/python/aisimulate/src/aisimulate_core/sdk/utils.py index 94d6251ce9..1a94f5a254 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/utils.py +++ b/python/aisimulate/src/aisimulate_core/sdk/utils.py @@ -17,6 +17,7 @@ import urllib.request from functools import cache from pathlib import Path +from urllib.parse import quote import yaml @@ -431,9 +432,9 @@ def _get_hf_auth_headers() -> dict[str, str]: return headers -def _download_hf_json(hf_id: str, filename: str, *, raise_on_404: bool = True) -> dict | None: +def _download_hf_json(hf_id: str, filename: str, *, raise_on_404: bool = True, revision: str = "main") -> dict | None: """Download and parse a JSON file from a HuggingFace model repo.""" - url = f"https://huggingface.co/{hf_id}/raw/main/{filename}" + url = f"https://huggingface.co/{hf_id}/raw/{quote(revision, safe='')}/{filename}" try: req = urllib.request.Request(url, headers=_get_hf_auth_headers()) with urllib.request.urlopen(req, timeout=30) as response: diff --git a/python/aisimulate/tests/cross_package/test_import_contract.py b/python/aisimulate/tests/cross_package/test_import_contract.py index 9c9d331fcd..1d2a82c23c 100644 --- a/python/aisimulate/tests/cross_package/test_import_contract.py +++ b/python/aisimulate/tests/cross_package/test_import_contract.py @@ -184,11 +184,14 @@ def test_fpm_profile_alias_instances_load_and_compile(namespace: str, tmp_path: from aisimulate_core.sdk import engine from aisimulate_core.sdk.fpm_profile import FpmModelProfile, load_fpm_profile + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + (checkpoint / "config.json").write_text(json.dumps({"architectures": ["ImportContractDecoderForCausalLM"]})) alias = importlib.import_module(f"{namespace}.fpm_profile") profile = alias.FpmModelProfile.model_validate( { "schema_version": 1, - "model": "test/unknown-decoder", + "model": str(checkpoint), "model_revision": "import-contract-fixture-v1", "architecture": "ImportContractDecoderForCausalLM", "context_length": 4096, diff --git a/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py b/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py index 39da2d837b..09f733eabf 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py @@ -289,26 +289,55 @@ def test_profile_preserves_explicit_prefill_envelope(no_models_or_timing_data): assert arguments[arguments.index("--max-num-seqs") + 1] == expected_batch -@pytest.mark.parametrize("actual_version", ["0.25.1", "0.24.0"]) -def test_profile_runtime_version_is_observed_before_execution(tmp_path, monkeypatch, actual_version): +@pytest.mark.parametrize( + ("expected_version", "actual_version"), + [ + ("0.25.1", "0.25.1"), + ("0.25.1+custom", "0.25.1+custom"), + ("0.25.2.dev3", "0.25.2.dev3"), + ("0.25.2.dev3+custom", "0.25.2.dev3+custom"), + ("0.25.1", "0.24.0"), + ("0.25.1", "0.25.1+custom"), + ("0.25.1+custom", "0.25.1"), + ("0.25.1+custom", "0.25.1+other"), + ("0.25.2.dev3", "0.25.2.dev4"), + ("0.25.2", "0.25.2.dev3"), + ], +) +def test_profile_runtime_version_is_observed_before_execution(tmp_path, monkeypatch, expected_version, actual_version): resource = object.__new__(runner.KubernetesCellRunner) monkeypatch.setattr(runner, "FPM_RESULTS_DIR", str(tmp_path)) monkeypatch.setattr(importlib.metadata, "version", lambda _: actual_version) def execute(_pod, command, timeout): if command[:2] == ["python3", "-c"]: - monkeypatch.setattr(sys, "argv", ["-c", *command[3:]]) - exec(command[2], {}) + with monkeypatch.context() as patch: + patch.setattr(sys, "argv", ["-c", *command[3:]]) + exec(command[2], {}) resource._exec_checked = execute - arguments = dict(cell_id="cell", plan_sha256="plan", attempt_id="attempt", expected_backend_version="0.25.1") - if actual_version != "0.25.1": - with pytest.raises(RuntimeError, match="profile runtime mismatch"): + arguments = dict( + cell_id="cell", plan_sha256="plan", attempt_id="attempt", expected_backend_version=expected_version + ) + if actual_version != expected_version: + with pytest.raises(RuntimeError, match="profile runtime mismatch") as failure: resource.prepare_attempt(["pod-0"], **arguments) + message = str(failure.value) + assert f"actual={actual_version!r}, expected={expected_version!r}" in message + assert "Use the pinned runtime" in message + assert f"create a new collection profile with backend_version={actual_version!r}" in message + assert "regenerate the collection plan in a new output directory" in message else: resource.prepare_attempt(["pod-0"], **arguments) provenance = json.loads((tmp_path / "collector-provenance.json").read_text()) - assert provenance["runtime"]["backend_version"] == actual_version + assert provenance == { + "schema_name": "aic_fpm_collector_provenance", + "schema_version": 1, + "cell_id": "cell", + "plan_sha256": "plan", + "attempt_id": "attempt", + "runtime": {"backend": "vllm", "backend_version": actual_version}, + } def test_profile_input_is_not_valid_for_op_collection(): diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_forward.py b/python/aisimulate/tests/unit/sdk/test_fpm_forward.py index 0462fe15ec..96fff7ac28 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_forward.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_forward.py @@ -1029,9 +1029,13 @@ def test_fpm_detail_distinguishes_memory_budget_from_runtime_capacity(external_f assert memory["estimated_num_gpu_blocks"] > 0 assert "num_gpu_blocks" not in memory assert set(sections) == {"summary", "memory", "time", "energy", "source"} - assert sections["time"]["diagnostics"]["status"] == "unavailable" - assert sections["source"]["status"] == "unavailable" - assert "whole-model FPM" in sections["source"]["unavailable_reason"] + assert sections["time"]["diagnostics"]["status"] == "available" + assert sections["source"]["status"] == "available" + operations = [op for phase in sections["source"]["phases"] for op in phase["operations"]] + assert operations and all(op["fpm_estimates"] for op in operations) + assert sections["energy"]["status"] == "unsupported" + assert sections["energy"]["diagnostics"]["power_w"] is None + assert sections["energy"]["diagnostics"]["power_coverage"] is None assert sections["time"]["serving_metrics"]["mean_ttft_ms"] > 0 diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py b/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py new file mode 100644 index 0000000000..8d7a7dd26f --- /dev/null +++ b/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py @@ -0,0 +1,252 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Explicit legacy estimator controls survive migration and saved-config reloads.""" + +import copy +import json +import pickle + +import pytest + +from aisimulate_core.sdk import ForwardPassPerfModelConfig, ForwardPassPerfOptions, RustForwardPassPerfModel + +pytestmark = pytest.mark.unit + + +def _legacy_config(estimator_config=None, **extra): + if estimator_config is not None: + extra["estimator_config"] = json.dumps(estimator_config) + return { + "schema_version": 1, + "model_name": "test/model", + "system_name": "test-system", + "backend": "vllm", + "tp_size": 1, + "pp_size": 1, + "extra": extra, + } + + +def _reload(canonical): + return ForwardPassPerfModelConfig(**json.loads(json.dumps(canonical.to_dict()))) + + +def test_migration_merges_saved_and_caller_controls_before_round_trip(): + legacy = _legacy_config( + { + "fpm_interpolation": {"method": "sol"}, + "features": {"ffn_token_weight": 2}, + "correction": {"enabled": False, "factor_bounds": {"max": 3}}, + "fpm_regression": {"fit": {"kind": "standardized_nnls"}}, + } + ) + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config( + legacy, + "prefill", + { + "max_observations": 128, + "min_observations": 8, + "min_faster_correction_factor": 0.9, + "regression_attention_kv_weight": 7, + "bucket_shape": [4, 16], + "regression_ridge_scale": 0, + }, + ) + controls = RustForwardPassPerfModel.normalize_config(_reload(canonical))["estimator_config"] + assert controls["fpm_interpolation"]["method"] == "sol" + assert controls["features"] == { + "attention_kv_weight": 7, + "prefill_attention_pair_weight": 1, + "ffn_token_weight": 2, + } + assert controls["correction"]["enabled"] is False + assert controls["correction"]["factor_bounds"] == {"min": 0.9, "max": 3} + for store in ("correction", "fpm_regression"): + assert controls[store]["sampling"] == {"bins_per_axis": [4, 16], "max_observations": 128} + assert controls[store]["min_observations"] == 8 + assert controls["fpm_regression"]["fit"] == {"kind": "standardized_nnls", "singular_ridge_scale": 0} + + +@pytest.mark.parametrize("options", [None, {}, ForwardPassPerfOptions()]) +def test_omitted_caller_fields_do_not_override_saved_controls(options): + saved = { + "correction": {"enabled": False, "sampling": {"max_observations": 128}}, + "fpm_regression": {"fit": {"singular_ridge_scale": 0}}, + "features": {"attention_kv_weight": 7}, + } + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(saved), "prefill", options) + controls = _reload(canonical).estimator_config + assert controls["correction"]["enabled"] is False + assert controls["correction"]["sampling"]["max_observations"] == 128 + assert controls["fpm_regression"]["sampling"]["max_observations"] == 64 + assert controls["fpm_regression"]["fit"]["singular_ridge_scale"] == 0 + assert controls["features"]["attention_kv_weight"] == 7 + + +def test_legacy_options_serialize_presence_including_explicit_defaults(): + options = ForwardPassPerfOptions(64, min_faster_correction_factor=None, regression_ridge_scale=0) + assert options.min_observations == 5 # Existing attribute defaults remain available. + assert options.to_dict() == { + "max_observations": 64, + "min_faster_correction_factor": None, + "regression_ridge_scale": 0, + } + loaded = ForwardPassPerfOptions(**json.loads(json.dumps(options.to_dict()))) + assert loaded.to_dict() == options.to_dict() + saved = {"features": {"attention_kv_weight": 7}} + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(saved), "decode", loaded) + controls = _reload(canonical).estimator_config + assert controls["features"]["attention_kv_weight"] == 7 + assert controls["correction"]["factor_bounds"]["min"] is None + assert controls["fpm_regression"]["fit"]["singular_ridge_scale"] == 0 + + +@pytest.mark.parametrize("clone", [copy.copy, copy.deepcopy]) +@pytest.mark.parametrize("controls", [{}, {"max_observations": 64, "regression_ridge_scale": 0}]) +def test_legacy_options_copy_preserves_explicit_fields(clone, controls): + assert clone(ForwardPassPerfOptions(**controls)).to_dict() == controls + + +@pytest.mark.parametrize("protocol", range(pickle.HIGHEST_PROTOCOL + 1)) +@pytest.mark.parametrize("controls", [{}, {"max_observations": 64, "regression_ridge_scale": 0}]) +def test_legacy_options_pickle_preserves_explicit_fields(protocol, controls): + options = ForwardPassPerfOptions(**controls) + assert pickle.loads(pickle.dumps(options, protocol=protocol)).to_dict() == controls + + +@pytest.mark.parametrize("protocol", range(pickle.HIGHEST_PROTOCOL + 1)) +def test_legacy_options_pickle_without_presence_preserves_saved_controls(protocol): + options = ForwardPassPerfOptions(max_observations=128, min_observations=8, regression_ridge_scale=0) + # Older pickles saved dataclass attributes without constructor-presence metadata. + object.__delattr__(options, "_explicit_fields") + restored = pickle.loads(pickle.dumps(options, protocol=protocol)) + assert restored.to_dict() == vars(options) + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(), "prefill", restored) + controls = _reload(canonical).estimator_config + for store in ("correction", "fpm_regression"): + assert controls[store]["sampling"]["max_observations"] == 128 + assert controls[store]["min_observations"] == 8 + assert controls["fpm_regression"]["fit"]["singular_ridge_scale"] == 0 + + +@pytest.mark.parametrize( + ("saved", "options"), + [ + ({"features": {"attention_kv_weight": 1}}, {"regression_attention_kv_weight": 1.0}), + ({"correction": {"factor_bounds": {"min": None}}}, {"min_faster_correction_factor": None}), + ({"correction": {"factor_bounds": {"min": 0.5}}}, {"min_faster_correction_factor": 0.5}), + ({"fpm_regression": {"sampling": {"bins_per_axis": [4, 16]}}}, {"bucket_shape": [4, 16]}), + ({"correction": {"sampling": {"bins_per_axis": [8, 8]}}}, {"bucket_count": 64}), + ], +) +def test_agreeing_overlap_is_accepted(saved, options): + saved = saved | {"fpm_interpolation": {"method": "sol"}} + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(saved), "decode", options) + assert ( + RustForwardPassPerfModel.normalize_config(_reload(canonical))["estimator_config"] == canonical.estimator_config + ) + + +@pytest.mark.parametrize( + ("saved", "options", "field"), + [ + ( + {"fpm_regression": {"sampling": {"max_observations": 128}}}, + ForwardPassPerfOptions(max_observations=64), + "fpm_regression.sampling.max_observations", + ), + ( + {"features": {"attention_kv_weight": 7}}, + {"regression_attention_kv_weight": 1}, + "features.attention_kv_weight", + ), + ( + {"correction": {"factor_bounds": {"min": 0.5}}}, + {"min_faster_correction_factor": None}, + "correction.factor_bounds.min", + ), + ( + {"correction": {"sampling": {"bins_per_axis": [4, 16]}}}, + {"bucket_shape": [4, 4]}, + "correction.sampling.bins_per_axis", + ), + ( + {"fpm_regression": {"fit": {"singular_ridge_scale": 1e-9}}}, + {"regression_ridge_scale": 0}, + "fpm_regression.fit.singular_ridge_scale", + ), + ], +) +def test_conflicting_overlap_identifies_canonical_field(saved, options, field): + with pytest.raises(ValueError, match=rf"conflicting explicit values for estimator_config\.{field}"): + ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(saved), "prefill", options) + + +def test_sampling_validation_uses_merged_values(): + saved = { + "correction": {"sampling": {"max_observations": 128}}, + "fpm_regression": {"sampling": {"max_observations": 128}}, + } + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config( + _legacy_config(saved), "prefill", {"min_observations": 80} + ) + controls = _reload(canonical).estimator_config + assert controls["correction"]["min_observations"] == 80 + assert controls["fpm_regression"]["min_observations"] == 80 + with pytest.raises(ValueError, match="min_observations must be <= max_observations"): + ForwardPassPerfModelConfig.from_legacy_engine_config( + _legacy_config(saved), "prefill", {"min_observations": 129} + ) + + +@pytest.mark.parametrize("canonical_method", [None, "direct"]) +def test_omitted_and_agreeing_interpolation_methods_accept_legacy_selection(canonical_method): + controls = {"fpm_interpolation": {} if canonical_method is None else {"method": canonical_method}} + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config( + _legacy_config(controls, fpm_interpolation="direct"), "prefill" + ) + assert _reload(canonical).estimator_config["fpm_interpolation"]["method"] == "direct" + + +@pytest.mark.parametrize("canonical_method,legacy_method", [("auto", "direct"), ("direct", "auto"), ("sol", "direct")]) +def test_explicit_interpolation_method_conflicts_are_symmetric(canonical_method, legacy_method): + with pytest.raises(ValueError, match=r"estimator_config\.fpm_interpolation\.method"): + ForwardPassPerfModelConfig.from_legacy_engine_config( + _legacy_config({"fpm_interpolation": {"method": canonical_method}}, fpm_interpolation=legacy_method), + "prefill", + ) + + +@pytest.mark.parametrize("legacy_path", ["/tmp/fpm.parquet", "/tmp/./fpm.parquet", "/tmp//fpm.parquet"]) +def test_equivalent_parquet_paths_merge(legacy_path): + legacy = _legacy_config({"fpm_interpolation": {"fpm_parquet_path": "/tmp/fpm.parquet"}}) + legacy.update(forward_model="fpm", fpm_parquet_path=legacy_path) + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config(legacy, "prefill") + assert _reload(canonical).estimator_config["fpm_interpolation"]["fpm_parquet_path"] == "/tmp/fpm.parquet" + + +@pytest.mark.parametrize("legacy_path", ["/tmp/different.parquet", "/tmp/subdir/../fpm.parquet"]) +def test_different_parquet_paths_conflict_without_filesystem_resolution(legacy_path): + legacy = _legacy_config({"fpm_interpolation": {"fpm_parquet_path": "/tmp/fpm.parquet"}}) + legacy.update(forward_model="fpm", fpm_parquet_path=legacy_path) + with pytest.raises(ValueError, match=r"estimator_config\.fpm_interpolation\.fpm_parquet_path"): + ForwardPassPerfModelConfig.from_legacy_engine_config(legacy, "prefill") + + +@pytest.mark.parametrize( + ("saved", "options", "reason"), + [ + ({"correction": {"enabld": False}}, {}, "unknown field"), + ({"fpm_regression": {"sampling": {"bins_per_axis": [4]}}}, {}, "length"), + ({"correction": None}, {}, "invalid type"), + ({}, {"max_observatons": 128}, "unknown field"), + ({}, {"min_observations": 0}, "min_observations must be >= 1"), + ({}, {"max_observations": False}, "invalid type"), + ({}, {"bucket_count": 3}, "bucket_count must be a perfect square"), + ({"fpm_regression": {"sampling": {"bins_per_axis": [0, 4]}}}, {}, "bins_per_axis must be positive"), + ], +) +def test_migration_rejects_unknown_and_invalid_controls(saved, options, reason): + with pytest.raises(ValueError, match=reason): + ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(saved), "prefill", options) diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_phase_readiness.py b/python/aisimulate/tests/unit/sdk/test_fpm_phase_readiness.py new file mode 100644 index 0000000000..63cfd2f9a3 --- /dev/null +++ b/python/aisimulate/tests/unit/sdk/test_fpm_phase_readiness.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Direct FPM construction checks each required phase before selecting a root.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +import aisimulate_core +from aisimulate_core.sdk import RustForwardPassPerfModel +from aisimulate_core.sdk.errors import PerfDataNotAvailableError +from aisimulate_core.sdk.fpm_profile import load_fpm_profile + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def phase_roots(tmp_path, monkeypatch): + """Use real metadata, Parquet loading, compilation and native root selection.""" + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + architecture = "UnregisteredDecoderForCausalLM" + (checkpoint / "config.json").write_text(json.dumps({"architectures": [architecture]})) + profile = { + "schema_version": 1, + "model": str(checkpoint), + "model_revision": "phase-readiness-fixture-v1", + "architecture": architecture, + "context_length": 4096, + "num_experts": 64, + "provenance": "Synthetic SDK readiness fixture, not silicon qualification.", + "deployments": [ + { + "system": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "tp": 1, + "dp": 1, + "moe_tp": 1, + "moe_ep": 1, + "gemm_quant_mode": "nvfp4", + "moe_quant_mode": "nvfp4", + "fmha_quant_mode": "fp8", + "comm_quant_mode": "half", + "kv_cache_dtype": "fp8", + "resources": { + "weights_bytes": 100, + "activations_bytes": 20, + "runtime_overhead_bytes": 30, + "comm_overhead_bytes": 50, + "kv_bytes_per_token": 10, + "cache_layout": "linear", + "max_num_tokens": 8192, + "max_batch_size": 256, + "provenance": "Synthetic per-rank resource declarations.", + }, + } + ], + } + config = { + "model": profile["model"], + "system": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "worker_type": "aggregated", + "tp": 1, + "attention_dp": 1, + "moe_tp_size": 1, + "moe_ep_size": 1, + "estimation_mode": "fpm_interpolation", + "fallback_policy": "deny", + "fpm_profile": profile, + "estimator_config": {"fpm_interpolation": {"method": "direct"}}, + } + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + + def make_root(name, *, prefill="genuine", decode="genuine"): + root = tmp_path / name + root.mkdir() + packaged = Path(aisimulate_core.__file__).parent / "systems/h200_sxm.yaml" + (root / packaged.name).write_bytes(packaged.read_bytes()) + identity = load_fpm_profile(profile).deployments[0].model_dump(mode="json", exclude={"resources"}) + rows = [ + { + **identity, + "model_path": profile["model"], + "cell_id": f"synthetic-{phase}", + "weight_quantization": "synthetic", + "workload_kind": phase, + "partition_policy": "balanced_v1", + "batch_size": 1, + "total_prefill_tokens": 1 if phase == "prefill" else 0, + "total_kv_read_tokens": 0 if phase == "prefill" else 1, + "latency_ms": 2.0 if phase == "prefill" else 3.0, + "kv_seed_regime": "real_kv" if kind == "genuine" else "fake_fallback", + } + for phase, kind in (("prefill", prefill), ("decode", decode)) + if kind != "absent" + ] + path = root / "data/h200_sxm/vllm/0.25.1/fpm_forward_perf.parquet" + path.parent.mkdir(parents=True) + pq.write_table(pa.Table.from_pylist(rows), path) + path.with_suffix(".metadata.json").write_text( + json.dumps( + { + "schema_name": "aic_fpm_forward_perf", + "schema_version": 6, + "coordinate_system": "iteration_totals_balanced_v1", + "measurement_policy": "dynamo_native_single_sample_v1", + "system": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "row_count": len(rows), + "parquet_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + ) + return str(root) + + return config, make_root + + +@pytest.mark.parametrize("missing_phase", ["prefill", "decode"]) +@pytest.mark.parametrize("rows", ["absent", "fake"]) +def test_direct_constructor_rejects_missing_genuine_phase(phase_roots, missing_phase, rows): + config, make_root = phase_roots + root = make_root("incomplete", **{missing_phase: rows}) + config["systems_paths"] = [root] + with pytest.raises(PerfDataNotAvailableError, match=f"direct FPM {missing_phase}") as exc: + RustForwardPassPerfModel.best_available(config) + message = str(exc.value) + assert "no genuine measurements" in message + assert f"Collect genuine {missing_phase} FPM rows" in message + assert root in message + + +@pytest.mark.parametrize("missing_phase", ["prefill", "decode"]) +@pytest.mark.parametrize("rows", ["absent", "fake"]) +@pytest.mark.parametrize("fallback", ["deny", "allow"]) +def test_direct_constructor_selects_later_root_with_both_required_phases(phase_roots, missing_phase, rows, fallback): + config, make_root = phase_roots + first = make_root("incomplete", **{missing_phase: rows}) + second = make_root("usable") + config.update(systems_paths=[first, second], fallback_policy=fallback) + model = RustForwardPassPerfModel.best_available(config) + before = model.diagnostics() + assert before["readiness"] == "ready" + provenance = before["provenance"] + assert provenance["selected_systems_root"] == second + assert provenance["config"]["systems_paths"] == [second] + assert provenance["selected_estimation_mode"] == "fpm_interpolation" + assert provenance["config"]["estimator_config"]["fpm_interpolation"]["method"] == "direct" + # The selected root's exact, hand-declared synthetic timings remain queryable. + assert ( + model.estimate_forward_pass_time_ms( + {"scheduled_requests": {"num_prefill_requests": 1, "sum_prefill_tokens": 1}} + ) + == 2.0 + ) + assert ( + model.estimate_forward_pass_time_ms( + {"scheduled_requests": {"num_decode_requests": 1, "sum_decode_kv_tokens": 1}} + ) + == 3.0 + ) + with pytest.raises(PerfDataNotAvailableError, match="direct"): + model.estimate_forward_pass_time_ms( + {"scheduled_requests": {"num_decode_requests": 1, "sum_decode_kv_tokens": 64}} + ) + assert model.diagnostics() == before diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py index 106d856057..aa387422d2 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py @@ -13,7 +13,7 @@ import pytest from pydantic import ValidationError -from aisimulate_core.sdk import ForwardPassPerfModelConfig, RustForwardPassPerfModel, engine, memory +from aisimulate_core.sdk import ForwardPassPerfModelConfig, RustForwardPassPerfModel, engine, fpm_profile, memory, utils from aisimulate_core.sdk.errors import PerfDataNotAvailableError from aisimulate_core.sdk.fpm_profile import FpmModelProfile, load_fpm_profile @@ -21,7 +21,26 @@ @pytest.fixture -def profile_dict(): +def checkpoint_configs(monkeypatch): + fpm_profile._load_pinned_checkpoint_config.cache_clear() + configs = { + "test/unknown-decoder": {"architectures": ["UnregisteredDecoderForCausalLM"]}, + "nvidia/GLM-5.2-NVFP4": {"architectures": ["GlmMoeDsaForCausalLM"]}, + "MiniMaxAI/MiniMax-M2.7": {"architectures": ["MiniMaxM2ForCausalLM"]}, + } + + def download(model, filename, *, revision): + assert filename == "config.json" + assert revision == "profile-test-fixture-v1" + return copy.deepcopy(configs[model]) + + monkeypatch.setattr(utils, "_download_hf_json", download) + yield configs + fpm_profile._load_pinned_checkpoint_config.cache_clear() + + +@pytest.fixture +def profile_dict(checkpoint_configs): return { "schema_version": 1, "model": "test/unknown-decoder", @@ -87,7 +106,135 @@ def _request(profile, method="auto", **overrides): def _normalize(config): - return json.loads(engine.aisimulate_core.RustForwardPassPerfModel.normalize_config(json.dumps(config))) + return RustForwardPassPerfModel.normalize_config(config) + + +@pytest.mark.parametrize( + "entry_point", [RustForwardPassPerfModel.normalize_config, RustForwardPassPerfModel.best_available] +) +@pytest.mark.parametrize("method", ["auto", "direct"]) +@pytest.mark.parametrize("architecture", ["LlamaForCausalLm", "Qwen2ForCausalLM"]) +def test_profile_architecture_mismatch_fails_before_selection( + profile_dict, checkpoint_configs, monkeypatch, entry_point, method, architecture +): + checkpoint_configs[profile_dict["model"]] = {"architectures": ["LlamaForCausalLM"]} + profile_dict["architecture"] = architecture + monkeypatch.setattr(engine, "compile_engine", _fail_graph) + with pytest.raises(ValueError, match="FPM profile architecture mismatch") as error: + entry_point(_request(profile_dict, method, estimation_mode="auto", fallback_policy="allow")) + assert "LlamaForCausalLM" in str(error.value) + assert architecture in str(error.value) + assert profile_dict["model_revision"] in str(error.value) + + +@pytest.mark.parametrize( + "entry_point", [RustForwardPassPerfModel.normalize_config, RustForwardPassPerfModel.best_available] +) +@pytest.mark.parametrize( + "metadata", + [ + None, + [], + {}, + {"model_type": "llama"}, + {"architectures": []}, + {"architectures": "LlamaForCausalLM"}, + {"architectures": [None]}, + {"architectures": [" "]}, + ], +) +def test_profile_missing_architecture_is_not_unregistered( + profile_dict, checkpoint_configs, monkeypatch, entry_point, metadata +): + checkpoint_configs[profile_dict["model"]] = metadata + monkeypatch.setattr(engine, "compile_engine", _fail_graph) + with pytest.raises(ValueError, match="checkpoint config.json must"): + entry_point(_request(profile_dict, estimation_mode="auto", fallback_policy="allow")) + + +@pytest.mark.parametrize( + "entry_point", [RustForwardPassPerfModel.normalize_config, RustForwardPassPerfModel.best_available] +) +def test_profile_unresolvable_checkpoint_is_not_unregistered(profile_dict, monkeypatch, entry_point): + def unavailable(*_args, **_kwargs): + raise utils.HuggingFaceDownloadError("pinned revision is unavailable") + + monkeypatch.setattr(utils, "_download_hf_json", unavailable) + monkeypatch.setattr(engine, "compile_engine", _fail_graph) + with pytest.raises(ValueError, match="cannot verify FPM profile architecture") as error: + entry_point(_request(profile_dict, estimation_mode="auto", fallback_policy="allow")) + assert profile_dict["model_revision"] in str(error.value) + assert "pinned revision is unavailable" in str(error.value) + + +@pytest.mark.parametrize( + "model,architecture,method", + [ + ("test/unknown-decoder", "UnregisteredDecoderForCausalLM", "direct"), + ("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM", "sol"), + ], +) +def test_profile_architecture_metadata_uses_pinned_revision(profile_dict, monkeypatch, model, architecture, method): + calls = [] + profile_dict.update(model=model, architecture=architecture, model_revision="a" * 40) + + def download(model, filename, *, revision): + calls.append((model, filename, revision)) + return {"architectures": [architecture]} + + monkeypatch.setattr(utils, "_download_hf_json", download) + monkeypatch.setattr(utils, "_load_pre_downloaded_hf_config", _fail_graph) + monkeypatch.setattr(utils, "_parse_hf_config_json", _fail_graph) + resolved = RustForwardPassPerfModel.normalize_config(_request(profile_dict)) + assert RustForwardPassPerfModel.normalize_config(_request(profile_dict)) == resolved + assert calls == [(profile_dict["model"], "config.json", "a" * 40)] + assert resolved["estimator_config"]["fpm_interpolation"]["method"] == method + + +def test_profile_metadata_cache_separates_checkpoint_revisions(profile_dict, monkeypatch): + calls = [] + architectures = {"a" * 40: "UnregisteredDecoderForCausalLM", "b" * 40: "LlamaForCausalLM"} + + def download(model, filename, *, revision): + calls.append((model, filename, revision)) + return {"architectures": [architectures[revision]]} + + monkeypatch.setattr(utils, "_download_hf_json", download) + for revision, method in (("a" * 40, "direct"), ("b" * 40, "sol"), ("a" * 40, "direct")): + profile_dict.update(model_revision=revision, architecture=architectures[revision]) + assert _normalize(_request(profile_dict))["estimator_config"]["fpm_interpolation"]["method"] == method + assert calls == [(profile_dict["model"], "config.json", revision) for revision in architectures] + + +def test_profile_architecture_matches_collector_text_config_view(profile_dict, checkpoint_configs): + checkpoint_configs[profile_dict["model"]] = { + "architectures": ["UnknownForConditionalGeneration"], + "text_config": {"architectures": [profile_dict["architecture"]]}, + } + assert _normalize(_request(profile_dict))["estimator_config"]["fpm_interpolation"]["method"] == "direct" + + +@pytest.mark.parametrize("metadata", [None, "not JSON"]) +def test_local_checkpoint_missing_or_invalid_config_fails_without_download( + profile_dict, monkeypatch, tmp_path, metadata +): + profile_dict["model"] = str(tmp_path) + if metadata is not None: + (tmp_path / "config.json").write_text(metadata) + monkeypatch.setattr(utils, "_download_hf_json", _fail_graph) + with pytest.raises(ValueError, match="cannot verify FPM profile architecture"): + RustForwardPassPerfModel.best_available(_request(profile_dict, fallback_policy="allow")) + + +def test_local_checkpoint_architecture_is_revalidated(profile_dict, monkeypatch, tmp_path): + profile_dict["model"] = str(tmp_path) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({"architectures": [profile_dict["architecture"]]})) + monkeypatch.setattr(utils, "_download_hf_json", _fail_graph) + assert _normalize(_request(profile_dict))["estimator_config"]["fpm_interpolation"]["method"] == "direct" + config_path.write_text(json.dumps({"architectures": ["LlamaForCausalLM"]})) + with pytest.raises(ValueError, match="FPM profile architecture mismatch"): + RustForwardPassPerfModel.best_available(_request(profile_dict, fallback_policy="allow")) @pytest.fixture @@ -437,7 +584,7 @@ def native_profile_config(profile_dict, monkeypatch): # Use a bundled system/version for native engine construction. Resource and # precision variants below are transport fixtures, not measured timing cells. monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") - profile_dict.update(model="MiniMaxAI/MiniMax-M2.7") + profile_dict.update(model="MiniMaxAI/MiniMax-M2.7", architecture="MiniMaxM2ForCausalLM") profile_dict["deployments"][0].update(system="h200_sxm") monkeypatch.setattr(engine, "get_model", _fail_graph) specs = [] @@ -532,6 +679,24 @@ def test_legacy_selector_migrates_to_nested_control(native_profile_config): assert isinstance(canonical["fpm_profile"], dict) +def test_saved_profile_compiler_controls_merge_with_caller_options(native_profile_config): + compile_config, _ = native_profile_config + saved = json.loads(json.dumps(compile_config())) + assert json.loads(saved["extra"]["estimator_config"]) == {"fpm_interpolation": {"method": "direct"}} + canonical = ForwardPassPerfModelConfig.from_legacy_engine_config( + saved, + "prefill", + {"max_observations": 128, "min_faster_correction_factor": 0.9, "regression_attention_kv_weight": 7}, + ) + reloaded = ForwardPassPerfModelConfig(**json.loads(json.dumps(canonical.to_dict()))) + controls = RustForwardPassPerfModel.normalize_config(reloaded)["estimator_config"] + assert controls["fpm_interpolation"]["method"] == "direct" + assert controls["correction"]["sampling"]["max_observations"] == 128 + assert controls["fpm_regression"]["sampling"]["max_observations"] == 128 + assert controls["correction"]["factor_bounds"]["min"] == 0.9 + assert controls["features"]["attention_kv_weight"] == 7 + + def _write_external_profile_pair(profile, directory): """Generate routing/identity evidence; these 2/3 ms rows are not silicon data.""" import pyarrow as pa @@ -576,21 +741,63 @@ def _write_external_profile_pair(profile, directory): @pytest.mark.parametrize( - ("model", "system", "tp", "dp", "moe_tp", "moe_ep", "gemm", "fmha"), + "architecture,method", + [("UnregisteredDecoderForCausalLM", "auto"), ("LlamaForCausalLM", "direct")], +) +def test_verified_local_checkpoint_queries_without_analytical_model( + profile_dict, monkeypatch, tmp_path, architecture, method +): + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + # A config-only checkpoint cannot supply weights or a model graph. Remote + # code declarations stay inert; direct timing needs just verified identity. + (checkpoint / "config.json").write_text( + json.dumps({"architectures": [architecture], "auto_map": {"AutoConfig": "untrusted.Configuration"}}) + ) + (checkpoint / "untrusted.py").write_text("raise AssertionError('remote model code was executed')\n") + profile_dict.update(model=str(checkpoint), architecture=architecture) + profile_dict["deployments"][0]["system"] = "h200_sxm" + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + monkeypatch.setattr(utils, "_download_hf_json", _fail_graph) + monkeypatch.setattr(utils, "_parse_hf_config_json", _fail_graph) + monkeypatch.setattr(engine, "get_model", _fail_graph) + monkeypatch.setattr(engine, "build_model_config", _fail_graph) + monkeypatch.setattr(engine, "_maybe_load_database", _fail_graph) + path = _write_external_profile_pair(profile_dict, tmp_path) + config = _request( + profile_dict, + estimator_config={"fpm_interpolation": {"method": method, "fpm_parquet_path": str(path)}}, + ) + resolved = RustForwardPassPerfModel.normalize_config(config) + assert resolved["estimator_config"]["fpm_interpolation"]["method"] == "direct" + forward = RustForwardPassPerfModel.best_available(config) + constructed = forward.diagnostics()["provenance"]["config"] + assert constructed["fpm_profile"] == resolved["fpm_profile"] + assert constructed["estimator_config"] == resolved["estimator_config"] + assert forward.estimate_forward_pass_time_ms( + {"scheduled_requests": {"num_prefill_requests": 1, "sum_prefill_tokens": 1}} + ) == pytest.approx(2.0) + assert forward.estimate_forward_pass_time_ms( + {"scheduled_requests": {"num_decode_requests": 1, "sum_decode_kv_tokens": 1}} + ) == pytest.approx(3.0) + + +@pytest.mark.parametrize( + ("model", "architecture", "system", "tp", "dp", "moe_tp", "moe_ep", "gemm", "fmha"), [ - ("MiniMaxAI/MiniMax-M2.7", "h200_sxm", 4, 1, 4, 1, "fp8_block", "bfloat16"), - ("nvidia/GLM-5.2-NVFP4", "b200_sxm", 1, 8, 1, 8, "nvfp4", "fp8"), - ("nvidia/GLM-5.2-NVFP4", "b200_sxm", 8, 1, 1, 8, "nvfp4", "fp8"), + ("MiniMaxAI/MiniMax-M2.7", "MiniMaxM2ForCausalLM", "h200_sxm", 4, 1, 4, 1, "fp8_block", "bfloat16"), + ("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM", "b200_sxm", 1, 8, 1, 8, "nvfp4", "fp8"), + ("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM", "b200_sxm", 8, 1, 1, 8, "nvfp4", "fp8"), ], ) def test_external_fpm_cells_query_with_model_construction_disabled( - profile_dict, monkeypatch, tmp_path, model, system, tp, dp, moe_tp, moe_ep, gemm, fmha + profile_dict, monkeypatch, tmp_path, model, architecture, system, tp, dp, moe_tp, moe_ep, gemm, fmha ): # Synthetic timing and resource declarations test the real checkpoint # identities through external storage with all graph constructors disabled. # Accuracy against retained silicon measurements is separate validation. monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") - profile_dict.update(model=model) + profile_dict.update(model=model, architecture=architecture) profile_dict["deployments"][0].update( system=system, tp=tp, diff --git a/python/aisimulate/tests/unit/sdk/test_utils.py b/python/aisimulate/tests/unit/sdk/test_utils.py index d764295f81..cf3a757ce0 100644 --- a/python/aisimulate/tests/unit/sdk/test_utils.py +++ b/python/aisimulate/tests/unit/sdk/test_utils.py @@ -7,12 +7,14 @@ Tests HuggingFace config parsing and model config retrieval. """ +import io +import json from types import SimpleNamespace from unittest.mock import patch import pytest -from aisimulate.sdk import common, config +from aisimulate.sdk import common, config, utils from aisimulate.sdk.backends.base_backend import BaseBackend from aisimulate.sdk.models import Gemma4MixModel, HybridMoEModel from aisimulate.sdk.utils import ( @@ -25,6 +27,26 @@ pytestmark = pytest.mark.unit +@pytest.mark.parametrize( + "revision,path_revision", + [(None, "main"), ("a" * 40, "a" * 40), ("release/v1", "release%2Fv1")], +) +def test_download_hf_config_json_revision(monkeypatch, revision, path_revision): + requests = [] + metadata = {"architectures": ["UnregisteredDecoderForCausalLM"]} + + def fetch(request, *, timeout): + requests.append(request.full_url) + assert timeout == 30 + return io.BytesIO(json.dumps(metadata).encode()) + + monkeypatch.setattr(utils, "_get_hf_auth_headers", lambda: {}) + monkeypatch.setattr(utils.urllib.request, "urlopen", fetch) + kwargs = {} if revision is None else {"revision": revision} + assert utils._download_hf_json("test/model", "config.json", **kwargs) == metadata + assert requests == [f"https://huggingface.co/test/model/raw/{path_revision}/config.json"] + + class TestParseHFConfig: """Test HuggingFace config parsing.""" diff --git a/python/aisimulate/tests/unit/test_fpm_config_imports.py b/python/aisimulate/tests/unit/test_fpm_config_imports.py index 55202ae007..04a59b74f5 100644 --- a/python/aisimulate/tests/unit/test_fpm_config_imports.py +++ b/python/aisimulate/tests/unit/test_fpm_config_imports.py @@ -69,7 +69,8 @@ def _config() -> dict: from pathlib import Path blocked = ('aisimulate._runtime', 'aisimulate.sweeper', 'numpy', 'pandas', 'aiconfigurator', - 'aiconfigurator_core', 'aisimulate_core._native', 'aisimulate_core.sdk') + 'aiconfigurator_core', 'aisimulate_core._native', 'aisimulate_core.sdk', + 'urllib.request', 'huggingface_hub', 'transformers') def assert_lightweight(): loaded = [name for name in sys.modules if any(name == item or name.startswith(item + '.') for item in blocked)] diff --git a/tests/test_fpm_profile_workflow.py b/tests/test_fpm_profile_workflow.py index 3751c47599..674dae9c63 100644 --- a/tests/test_fpm_profile_workflow.py +++ b/tests/test_fpm_profile_workflow.py @@ -35,7 +35,9 @@ @pytest.fixture -def profile(): +def profile(tmp_path_factory): + checkpoint = tmp_path_factory.mktemp("workflow-checkpoint") + (checkpoint / "config.json").write_text(json.dumps({"architectures": ["UnregisteredMoeForCausalLM"]})) # Deliberately fictional metadata, with declared byte bounds, proves the # workflow cannot recover resources through an existing model registration. common = { @@ -61,7 +63,7 @@ def profile(): } return { "schema_version": 1, - "model": "test-only/UnregisteredMoe", + "model": str(checkpoint), "model_revision": "workflow-fixture-v1", "architecture": "UnregisteredMoeForCausalLM", "context_length": 4096, diff --git a/tests/test_fpm_query_evidence.py b/tests/test_fpm_query_evidence.py new file mode 100644 index 0000000000..8397148858 --- /dev/null +++ b/tests/test_fpm_query_evidence.py @@ -0,0 +1,454 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executed public FPM evidence over real metadata and synthetic Parquets.""" + +import hashlib +import json +from copy import deepcopy +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +# Reuse the config-only, verified unregistered architecture and its guards. +import test_fpm_profile_workflow as workflow + +from aisimulate.compiler import prediction_to_replay_spec +from aisimulate.config import CorePredictionConfig, CoreRecommendationConfig +from aisimulate.detail import build_prediction_details +from aisimulate.recommend import run_recommendation +from aisimulate.runner import EngineReplayRunnerFactory, _normalize_engine_replay_report +from aisimulate.sweeper.replay import ReplayOutputRequirements +from aisimulate.sweeper.result import SweepResult +from aisimulate_core.sdk import RustForwardPassPerfModel + +pytestmark = pytest.mark.unit + +profile = workflow.profile +timing_systems = workflow.timing_systems +forbid_registered_model = workflow.forbid_registered_model + + +@pytest.fixture +def evidence_systems(timing_systems): + path = Path(timing_systems) / "data/h200_sxm/vllm/0.25.1/fpm_forward_perf.parquet" + templates = [row for row in pq.read_table(path).to_pylist() if row["workload_kind"] == "prefill"] + rows = [] + # Prefill token interpolation at 6 lies 1/4 along [4,12]; KV=2 is + # 1/4 along [0,8]. The four weights are 9/16,3/16,3/16,1/16. + points = [ + ("prefill", 1, 1, 0, 2.0), + ("prefill", 1, 4, 0, 10.0), + ("prefill", 1, 12, 0, 18.0), + ("prefill", 1, 4, 8, 30.0), + ("prefill", 1, 12, 8, 46.0), + ("decode", 1, 0, 0, 3.0), + ("decode", 1, 0, 1, 4.0), + ("decode", 1, 0, 64, 67.0), + ("decode", 2, 0, 4, 10.0), + ("decode", 2, 0, 20, 26.0), + ("decode", 4, 0, 8, 20.0), + ("decode", 4, 0, 24, 52.0), + # A decreasing measured curve exercises marginal-decode clamping. + ("decode", 6, 0, 12, 30.0), + ("decode", 6, 0, 24, 10.0), + ] + for template in templates: + for phase, batch, tokens, kv, latency in points: + rows.append( + { + **template, + "workload_kind": phase, + "batch_size": batch, + "total_prefill_tokens": tokens, + "total_kv_read_tokens": kv, + "latency_ms": latency, + } + ) + pq.write_table(pa.Table.from_pylist(rows), path) + metadata = json.loads(path.with_suffix(".metadata.json").read_text()) + metadata.update(row_count=len(rows), parquet_sha256=hashlib.sha256(path.read_bytes()).hexdigest()) + path.with_suffix(".metadata.json").write_text(json.dumps(metadata)) + return timing_systems + + +def canonical(profile, systems): + return { + "model": profile["model"], + "fpm_profile": profile, + "system": "h200_sxm", + "backend": "vllm", + "backend_version": "0.25.1", + "worker_type": "aggregated", + "tp": 2, + "moe_tp_size": 2, + "moe_ep_size": 1, + "systems_paths": [systems], + "estimation_mode": "fpm_interpolation", + "estimator_config": {"fpm_interpolation": {"method": "direct"}, "correction": {"min_observations": 1}}, + } + + +def prefill(tokens=4, kv=0): + return { + "scheduled_requests": {"num_prefill_requests": 1, "sum_prefill_tokens": tokens, "sum_prefill_kv_tokens": kv} + } + + +def decode(batch=3, kv=12): + return {"scheduled_requests": {"num_decode_requests": batch, "sum_decode_kv_tokens": kv}} + + +@pytest.mark.parametrize( + "metrics,latency,resolution,weights,measurements", + [ + (prefill(), 10.0, "exact_lookup", [1.0], [(1, 4, 0, 10)]), + (prefill(6), 12.0, "within_curve_interpolation", [0.75, 0.25], [(1, 4, 0, 10), (1, 12, 0, 18)]), + ( + prefill(6, 2), + 17.5, + "cross_kv_interpolation", + [0.5625, 0.1875, 0.1875, 0.0625], + [(1, 4, 0, 10), (1, 12, 0, 18), (1, 4, 8, 30), (1, 12, 8, 46)], + ), + (decode(2, 4), 10.0, "exact_lookup", [1.0], [(2, None, 4, 10)]), + (decode(2, 12), 18.0, "within_curve_interpolation", [0.5, 0.5], [(2, None, 4, 10), (2, None, 20, 26)]), + ( + decode(), + 23.0, + "cross_batch_interpolation", + [0.25, 0.25, 0.375, 0.125], + [(2, None, 4, 10), (2, None, 20, 26), (4, None, 8, 20), (4, None, 24, 52)], + ), + ], +) +def test_canonical_detailed_query_support( + profile, evidence_systems, metrics, latency, resolution, weights, measurements +): + model = RustForwardPassPerfModel.best_available(canonical(profile, evidence_systems)) + detailed = model.estimate_forward_pass_detailed(metrics) + assert detailed["latency_ms"] == model.estimate_forward_pass_time_ms(metrics) == latency + assert detailed["native_latency_ms"] == latency + assert detailed["correction_factor"] == 1.0 + evidence = detailed["ranks"][0]["queries"][0] + assert evidence["model_path"] == profile["model"] + assert evidence["resolution"] == resolution + assert evidence["latency_ms"] == latency + assert evidence["decode_baseline"] is False + assert [point["weight"] for point in evidence["support"]] == weights + assert [ + ( + point["coordinates"]["batch_size"], + point["coordinates"]["total_prefill_tokens"], + point["coordinates"]["total_kv_read_tokens"], + point["latency_ms"], + ) + for point in evidence["support"] + ] == measurements + assert json.loads(json.dumps(detailed)) == detailed + + +def test_mixed_baseline_rank_max_correction_and_clamp(profile, evidence_systems): + model = RustForwardPassPerfModel.best_available(canonical(profile, evidence_systems)) + mixed = {"scheduled_requests": {**prefill()["scheduled_requests"], **decode()["scheduled_requests"]}} + metrics = [mixed, decode(4, 24)] + detail = model.estimate_forward_pass_detailed(metrics) + assert detail["latency_ms"] == model.estimate_forward_pass_time_ms(metrics) == 52.0 + assert detail["max_rank"] == 1 + rank = detail["ranks"][0] + assert [ + rank[key] for key in ("prefill_ms", "decode_ms", "decode_baseline_ms", "marginal_decode_ms", "latency_ms") + ] == [10, 23, 15, 8, 18] + baseline = rank["queries"][2] + assert baseline["decode_baseline"] is True + assert baseline["query"]["total_kv_read_tokens"] == 12 + assert [(p["coordinates"]["total_kv_read_tokens"], p["weight"]) for p in baseline["support"]] == [ + (4, 0.5), + (8, 0.5), + ] + observed = deepcopy(metrics) + observed[1]["wall_time"] = 0.104 # seconds: twice the 52 ms native maximum + model.tune_with_fpms([observed]) + corrected = model.estimate_forward_pass_detailed(metrics) + assert corrected["correction_factor"] == 2.0 + assert corrected["latency_ms"] == model.estimate_forward_pass_time_ms(metrics) == 104.0 + assert corrected["ranks"] == detail["ranks"] + clamped = {"scheduled_requests": {**prefill()["scheduled_requests"], **decode(6, 24)["scheduled_requests"]}} + detail = model.estimate_forward_pass_detailed(clamped) + assert detail["ranks"][0]["decode_ms"] == 10 + assert detail["ranks"][0]["decode_baseline_ms"] == 30 + assert detail["ranks"][0]["marginal_decode_ms"] == 0 + assert detail["ranks"][0]["latency_ms"] == 10 + + +def test_uncovered_empty_and_non_fpm_results(profile, evidence_systems): + config = canonical(profile, evidence_systems) + model = RustForwardPassPerfModel.best_available(config) + for query in (prefill(20), prefill(4, 20), decode(3, 30)): + for method in (model.estimate_forward_pass_time_ms, model.estimate_forward_pass_detailed): + with pytest.raises(Exception, match="unsupported|outside"): + method(query) + empty = model.estimate_forward_pass_detailed({}) + assert empty == { + "latency_ms": 0.0, + "native_latency_ms": 0.0, + "correction_factor": 1.0, + "max_rank": None, + "ranks": [], + } + config["estimation_mode"] = "fpm_regression" + regression = RustForwardPassPerfModel.best_available(config) + assert regression.estimate_forward_pass_detailed(decode()) == { + "latency_ms": None, + "native_latency_ms": None, + "correction_factor": None, + "max_rank": None, + "ranks": [], + } + + +def prediction(profile, systems): + return { + "engine": {**workflow._engine(profile), "systems_paths": [systems]}, + "traffic": { + "source": {"type": "synthetic", "input_tokens": 1, "output_tokens": 3}, + "load": {"type": "concurrency", "concurrency": 1}, + "stop": {"requests": 3}, + }, + } + + +def records(evidence): + return [ + record + for provider in evidence["providers"] + for phase in provider["phases"] + for operation in phase["operations"] + for record in operation["fpm_estimates"] + ] + + +def test_native_prediction_replay_and_source_detail_roundtrip(profile, evidence_systems): + import jsonschema + + config = CorePredictionConfig.model_validate(prediction(profile, evidence_systems)) + # Round-trip the saved public config before lowering it to the real runner. + config = CorePredictionConfig.model_validate_json(config.model_dump_json()) + spec = prediction_to_replay_spec(config) + runner = EngineReplayRunnerFactory().create(0) + try: + plain = runner.run(spec) + report = runner.run(spec, output_requirements=ReplayOutputRequirements(capture_performance_diagnostics=True)) + finally: + runner.close() + assert "fpm_query_evidence" not in plain.metadata + wall_clock_metrics = {"wall_time_ms", "processed_tokens_per_s", "processed_output_tokens_per_s"} + assert {k: v for k, v in report.metrics.items() if k not in wall_clock_metrics} == { + k: v for k, v in plain.metrics.items() if k not in wall_clock_metrics + } + assert report.metrics["power_w"] is None + assert report.metrics["power_coverage"] is None + evidence = report.metadata["fpm_query_evidence"] + assert evidence["aggregation"] == "identical_estimates_with_invocation_counts" + estimates = records(evidence) + assert len(estimates) == 4 # prefill at 1 token; three decode steps read past KV=0,1,2 + assert all(record["count"] == 3 for record in estimates) + assert sum(record["count"] for record in estimates) == 12 + assert all(record["latency_scale"] == 1.0 for record in estimates) + assert {q["resolution"] for r in estimates for rank in r["estimate"]["ranks"] for q in rank["queries"]} == { + "exact_lookup", + "within_curve_interpolation", + } + native = report.metadata["native_report"] + detail = build_prediction_details(native, ["source"]) + schema = json.loads((Path(__file__).resolve().parents[1] / "docs/cli/prediction-details.schema.json").read_text()) + jsonschema.validate(detail, schema) + jsonschema.validate(build_prediction_details(native, ["time"]), schema) + assert detail["sections"]["source"]["status"] == "available" + operations = [op for phase in detail["sections"]["source"]["phases"] for op in phase["operations"]] + assert all(op["source"] == "silicon" and op["fpm_estimates"] for op in operations) + assert all(op["sol"] is None for phase in native["performance_diagnostics"]["phases"] for op in phase["operations"]) + normalized = _normalize_engine_replay_report(json.loads(json.dumps(native)), include_native_report=False) + assert "native_report" not in normalized.metadata + assert normalized.metadata["fpm_query_evidence"] == evidence + + +@pytest.mark.parametrize("supervised", [False, True]) +def test_public_recommendation_retains_requested_evidence(profile, evidence_systems, monkeypatch, supervised): + # Keep orchestration in this process so the analytical-graph guards remain + # active; estimation, candidate scoring and replay use the real native API. + if not supervised: + monkeypatch.setattr("aisimulate.supervision.in_supervised_process", lambda: True) + monkeypatch.setattr("aisimulate.resources.GuardedRunnerFactory.admit_wave", None) + raw = prediction(profile, evidence_systems) + raw["engine"]["mode"] = "aggregated" + raw["engine"]["workers"]["aggregated"]["parallelism"] = {"preset": "default"} + raw["optimization"] = {"constraints": {"max_candidate_gpus": 2}} + raw["optimizer"] = {"algorithm": "random", "max_trials": 1, "parallelism": 1} + config = CoreRecommendationConfig.model_validate(raw) + result = run_recommendation( + config, + stack="engine", + runner_factory=EngineReplayRunnerFactory(), + show_progress=False, + output_requirements=ReplayOutputRequirements(capture_performance_diagnostics=True), + ) + assert result.counts.feasible == 1 + saved = SweepResult.model_validate_json(result.model_dump_json()) + metadata = saved.candidates[0].provenance.runner_metadata + evidence = metadata["fpm_query_evidence"] + assert records(evidence) + assert all(record["count"] == 3 for record in records(evidence)) + assert saved.candidates[0].metrics["power_w"] is None + + +def test_prediction_cli_saves_direct_query_evidence(profile, evidence_systems, tmp_path, capsys): + import aisimulate.main as cli + + config = tmp_path / "prediction.json" + config.write_text(json.dumps(prediction(profile, evidence_systems))) + output = tmp_path / "output" + assert ( + cli.main( + ["predict", "--config", str(config), "--output-dir", str(output), "--detail", "source", "--format", "json"] + ) + == 0 + ) + displayed = json.loads(capsys.readouterr().out) + assert displayed["details"]["sections"]["source"]["status"] == "available" + saved = json.loads((output / "prediction.json").read_text()) + assert records(saved["fpm_query_evidence"]) + assert saved["details"] == displayed["details"] + + +@pytest.fixture +def native_prediction_report(profile, evidence_systems): + spec = prediction_to_replay_spec(CorePredictionConfig.model_validate(prediction(profile, evidence_systems))) + runner = EngineReplayRunnerFactory().create(0) + try: + report = runner.run(spec, output_requirements=ReplayOutputRequirements(capture_performance_diagnostics=True)) + finally: + runner.close() + return report.metadata["native_report"] + + +@pytest.mark.parametrize( + "path,value", + [ + (("estimate",), {}), + (("estimate", "latency_ms"), float("inf")), + (("estimate", "native_latency_ms"), float("nan")), + (("estimate", "correction_factor"), -1), + (("estimate", "max_rank"), True), + (("estimate", "max_rank"), 0.5), + (("estimate", "ranks"), {}), + (("estimate", "ranks", 0), None), + (("estimate", "ranks", 0, "rank"), -1), + (("estimate", "ranks", 0, "rank"), True), + (("estimate", "ranks", 0, "latency_ms"), True), + (("estimate", "ranks", 0, "prefill_ms"), "1"), + (("estimate", "ranks", 0, "decode_ms"), float("inf")), + (("estimate", "ranks", 0, "decode_baseline_ms"), False), + (("estimate", "ranks", 0, "marginal_decode_ms"), -1), + (("estimate", "ranks", 0, "queries"), {}), + (("estimate", "ranks", 0, "queries", 0), []), + (("estimate", "ranks", 0, "queries", 0, "phase"), "generation"), + (("estimate", "ranks", 0, "queries", 0, "model_path"), 1), + (("estimate", "ranks", 0, "queries", 0, "model_path"), ""), + (("estimate", "ranks", 0, "queries", 0, "decode_baseline"), 0), + (("estimate", "ranks", 0, "queries", 0, "query"), {}), + (("estimate", "ranks", 0, "queries", 0, "query", "batch_size"), True), + (("estimate", "ranks", 0, "queries", 0, "query", "total_prefill_tokens"), -1), + (("estimate", "ranks", 0, "queries", 0, "query", "total_kv_read_tokens"), float("inf")), + (("estimate", "ranks", 0, "queries", 0, "resolution"), "nearest"), + (("estimate", "ranks", 0, "queries", 0, "latency_ms"), float("nan")), + (("estimate", "ranks", 0, "queries", 0, "support"), []), + (("estimate", "ranks", 0, "queries", 0, "support"), {}), + (("estimate", "ranks", 0, "queries", 0, "support", 0), None), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "coordinates"), []), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "coordinates", "batch_size"), -1), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "coordinates", "total_prefill_tokens"), "1"), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "coordinates", "total_kv_read_tokens"), float("nan")), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "latency_ms"), "1"), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "weight"), float("nan")), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "weight"), -0.1), + (("estimate", "ranks", 0, "queries", 0, "support", 0, "weight"), 1.1), + ], +) +def test_prediction_details_reject_malformed_fpm_evidence(native_prediction_report, path, value): + record = native_prediction_report["performance_diagnostics"]["phases"][0]["operations"][0]["fpm_estimates"][0] + for key in path[:-1]: + record = record[key] + record[path[-1]] = value + expected_path = "performance_diagnostics.phases[0].operations[0].fpm_estimates[0]" + "".join( + f"[{key}]" if isinstance(key, int) else f".{key}" for key in path + ) + for section in ("source", "time"): + with pytest.raises(ValueError) as error: + build_prediction_details(native_prediction_report, (section,)) + assert str(error.value).startswith(expected_path) + + +def test_prediction_details_reject_missing_and_unknown_fpm_fields(native_prediction_report): + prefix = ("performance_diagnostics", "phases", 0, "operations", 0, "fpm_estimates", 0) + for path in [ + (), + ("estimate",), + ("estimate", "ranks", 0), + ("estimate", "ranks", 0, "queries", 0), + ("estimate", "ranks", 0, "queries", 0, "query"), + ("estimate", "ranks", 0, "queries", 0, "support", 0), + ("estimate", "ranks", 0, "queries", 0, "support", 0, "coordinates"), + ]: + record = native_prediction_report + for key in (*prefix, *path): + record = record[key] + for field in (*record, "unknown"): + invalid = deepcopy(native_prediction_report) + target = invalid + for key in (*prefix, *path): + target = target[key] + if field == "unknown": + target[field] = None + else: + del target[field] + expected_path = "".join( + f"[{key}]" if isinstance(key, int) else f".{key}" for key in (*prefix, *path, field) + ).lstrip(".") + for section in ("source", "time"): + with pytest.raises(ValueError) as error: + build_prediction_details(invalid, (section,)) + assert str(error.value).startswith(f"{expected_path} must be") + + +def test_prediction_details_preserve_valid_nested_fpm_evidence(profile, evidence_systems, native_prediction_report): + import jsonschema + + config = canonical(profile, evidence_systems) + model = RustForwardPassPerfModel.best_available(config) + estimates = [ + model.estimate_forward_pass_detailed(metrics) + for metrics in (prefill(), prefill(6), prefill(6, 2), decode(), {}) + ] + config["estimation_mode"] = "fpm_regression" + estimates.append(RustForwardPassPerfModel.best_available(config).estimate_forward_pass_detailed(decode())) + operation = native_prediction_report["performance_diagnostics"]["phases"][0]["operations"][0] + operation["fpm_estimates"] = [{"estimate": estimate, "count": 1, "latency_scale": 1.0} for estimate in estimates] + original = deepcopy(native_prediction_report) + details = build_prediction_details(native_prediction_report, ("source", "time")) + schema = json.loads((Path(__file__).resolve().parents[1] / "docs/cli/prediction-details.schema.json").read_text()) + jsonschema.validate(details, schema) + for section in (details["sections"]["source"], details["sections"]["time"]["diagnostics"]): + published = section["phases"][0]["operations"][0]["fpm_estimates"] + assert published == operation["fpm_estimates"] + published[0]["estimate"]["ranks"].clear() + assert native_prediction_report == original + + # FPM evidence remains optional for pre-existing diagnostics producers. + del operation["fpm_estimates"] + details = build_prediction_details(native_prediction_report, ("source", "time")) + jsonschema.validate(details, schema) + for section in (details["sections"]["source"], details["sections"]["time"]["diagnostics"]): + assert "fpm_estimates" not in section["phases"][0]["operations"][0] From 3e7d1d2172e35c041934b1a84306f04aab4c2b24 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Tue, 29 Sep 2026 02:38:59 +0800 Subject: [PATCH 20/27] fix: complete the detailed FPM API declaration Signed-off-by: Yiming Liu --- crates/core/src/perfmodel/operators/fpm_forward.rs | 4 ++-- python/aisimulate/src/aisimulate_core/_native.pyi | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/core/src/perfmodel/operators/fpm_forward.rs b/crates/core/src/perfmodel/operators/fpm_forward.rs index 41f9897ce7..b0c1929980 100644 --- a/crates/core/src/perfmodel/operators/fpm_forward.rs +++ b/crates/core/src/perfmodel/operators/fpm_forward.rs @@ -832,8 +832,6 @@ impl FpmForwardOp { } } -/// Exact lookup or raw linear interpolation strictly inside one measured -/// curve. There is deliberately no callback, boundary hold, or SOL input. fn fpm_coordinates(phase: FpmPhase, coords: &[f64]) -> FpmCoordinates { FpmCoordinates { batch_size: coords[0], @@ -879,6 +877,8 @@ fn direct_curve_support( ] } +/// Exact lookup or raw linear interpolation strictly inside one measured +/// curve. There is deliberately no callback, boundary hold, or SOL input. fn direct_curve_value(curve: &BTreeMap, coordinate: f64) -> Option { if !coordinate.is_finite() || coordinate < 0.0 || coordinate > u32::MAX as f64 { return None; diff --git a/python/aisimulate/src/aisimulate_core/_native.pyi b/python/aisimulate/src/aisimulate_core/_native.pyi index 4e5e4358ba..df09cbf3ec 100644 --- a/python/aisimulate/src/aisimulate_core/_native.pyi +++ b/python/aisimulate/src/aisimulate_core/_native.pyi @@ -175,6 +175,7 @@ class RustForwardPassPerfModel: config_json: str, worker_type: str, options_json: str | None = None, allow_regression: bool = False ) -> str: ... def estimate_forward_pass_time_ms(self, fpm_json: str) -> float | None: ... + def estimate_forward_pass_detailed(self, fpm_json: str) -> str: ... def tune_with_fpms(self, iterations_json: str) -> None: ... def static_phase_latency(self, batch_size: int, input_tokens: int, output_tokens: int, prefill: bool) -> float: ... def static_phase_diagnostics(self, batch_size: int, context_length: int, prefix: int, prefill: bool) -> str: ... From 6236b65fee06fcd3aed6aa8e7c307aeed2fd6913 Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 29 Sep 2026 16:40:53 -0700 Subject: [PATCH 21/27] fix: preserve worker data roots in performance metadata Signed-off-by: Simone Chen --- docs/fpm-self-service.md | 6 +++--- python/aisimulate/src/aisimulate/compiler.py | 5 +++-- .../tests/unit/test_systems_path.py | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index cac5549435..5a172eaddf 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # FPM self-service -`aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. No other draft PR needs to be merged first. +`aisimulate onboard` guides onboarding a new model for FPM simulation on your designated hardware platform. It records the model, runtime, target GPU system and allocation, plans one pure tensor-parallel worker, and produces ordinary `predict` and `recommend` configurations that read your collected FPM data. It builds on AISimulate's existing per-worker FPM support and packaged collector. Planning works before the model has an AISimulate model class or measured FPM timings. A valid request records your choices; model metadata and execution readiness, runtime compatibility, and data readiness remain **unchecked**, and accuracy is **not assessed**. This setup does not run target preflight checks, provision GPU resources, or establish measured accuracy. @@ -55,7 +55,7 @@ aisimulate onboard collect-fpm \ The first command saves the request, `support-plan.json`, `commands.json`, `predict/pilot.yaml`, `recommend/pilot.yaml`, and a local `systems/` directory. The second prints the collector command without launching it. Generated command vectors and printed next commands use absolute output paths and preserve spaces or shell punctuation. Use a separate output directory for each request. `--overwrite` can repair missing generated files for the identical saved request, including an interrupted write before `support-plan.json` was created. Repair requires an intact matching `request.yaml`, rejects changed inputs, and preserves existing collected data. -If a newer draft revision changes generated guidance, recreate the plan in a new output directory: repair compares generated files byte for byte and does not migrate earlier draft plans. Guidance changes do not change the saved-request identity checks used by collection. +If a software update changes generated guidance, recreate the plan in a new output directory: repair compares generated files byte for byte and does not migrate older plans. Guidance changes do not change the saved-request identity checks used by collection. The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. @@ -129,7 +129,7 @@ The loop reports each command's exit status and attempts all commands, including Run either the individual recommendation command or the loop against fresh result directories. The loop also handles a one-candidate plan and does not execute entries from `commands.json`. Results remain separate under `recommend-results/pilot` and, when present, `recommend-results/replicas-N`; compare their objective and latency results for the same workload. This plan does not produce a combined ranking or search additional TP sizes or scheduler settings. -The generated configurations select `engine.workers.aggregated.timing.estimation_mode: fpm_interpolation` with `fallback_policy: deny` and set `engine.systems_paths` to a list containing the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. The single-root `engine.systems_path` input is an alias added by onboarding; saved configurations use the canonical `engine.systems_paths` list. Both spellings resolve relative local paths against the working directory when the configuration is loaded, so saving and reloading from another directory preserves the selected roots. The canonical list's `default` entry continues to select packaged data. Empty or whitespace-only roots are rejected by the shared root type in both prediction and recommendation configurations. +The generated configurations select `engine.workers.aggregated.timing.estimation_mode: fpm_interpolation` with `fallback_policy: deny` and set `engine.systems_paths` to a list containing the plan's absolute local systems directory. The same root supplies hardware and collected FPM data. Recommendation preserves it in exported prediction configs. Moving the plan to another machine requires updating absolute paths or regenerating it there. The single-root `engine.systems_path` input is an alias added by onboarding; saved configurations use the canonical `engine.systems_paths` list. Both spellings resolve relative local paths against the working directory when the configuration is loaded, so saving and reloading from another directory preserves the selected roots. The canonical list's `default` entry continues to select packaged data. A worker’s `timing.systems_paths` overrides engine-level roots; performance metadata records those effective roots. Empty or whitespace-only roots are rejected by the shared root type in both prediction and recommendation configurations. These configurations target AISimulate's standalone `predict` and `recommend` commands. Dynamo's replay Planner AIC session adapter does not currently forward custom systems roots; using that downstream path requires a separate adapter update. diff --git a/python/aisimulate/src/aisimulate/compiler.py b/python/aisimulate/src/aisimulate/compiler.py index 8b7856c51b..7089d0e090 100644 --- a/python/aisimulate/src/aisimulate/compiler.py +++ b/python/aisimulate/src/aisimulate/compiler.py @@ -420,8 +420,9 @@ def _worker_performance_model_metadata( if value is not None: config[field] = value config["database_mode"] = worker.timing.database_mode or engine.database_mode - if engine.systems_paths is not None: - config["systems_paths"] = engine.systems_paths + systems_paths = worker.timing.systems_paths or engine.systems_paths + if systems_paths is not None: + config["systems_paths"] = systems_paths if engine.speculation is not None: config["speculation"] = engine.speculation.cost_config() if worker.timing.fpm_parquet_path is not None: diff --git a/python/aisimulate/tests/unit/test_systems_path.py b/python/aisimulate/tests/unit/test_systems_path.py index 31074f3d07..5e6870ab00 100644 --- a/python/aisimulate/tests/unit/test_systems_path.py +++ b/python/aisimulate/tests/unit/test_systems_path.py @@ -281,6 +281,27 @@ def _predict(raw): runner.close() +@pytest.mark.parametrize("mode", ["aggregated", "disaggregated"]) +@pytest.mark.parametrize("engine_roots", [False, True]) +def test_worker_timing_roots_match_runtime_and_metadata(local_profiles, mode, engine_roots): + fast, slow = local_profiles + raw = _local_request(fast, mode) + if not engine_roots: + raw["engine"].pop("systems_path") + for worker in raw["engine"]["workers"].values(): + worker["timing"]["systems_paths"] = [str(slow)] + + config = CorePredictionConfig.model_validate(raw) + saved = config.model_dump(mode="json", exclude_none=True) + deployment = prediction_to_replay_spec(CorePredictionConfig.model_validate(saved)).backend_deployment + for metadata in deployment.performance_model_metadata.values(): + assert metadata["config"]["systems_paths"] == [str(slow)] + report = _predict(saved) + assert report.metrics["completed_requests"] == 1 + assert report.metrics["mean_ttft_ms"] == pytest.approx(200.0 if mode == "aggregated" else 400.0) + assert report.metrics["mean_itl_ms"] == pytest.approx(200.0) + + @pytest.mark.parametrize("root_key", ["systems_path", "systems_paths"]) def test_reloaded_prediction_consumes_relative_systems_roots(local_profiles, root_key, tmp_path, monkeypatch): fast, slow = local_profiles From a7c1ccd1421acf3483ae703036982c5fa63aa941 Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 29 Sep 2026 16:48:47 -0700 Subject: [PATCH 22/27] docs: fix renamed FPM collection guide link Signed-off-by: Simone Chen --- docs/fpm-self-service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index 5a172eaddf..71d5fc9f00 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -59,7 +59,7 @@ If a software update changes generated guidance, recreate the plan in a new outp The search uses one selected TP size. By default it evaluates a single worker, so recommendation is not a broad deployment search. `--max-candidates 2` additionally considers the largest count of identical workers that fits the allocation, when that differs from one worker. Each choice gets an independent recommendation config pinned to that replica count with a one-trial budget. The single worker keeps `recommend/pilot.yaml`; the second choice uses `recommend/replicas-N.yaml`, where `N` is its replica count. The plan reports the actual candidate count and lists both config and result paths. Dense collection uses the `tp` preset; MoE uses `pure_tp`; the selected TP size remains exact. -Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/end-to-end-workflow.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate onboard collect-fpm --execute` command for collection, alongside a read-only collector planning command. +Before execution, prepare the real checkpoint and the pinned runtime using the existing [FPM collection guide](../python/aisimulate/docs/fpm/self-benchmarking-and-onboarding.md). The packaged collector invokes a Generator-resolved Dynamo/vLLM deployment and needs the corresponding GPU resources, deployment configuration, permissions, and model access. Invoking its command locally does not create that environment. `commands.json` publishes the guarded `aisimulate onboard collect-fpm --execute` command for collection, alongside a read-only collector planning command. After those prerequisites are ready, explicitly launch collection from that environment: From d81b2569976d6f8dab81163606332ca6b20d86e0 Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 29 Sep 2026 18:17:26 -0700 Subject: [PATCH 23/27] ci: assign ownership for FPM query evidence tests Signed-off-by: Simone Chen --- .github/codeowners/areas.yaml | 3 +++ CODEOWNERS | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 4c3dc30f9d..95489e4ee0 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -34,6 +34,7 @@ areas: - python/aisimulate/src/aisimulate/capacity.py - tests/test_aic.py - tests/test_fpm_profile_workflow.py + - tests/test_fpm_query_evidence.py - label: sweeper github_team: "@ai-dynamo/aisimulate-sweeper-codeowners" path_globs: @@ -161,6 +162,8 @@ shared: owners: [forward-pass-engine, maintainers] - glob: tests/test_fpm_profile_workflow.py owners: [forward-pass-engine, maintainers] + - glob: tests/test_fpm_query_evidence.py + owners: [forward-pass-engine, replay, maintainers] - glob: python/aisimulate/src/aisimulate/sweeper/ owners: [sweeper, maintainers] - glob: docs/sweeper/ diff --git a/CODEOWNERS b/CODEOWNERS index cf49816355..277f513a2b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -45,6 +45,7 @@ /crates/tests/public-api/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners /crates/core/src/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners /crates/core/tests/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners +/tests/test_fpm_query_evidence.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners /tests/test_fpm_profile_workflow.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners /crates/core/parity_tests/perfmodel/ @ai-dynamo/aisimulate-forward-pass-engine-codeowners /python/aisimulate/src/aisimulate/capacity.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners @@ -180,6 +181,7 @@ /crates/core/src/engine/timing.rs @ai-dynamo/aisimulate-mocker-codeowners @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/access-aisimulate-maintain /python/aisimulate/.gitattributes @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain /python/aisimulate/pyproject.toml @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/aisimulate-sweeper-codeowners @ai-dynamo/aisimulate-replay-codeowners @ai-dynamo/aisimulate-mocker-codeowners @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain +/tests/test_fpm_query_evidence.py @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/aisimulate-replay-codeowners @ai-dynamo/access-aisimulate-maintain /tests/test_resource_scheduler.py @ai-dynamo/aisimulate-replay-codeowners @ai-dynamo/aisimulate-sweeper-codeowners @ai-dynamo/access-aisimulate-maintain /tests/test_state_cache_config.py @ai-dynamo/aisimulate-replay-codeowners @ai-dynamo/aisimulate-mocker-codeowners @ai-dynamo/access-aisimulate-maintain /crates/tests/public-api/Cargo.toml @ai-dynamo/aisimulate-forward-pass-engine-codeowners @ai-dynamo/aisimulate-infra-codeowners @ai-dynamo/access-aisimulate-maintain From 8a4a8041915f95d0812d700244f633b184a05e7b Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 29 Sep 2026 18:21:55 -0700 Subject: [PATCH 24/27] fix: enforce profile decode collection bounds Signed-off-by: Simone Chen --- crates/core/src/perfmodel/fpm/model.rs | 32 +++++++++---------- docs/fpm-self-service.md | 2 +- .../collector/fpm_forward/planner.py | 15 ++++++++- .../aisimulate/src/aisimulate/fpm_profile.py | 1 - .../collector/test_fpm_profile_collection.py | 14 ++++++++ 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/crates/core/src/perfmodel/fpm/model.rs b/crates/core/src/perfmodel/fpm/model.rs index 208a19f383..c37bfcec3a 100644 --- a/crates/core/src/perfmodel/fpm/model.rs +++ b/crates/core/src/perfmodel/fpm/model.rs @@ -448,14 +448,7 @@ impl ForwardPassPerfModel { return Ok(Some(0.0)); }; let native = engine.forward_pass_time_ms(metrics_by_rank)?; - let corrected = native - * if self.is_correction_enabled { - corrections - .store(feature.workload_kind) - .correction_factor_for(&feature.x) - } else { - 1.0 - }; + let corrected = native * self.correction_factor(corrections, &feature); Ok(Some(corrected)) } ForwardPassPerfMode::Regression { @@ -497,14 +490,7 @@ impl ForwardPassPerfModel { }); }; let mut estimate = engine.forward_pass_estimate(metrics_by_rank, true)?; - let factor = - Some(feature) - .filter(|_| self.is_correction_enabled) - .map_or(1.0, |feature| { - corrections - .store(feature.workload_kind) - .correction_factor_for(&feature.x) - }); + let factor = self.correction_factor(corrections, &feature); estimate.correction_factor = Some(factor); estimate.latency_ms = estimate.native_latency_ms.map(|native| native * factor); Ok(estimate) @@ -816,6 +802,20 @@ impl ForwardPassPerfModel { self.provenance.as_ref() } + fn correction_factor( + &self, + corrections: &WorkloadStores, + feature: &IterationFeatures, + ) -> f64 { + if self.is_correction_enabled { + corrections + .store(feature.workload_kind) + .correction_factor_for(&feature.x) + } else { + 1.0 + } + } + fn correction_factors(&self) -> Vec { match &self.mode { ForwardPassPerfMode::Native { corrections, .. } => corrections.correction_factors(), diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index 59681a94f4..88083835cc 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -70,7 +70,7 @@ For a model without an analytical class, create a JSON or YAML FPM profile and p Each `resources` record requires integer `weights_bytes`, `activations_bytes`, `runtime_overhead_bytes`, `comm_overhead_bytes`, positive `kv_bytes_per_token`, `cache_layout: linear`, positive `max_num_tokens` and `max_batch_size`, and a `provenance` explanation. Obtain these from checkpoint tensor/storage metadata and serving-runtime accounting or conservative explicit estimates. A profile is an input declaration; AISimulate does not certify that its values describe your runtime. -All resource bytes are **per rank**, and must bound every rank. Scheduler envelope limits are also per attention-DP rank; they are not the summed batch and token totals used by the worker's FPM timing query. Do not copy TP resource values into DEP/TEP merely because GPU counts match. Include all non-KV storage, including quantization overheads. Overhead fields exclude the separate `kv_cache.capacity.cuda_graph_reserved_bytes` reservation. Requests above the declared scheduler envelope fail explicitly. +All resource bytes are **per rank**, and must bound every rank. Scheduler envelope limits are also per attention-DP rank; they are not the summed batch and token totals used by the worker's FPM timing query. Do not copy TP resource values into DEP/TEP merely because GPU counts match. Include all non-KV storage, including quantization overheads. Overhead fields exclude the separate `kv_cache.capacity.cuda_graph_reserved_bytes` reservation. Requests above the declared scheduler envelope fail explicitly. Collection validates both prefill and explicit decode batch limits against that envelope before launching workers. Automatic KV admission subtracts these non-KV bounds and CUDA graph reservation from the configured fraction of GPU memory, then divides by the declared KV bytes per token. `context_length: max` uses the profile limit; `bytes_per_token: auto` uses the exact deployment's cache geometry. These operations need a hardware specification and profile, but no FPM timing files or model graph. diff --git a/python/aisimulate/collector/fpm_forward/planner.py b/python/aisimulate/collector/fpm_forward/planner.py index 5deadc6ba7..3e3deb1946 100644 --- a/python/aisimulate/collector/fpm_forward/planner.py +++ b/python/aisimulate/collector/fpm_forward/planner.py @@ -687,7 +687,16 @@ def build_collection_plan( ) policies = _backend_policies(options, collector_config, backend=backend) if profile is not None: - _validate_profile_identities(profile, capability, candidate_topologies, policies, model_path, system, backend) + _validate_profile_identities( + profile, + capability, + candidate_topologies, + policies, + model_path, + system, + backend, + max_decode_batch_size=options.max_decode_batch_size, + ) topologies, topology_memory_admission = filter_memory_infeasible_topologies( backend=backend, model_path=model_path, @@ -805,6 +814,8 @@ def _validate_profile_identities( model_path: str, system: str, backend: str, + *, + max_decode_batch_size: int | None = None, ) -> None: """Require a resource declaration for every requested cell, before admission.""" if profile.architecture != capability.architecture: @@ -837,6 +848,8 @@ def _validate_profile_identities( moe_ep_size=topology.moe_ep, cp_size=topology.cp, ) + if max_decode_batch_size is not None: + deployment.resources.validate_envelope(max_num_tokens=1, max_batch_size=max_decode_batch_size) for kv_dtype in capability.dtype.kv_cache_dtypes: for policy in policies: resolved = [ diff --git a/python/aisimulate/src/aisimulate/fpm_profile.py b/python/aisimulate/src/aisimulate/fpm_profile.py index 7d49dd3225..c81b1ebaaa 100644 --- a/python/aisimulate/src/aisimulate/fpm_profile.py +++ b/python/aisimulate/src/aisimulate/fpm_profile.py @@ -3,7 +3,6 @@ """Compatibility exports for the lightweight core fpm_profile types.""" -from aisimulate_core.fpm_profile import _MUTABLE_REFERENCES as _MUTABLE_REFERENCES from aisimulate_core.fpm_profile import FpmDeploymentProfile as FpmDeploymentProfile from aisimulate_core.fpm_profile import FpmModelProfile as FpmModelProfile from aisimulate_core.fpm_profile import FpmResourceProfile as FpmResourceProfile diff --git a/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py b/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py index 09f733eabf..110a1190e7 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_profile_collection.py @@ -354,3 +354,17 @@ def test_multiple_profile_runtime_versions_fail_explicitly(no_models_or_timing_d profile["deployments"].append(other) with pytest.raises(ValueError, match="must identify one runtime version"): _plan(profile) + + +@pytest.mark.parametrize("decode_batch", [1024, 1025]) +def test_decode_collection_respects_profile_scheduler_envelope(no_models_or_timing_data, decode_batch): + profile = _profile() + options = replace( + FPMCollectionOptions.from_args(cli._parser().parse_args(_argv(profile))), + max_decode_batch_size=decode_batch, + ) + if decode_batch > profile["deployments"][0]["resources"]["max_batch_size"]: + with pytest.raises(ValueError, match="max_batch_size"): + _plan(profile, options=options) + else: + assert _plan(profile, options=options).cells From c1645652ee14a5b24a58b8ff9eef132494df076f Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 29 Sep 2026 18:41:27 -0700 Subject: [PATCH 25/27] fix: preserve FPM options and enforce qualified API boundaries Signed-off-by: Simone Chen --- crates/core/src/perfmodel/fpm/config.rs | 3 +++ crates/core/src/perfmodel/fpm/estimator.rs | 16 +++++++++++- crates/core/src/perfmodel/fpm/model.rs | 2 ++ crates/core/src/perfmodel/py.rs | 26 +++++++++++++++++-- docs/fpm-self-service.md | 2 +- .../src/aisimulate_core/_native.pyi | 5 +++- .../src/aisimulate_core/sdk/engine.py | 15 +++++++++++ .../unit/sdk/test_fpm_legacy_migration.py | 9 +++++++ .../tests/unit/sdk/test_fpm_profile.py | 21 +++++++++++++++ .../unit/sdk/test_prefill_graph_canonical.py | 2 ++ 10 files changed, 96 insertions(+), 5 deletions(-) diff --git a/crates/core/src/perfmodel/fpm/config.rs b/crates/core/src/perfmodel/fpm/config.rs index c36165fa3a..1424266150 100644 --- a/crates/core/src/perfmodel/fpm/config.rs +++ b/crates/core/src/perfmodel/fpm/config.rs @@ -243,6 +243,9 @@ impl ForwardPassPerfModelConfig { self.resolve_prefill_graph_profile()?; self.validate()?; let registered = if self.fpm_profile.is_some() { + self.estimator_config + .fpm_interpolation + .validate_profile_options()?; if self.dcp.is_some() { return Err(invalid_config( "fpm_profile does not describe recorded DCP identity", diff --git a/crates/core/src/perfmodel/fpm/estimator.rs b/crates/core/src/perfmodel/fpm/estimator.rs index 461167f13f..210f0abee6 100644 --- a/crates/core/src/perfmodel/fpm/estimator.rs +++ b/crates/core/src/perfmodel/fpm/estimator.rs @@ -61,6 +61,15 @@ pub enum UnrecordedFpmQuantMode { } impl FpmInterpolationConfig { + pub(crate) fn validate_profile_options(&self) -> Result<(), AicError> { + if self.text_only || !self.unrecorded_quant_modes.is_empty() { + return Err(super::config::invalid_config( + "fpm_profile does not support text_only or unrecorded_quant_modes overrides", + )); + } + Ok(()) + } + pub(crate) fn validate_quant_modes( &self, fmha: Option<&str>, @@ -479,7 +488,12 @@ fn explicit_fields( serde_json::Value::Object(fields) => serde_json::Value::Object( fields .iter() - .map(|(key, value)| (key.clone(), explicit_fields(value, &normalized[key]))) + .map(|(key, value)| { + ( + key.clone(), + explicit_fields(value, normalized.get(key).unwrap_or(value)), + ) + }) .collect(), ), _ => normalized.clone(), diff --git a/crates/core/src/perfmodel/fpm/model.rs b/crates/core/src/perfmodel/fpm/model.rs index c37bfcec3a..4e14e1b679 100644 --- a/crates/core/src/perfmodel/fpm/model.rs +++ b/crates/core/src/perfmodel/fpm/model.rs @@ -475,6 +475,7 @@ impl ForwardPassPerfModel { &self, metrics_by_rank: &[ForwardPassMetrics], ) -> Result { + self.require_general_forward_api()?; if let ForwardPassPerfMode::Native { engine, corrections, @@ -716,6 +717,7 @@ impl ForwardPassPerfModel { input_tokens: u32, prefix: u32, ) -> Result { + self.require_general_forward_api()?; self.native_engine() .ok_or_else(|| { AicError::InvalidEngineConfig("static prefill requires a native estimator".into()) diff --git a/crates/core/src/perfmodel/py.rs b/crates/core/src/perfmodel/py.rs index 8a533f94f2..5aea8651e1 100644 --- a/crates/core/src/perfmodel/py.rs +++ b/crates/core/src/perfmodel/py.rs @@ -1796,7 +1796,11 @@ pub(crate) fn compile_engine_to_engine( request.fpm_interpolation.as_deref(), request.fpm_parquet_path.as_deref(), )?; - request.fpm_interpolation = Some(controls.fpm_interpolation.method.as_str().to_owned()); + request.fpm_options = controls.fpm_interpolation.clone(); + request.fpm_interpolation = match controls.fpm_interpolation.method { + crate::FpmInterpolationMethod::Auto => None, + method => Some(method.as_str().to_owned()), + }; request.fpm_parquet_path = controls .fpm_interpolation .fpm_parquet_path @@ -2007,16 +2011,20 @@ impl PyForwardPassPerfModel { /// Share typed FPM option validation with the compilation adapter. #[staticmethod] - #[pyo3(signature = (options_json, fmha_quant_mode=None, comm_quant_mode=None))] + #[pyo3(signature = (options_json, fmha_quant_mode=None, comm_quant_mode=None, has_profile=false))] fn _normalize_fpm_options( options_json: &str, fmha_quant_mode: Option<&str>, comm_quant_mode: Option<&str>, + has_profile: bool, ) -> PyResult { let options: crate::FpmInterpolationConfig = serde_json::from_str(options_json).map_err(|e| { PyValueError::new_err(format!("invalid FPM interpolation options: {e}")) })?; + if has_profile { + options.validate_profile_options().map_err(aic_to_py)?; + } options .validate_quant_modes(fmha_quant_mode, comm_quant_mode) .map_err(aic_to_py)?; @@ -2456,6 +2464,20 @@ mod tests { } } + #[test] + fn legacy_compilation_with_only_correction_controls_keeps_default_method() { + py_init(); + let mut config = fixture_engine_config(); + config.extra.insert( + "estimator_config".into(), + r#"{"correction":{"enabled":false},"fpm_interpolation":{"text_only":false,"unrecorded_quant_modes":[]}}"#.into(), + ); + for forward_model in [None, Some("fpm".to_owned())] { + config.forward_model = forward_model; + compile_engine_to_engine(&config, systems_root().to_str()).unwrap(); + } + } + #[test] fn legacy_engine_rejects_invalid_fpm_paths() { for path in ["", "/missing/fpm.parquet"] { diff --git a/docs/fpm-self-service.md b/docs/fpm-self-service.md index 88083835cc..d3681ea0ef 100644 --- a/docs/fpm-self-service.md +++ b/docs/fpm-self-service.md @@ -70,7 +70,7 @@ For a model without an analytical class, create a JSON or YAML FPM profile and p Each `resources` record requires integer `weights_bytes`, `activations_bytes`, `runtime_overhead_bytes`, `comm_overhead_bytes`, positive `kv_bytes_per_token`, `cache_layout: linear`, positive `max_num_tokens` and `max_batch_size`, and a `provenance` explanation. Obtain these from checkpoint tensor/storage metadata and serving-runtime accounting or conservative explicit estimates. A profile is an input declaration; AISimulate does not certify that its values describe your runtime. -All resource bytes are **per rank**, and must bound every rank. Scheduler envelope limits are also per attention-DP rank; they are not the summed batch and token totals used by the worker's FPM timing query. Do not copy TP resource values into DEP/TEP merely because GPU counts match. Include all non-KV storage, including quantization overheads. Overhead fields exclude the separate `kv_cache.capacity.cuda_graph_reserved_bytes` reservation. Requests above the declared scheduler envelope fail explicitly. Collection validates both prefill and explicit decode batch limits against that envelope before launching workers. +All resource bytes are **per rank**, and must bound every rank. Scheduler envelope limits are also per attention-DP rank; they are not the summed batch and token totals used by the worker's FPM timing query. Do not copy TP resource values into DEP/TEP merely because GPU counts match. Include all non-KV storage, including quantization overheads. Overhead fields exclude the separate `kv_cache.capacity.cuda_graph_reserved_bytes` reservation. Profiles carry explicit quantization identity; `text_only` and `unrecorded_quant_modes` overrides are rejected on this route. Requests above the declared scheduler envelope fail explicitly. Collection validates both prefill and explicit decode batch limits against that envelope before launching workers. Automatic KV admission subtracts these non-KV bounds and CUDA graph reservation from the configured fraction of GPU memory, then divides by the declared KV bytes per token. `context_length: max` uses the profile limit; `bytes_per_token: auto` uses the exact deployment's cache geometry. These operations need a hardware specification and profile, but no FPM timing files or model graph. diff --git a/python/aisimulate/src/aisimulate_core/_native.pyi b/python/aisimulate/src/aisimulate_core/_native.pyi index f3f1ee78bf..6f6bf76997 100644 --- a/python/aisimulate/src/aisimulate_core/_native.pyi +++ b/python/aisimulate/src/aisimulate_core/_native.pyi @@ -174,7 +174,10 @@ class RustForwardPassPerfModel: def normalize_config(config_json: str) -> str: ... @staticmethod def _normalize_fpm_options( - options_json: str, fmha_quant_mode: str | None = None, comm_quant_mode: str | None = None + options_json: str, + fmha_quant_mode: str | None = None, + comm_quant_mode: str | None = None, + has_profile: bool = False, ) -> str: ... @staticmethod def legacy_estimator_config(options_json: str) -> str: ... diff --git a/python/aisimulate/src/aisimulate_core/sdk/engine.py b/python/aisimulate/src/aisimulate_core/sdk/engine.py index c2e338f054..45e9ebc7ac 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/engine.py +++ b/python/aisimulate/src/aisimulate_core/sdk/engine.py @@ -539,6 +539,21 @@ def compile_engine( interpolation = fpm_interpolation deployment = None if profile is not None: + options = json.loads( + aisimulate_core.RustForwardPassPerfModel._normalize_fpm_options( + json.dumps({} if fpm_options is None else fpm_options), + fmha_quant_mode, + comm_quant_mode, + True, + ) + ) + options_path = options["fpm_parquet_path"] + if options_path is not None: + if fpm_parquet_path is not None and fpm_parquet_path != options_path: + raise InvalidEngineConfigurationError("conflicting fpm_parquet_path and fpm_options.fpm_parquet_path") + if forward_model != "fpm": + raise InvalidEngineConfigurationError("fpm_parquet_path requires forward_model='fpm'") + fpm_parquet_path = options_path if dcp_size is not None: raise InvalidEngineConfigurationError("fpm_profile does not describe recorded DCP identity") if nextn or speculation is not None: diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py b/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py index 8394e244b8..8cc8f117af 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_legacy_migration.py @@ -254,3 +254,12 @@ def test_different_parquet_paths_conflict_without_filesystem_resolution(legacy_p def test_migration_rejects_unknown_and_invalid_controls(saved, options, reason): with pytest.raises(ValueError, match=reason): ForwardPassPerfModelConfig.from_legacy_engine_config(_legacy_config(saved), "prefill", options) + + +@pytest.mark.parametrize("options", [{"text_only": False}, {"unrecorded_quant_modes": []}]) +def test_explicit_skipped_defaults_migrate_without_becoming_null(options): + config = ForwardPassPerfModelConfig.from_legacy_engine_config( + _legacy_config({"fpm_interpolation": options}), "prefill" + ) + normalized = RustForwardPassPerfModel.normalize_config(_reload(config)) + assert normalized["estimator_config"]["fpm_interpolation"]["method"] == "sol" diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py index 5522243857..9090bb0abb 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py @@ -1114,3 +1114,24 @@ def test_registered_profile_outer_auto_keeps_op_level_priority_with_inner_direct assert provenance["selected_estimation_mode"] == "op_level" assert provenance["selection_failures"] == [] assert provenance["config"]["estimator_config"]["fpm_interpolation"]["method"] == "direct" + + +@pytest.mark.parametrize("options", [{"text_only": True}, {"unrecorded_quant_modes": ["fmha"]}]) +def test_profile_rejects_unsupported_fpm_options_at_canonical_and_compile_boundaries(profile_dict, options): + config = _request(profile_dict, "direct") + config["estimator_config"] = {"fpm_interpolation": {"method": "direct", **options}} + with pytest.raises(ValueError, match="fpm_profile does not support"): + _normalize(config) + with pytest.raises(ValueError, match="fpm_profile does not support"): + engine.compile_engine( + "test/unknown-decoder", + "test_gpu", + "vllm", + "0.25.1", + tp_size=2, + moe_tp_size=2, + forward_model="fpm", + fpm_profile=profile_dict, + fpm_interpolation="direct", + fpm_options=options, + ) diff --git a/python/aisimulate/tests/unit/sdk/test_prefill_graph_canonical.py b/python/aisimulate/tests/unit/sdk/test_prefill_graph_canonical.py index 15d283b676..da626b0f5e 100644 --- a/python/aisimulate/tests/unit/sdk/test_prefill_graph_canonical.py +++ b/python/aisimulate/tests/unit/sdk/test_prefill_graph_canonical.py @@ -218,6 +218,8 @@ def test_profile_rejects_telemetry_tuning_and_energy_routes_including_empty_inpu for payload in [[], {}, [{"scheduled_requests": []}]]: with pytest.raises((PrefillGraphProfileError, ValueError)): graph_model.estimate_forward_pass_time_ms(payload) + with pytest.raises((PrefillGraphProfileError, ValueError)): + graph_model.estimate_forward_pass_detailed(payload) with pytest.raises(PrefillGraphProfileError, match="only direct"): graph_model.tune_with_fpms([]) for prefill in [False, True]: From b946545a2f12bd052c7282714f9d13ef19c36391 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Wed, 30 Sep 2026 10:34:06 +0800 Subject: [PATCH 26/27] test: include FPM profile in positional config expectations Signed-off-by: Yiming Liu --- .../tests/unit/sdk/test_rust_engine_step.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/python/aisimulate/tests/unit/sdk/test_rust_engine_step.py b/python/aisimulate/tests/unit/sdk/test_rust_engine_step.py index 3bdf44073a..fe0bf74504 100644 --- a/python/aisimulate/tests/unit/sdk/test_rust_engine_step.py +++ b/python/aisimulate/tests/unit/sdk/test_rust_engine_step.py @@ -1014,11 +1014,23 @@ def test_forward_pass_config_preserves_existing_positional_arguments(tmp_path: P "wideep_num_slots": 64, } config = ForwardPassPerfModelConfig(*legacy_fields.values()) - assert vars(config) == {**legacy_fields, "dcp": None, "fpm_fmha_quant_mode": None, "moe_kernel_source": None} + assert vars(config) == { + **legacy_fields, + "fpm_profile": None, + "dcp": None, + "fpm_fmha_quant_mode": None, + "moe_kernel_source": None, + } source = " source_with_spaces " pinned = ForwardPassPerfModelConfig(*legacy_fields.values(), moe_kernel_source=source) - assert vars(pinned) == {**legacy_fields, "dcp": None, "fpm_fmha_quant_mode": None, "moe_kernel_source": source} + assert vars(pinned) == { + **legacy_fields, + "fpm_profile": None, + "dcp": None, + "fpm_fmha_quant_mode": None, + "moe_kernel_source": source, + } assert json.loads(json.dumps(pinned.to_dict()))["moe_kernel_source"] == source From 9d8b695298fc0c06c0ec980fd39867f5f301c01f Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Tue, 29 Sep 2026 21:34:35 -0700 Subject: [PATCH 27/27] docs: clarify FPM engine spec schema migration Signed-off-by: Simone Chen --- docs/core-api.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/core-api.md b/docs/core-api.md index 47dd70a508..6ee9a47382 100644 --- a/docs/core-api.md +++ b/docs/core-api.md @@ -533,9 +533,10 @@ fidelity to the generic Replay cache/scheduler. This API change enables timing consumption, not full hybrid-cache simulation or multimodal prediction from text-only measurements. -The positional engine-spec wire version remains unchanged: the engine identity -is JSON-encoded and the FPM match identity is already variable-length. Legacy -configuration and profiles without DCP remain accepted as unrecorded DCP. +The positional engine-spec wire format is version 25, including the new FPM +interpolation field. Recompile older binary EngineSpecs: schema 24 and other +incompatible versions are rejected before payload decoding. Legacy configuration +and profiles without DCP remain accepted as unrecorded DCP. ### Migrating saved configuration