diff --git a/Cargo.lock b/Cargo.lock index 1808078eb21d..d7c9ea9a832a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.1.0-dev.2" +version = "0.12.0-dev.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aa0422f239fd9b9ca92d4a57fe7069771e158293d266e05a20fde477d15b9af" +checksum = "d4db6a085073ecb1c2569f796ba44742ed4681f8f4060daaed2456ef4ea9b652" dependencies = [ "anyhow", "bincode 1.3.3", @@ -45,6 +45,7 @@ dependencies = [ "derive_builder", "flate2", "indicatif 0.18.4", + "log", "parquet", "pep440_rs", "quick_cache", diff --git a/Cargo.toml b/Cargo.toml index ee9e06eb0945..e5bfe95b4187 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,7 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"] [workspace.dependencies] # Published separately by https://github.com/ai-dynamo/aisimulate. -aisimulate-core = { version = "=0.1.0-dev.2" } +aisimulate-core = { version = "=0.12.0-dev.1" } # Local crates dynamo-runtime = { path = "lib/runtime", version = "1.5.0" } diff --git a/benchmarks/mocker/bench_aic_concurrency.py b/benchmarks/mocker/bench_aic_concurrency.py index 05bb9687dd68..5cc2a8e92279 100644 --- a/benchmarks/mocker/bench_aic_concurrency.py +++ b/benchmarks/mocker/bench_aic_concurrency.py @@ -39,7 +39,7 @@ MODEL = "Qwen/Qwen3-32B" SYSTEM = "h200_sxm" BACKEND = "vllm" -BACKEND_VERSION = "0.19.0" +BACKEND_VERSION = "current" BS, ISL = 16, 2048 THREAD_COUNTS = (1, 2, 4, 8, 12, 16, 24) diff --git a/benchmarks/mocker/bench_aic_rust_callback.py b/benchmarks/mocker/bench_aic_rust_callback.py index b53bd15721a0..158c65859056 100644 --- a/benchmarks/mocker/bench_aic_rust_callback.py +++ b/benchmarks/mocker/bench_aic_rust_callback.py @@ -65,7 +65,7 @@ MODEL = "Qwen/Qwen3-32B" SYSTEM = "h200_sxm" BACKEND = "vllm" -BACKEND_VERSION = "0.19.0" +BACKEND_VERSION = "current" # Workloads chosen to stress the predict hot path differently. ``num_workers`` > 1 # is where the Python GIL serialises predictions, so the Rust win is largest there. diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml index 7b989453d1c0..221802fd115e 100644 --- a/benchmarks/pyproject.toml +++ b/benchmarks/pyproject.toml @@ -39,7 +39,7 @@ classifiers = [ ] dependencies = [ - "aisimulate==0.1.0.dev2; python_version >= '3.11' and python_version < '3.14'", + "aisimulate==0.12.0.dev1; python_version >= '3.11' and python_version < '3.14'", "aiperf==0.12.0", "matplotlib", "networkx", diff --git a/components/src/dynamo/mocker/args.py b/components/src/dynamo/mocker/args.py index ecbc9df49052..6974ccf1b4fa 100644 --- a/components/src/dynamo/mocker/args.py +++ b/components/src/dynamo/mocker/args.py @@ -344,8 +344,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--aic-backend-version", type=str, default=None, - help="AIC backend engine version (e.g., '0.19.0' for vLLM, '0.5.10' for SGLang, " - "'1.3.0rc10' for TRT-LLM). If not set, uses the default version for the backend.", + help="AIC performance-database version: 'current', 'previous', or 'next' " + "when available, or a version assigned to one of those slots. " + "Defaults to the release database's 'current' slot.", ) parser.add_argument( "--aic-tp-size", diff --git a/components/src/dynamo/planner/tests/integration/test_aic_native_estimator.py b/components/src/dynamo/planner/tests/integration/test_aic_native_estimator.py index 0237c52dbbda..d00a2d8cc680 100644 --- a/components/src/dynamo/planner/tests/integration/test_aic_native_estimator.py +++ b/components/src/dynamo/planner/tests/integration/test_aic_native_estimator.py @@ -40,7 +40,7 @@ def _config() -> PlannerConfig: hf_id="Qwen/Qwen3-32B", system="h200_sxm", backend="vllm", - backend_version="0.19.0", + backend_version="current", prefill_pick=pick, decode_pick=pick, ), diff --git a/components/src/dynamo/profiler/tests/integration/test_aic_task_v2_contract.py b/components/src/dynamo/profiler/tests/integration/test_aic_task_v2_contract.py index 60a82c4faf34..8d9032d5d5e4 100644 --- a/components/src/dynamo/profiler/tests/integration/test_aic_task_v2_contract.py +++ b/components/src/dynamo/profiler/tests/integration/test_aic_task_v2_contract.py @@ -31,7 +31,7 @@ def test_rapid_task_v2_build_and_execute_contract() -> None: total_gpus=2, system="h200_sxm", backend="vllm", - backend_version="0.19.0", + backend_version="current", isl=128, osl=8, ttft=100_000.0, diff --git a/components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py b/components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py index 92452d2305de..5d02988751ff 100644 --- a/components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py +++ b/components/src/dynamo/profiler/tests/unit/test_dgd_generation_aic.py @@ -313,21 +313,21 @@ def test_injects_all_required_flags(self): assert out[out.index("--aic-attention-dp-size") + 1] == "8" # trtllm is not a mocker engine_type; leave --engine-type alone. assert "--engine-type" not in out - assert out[out.index("--aic-backend-version") + 1] == "1.3.0rc10" + assert out[out.index("--aic-backend-version") + 1] == "current" def test_matches_engine_type_for_vllm(self): spec = self._spec("vllm") out = _inject_mocker_aic_args([], spec, spec.prefill_pick) assert out[out.index("--engine-type") + 1] == "vllm" assert out[out.index("--aic-backend") + 1] == "vllm" - assert out[out.index("--aic-backend-version") + 1] == "0.14.0" + assert out[out.index("--aic-backend-version") + 1] == "current" def test_matches_engine_type_for_sglang(self): spec = self._spec("sglang") out = _inject_mocker_aic_args([], spec, spec.decode_pick) assert out[out.index("--engine-type") + 1] == "sglang" assert out[out.index("--aic-backend") + 1] == "sglang" - assert out[out.index("--aic-backend-version") + 1] == "0.5.6.post2" + assert out[out.index("--aic-backend-version") + 1] == "current" class TestBuildPlannerConfigEmbedsAicSpec: diff --git a/components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py b/components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py index 9f9722c74df6..775a506b31ee 100644 --- a/components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py +++ b/components/src/dynamo/profiler/tests/unit/test_replay_aic_parity.py @@ -19,8 +19,8 @@ AIC_PARITY_MODEL = "Qwen/Qwen3-32B" AIC_PARITY_SYSTEM = "h200_sxm" AIC_PARITY_VERSIONS = { - "vllm": "0.19.0", - "sglang": "0.5.6.post2", + "vllm": "current", + "sglang": "current", } AIC_PARITY_BACKENDS = [ pytest.param("vllm", id="vllm"), diff --git a/components/src/dynamo/profiler/utils/dgd_generation.py b/components/src/dynamo/profiler/utils/dgd_generation.py index 23e8e548453d..f93621369964 100644 --- a/components/src/dynamo/profiler/utils/dgd_generation.py +++ b/components/src/dynamo/profiler/utils/dgd_generation.py @@ -23,6 +23,9 @@ import numpy as np import yaml +from dynamo._internal.aic import ( + DEFAULT_BACKEND_VERSIONS as _MOCKER_AIC_BACKEND_VERSIONS, +) from dynamo.common.utils.paths import get_workspace_dir from dynamo.planner.config.aic_interpolation_spec import AICInterpolationSpec from dynamo.planner.config.backend_components import MockerComponentName @@ -54,15 +57,9 @@ needs_mocker_aic_perf_model, needs_profile_data, ) -from dynamo.profiler.utils.replay_optimize.constants import AIC_BACKEND_VERSIONS logger = logging.getLogger(__name__) -_MOCKER_AIC_BACKEND_VERSIONS = { - **AIC_BACKEND_VERSIONS, - "trtllm": "1.3.0rc10", -} - def _load_latest_database_version() -> Optional[Callable[..., Optional[str]]]: try: diff --git a/components/src/dynamo/profiler/utils/replay_optimize/constants.py b/components/src/dynamo/profiler/utils/replay_optimize/constants.py index 793b5a68c25d..6d0bd6ac412c 100644 --- a/components/src/dynamo/profiler/utils/replay_optimize/constants.py +++ b/components/src/dynamo/profiler/utils/replay_optimize/constants.py @@ -5,9 +5,10 @@ import os +from dynamo._internal.aic import DEFAULT_BACKEND_VERSIONS + AIC_BACKEND_VERSIONS = { - "vllm": "0.14.0", - "sglang": "0.5.6.post2", + backend: DEFAULT_BACKEND_VERSIONS[backend] for backend in ("vllm", "sglang") } DEFAULT_OVERLAP_SCORE_CREDITS = (1.0,) diff --git a/components/src/dynamo/replay/config.py b/components/src/dynamo/replay/config.py index 32b1d02f4775..8d9778799b3b 100644 --- a/components/src/dynamo/replay/config.py +++ b/components/src/dynamo/replay/config.py @@ -13,6 +13,7 @@ from aisimulate.aic import materialize_aic_num_gpu_blocks +from dynamo._internal.aic import resolve_backend_version from dynamo.mocker import MockEngineArgs from dynamo.mocker.args import ( resolve_planner_profile_data as _resolve_mocker_planner_profile_data, @@ -26,6 +27,13 @@ class PlannerProfileDataResult(Protocol): def resolve_aic_num_gpu_blocks(raw: dict[str, Any]) -> None: """Materialize AIC KV capacity in-place for SDK compatibility.""" + if ( + raw.get("aic_backend") is not None + or raw.get("aic_attention_dp_size") is not None + ): + raw["aic_backend_version"] = resolve_backend_version( + raw.get("aic_backend") or "vllm", raw.get("aic_backend_version") + ) lowered = materialize_aic_num_gpu_blocks(raw) raw.clear() raw.update(lowered) diff --git a/components/src/dynamo/replay/simulation.py b/components/src/dynamo/replay/simulation.py index 54edaecf3ec2..f10136e89d3e 100644 --- a/components/src/dynamo/replay/simulation.py +++ b/components/src/dynamo/replay/simulation.py @@ -17,7 +17,6 @@ from numbers import Real from typing import Any -from aisimulate.aic import materialize_aic_num_gpu_blocks from aisimulate.sweeper.provider import JSONValue, RuntimeHookSpec from aisimulate.sweeper.replay import ( HookCapability, @@ -30,6 +29,7 @@ from dynamo.llm import AicPerfConfig, KvRouterConfig from dynamo.mocker import MockEngineArgs from dynamo.replay.api import run_synthetic_trace_replay, run_trace_replay +from dynamo.replay.config import resolve_aic_num_gpu_blocks _PLANNER_HOOK = HookCapability( provider="dynamo.planner", @@ -260,7 +260,7 @@ def _engine_args(payload: dict[str, JSONValue] | None) -> MockEngineArgs: "aic_model_path", ): lowered.pop(name, None) - lowered = materialize_aic_num_gpu_blocks(lowered) + resolve_aic_num_gpu_blocks(lowered) # Pipeline parallelism is already represented in the public parallel # mapping and used for AIC capacity. MockEngineArgs has no PP field. lowered.pop("aic_pp_size", None) diff --git a/components/src/dynamo/replay/tests/test_simulation.py b/components/src/dynamo/replay/tests/test_simulation.py index a20c93399926..61f7c2ca0778 100644 --- a/components/src/dynamo/replay/tests/test_simulation.py +++ b/components/src/dynamo/replay/tests/test_simulation.py @@ -369,7 +369,7 @@ def test_fixed_timing_keeps_aic_identity_out_of_runtime_args(monkeypatch) -> Non monkeypatch.setattr(simulation, "MockEngineArgs", _FakeEngineArgs) monkeypatch.setattr( simulation, - "materialize_aic_num_gpu_blocks", + "resolve_aic_num_gpu_blocks", lambda payload: payload, ) engine_args = simulation.DynamoReplayRunner._engine_args( diff --git a/components/src/dynamo/replay/tests/test_simulation_integration.py b/components/src/dynamo/replay/tests/test_simulation_integration.py index 756f5a3c6ede..160bdfcdf5f7 100644 --- a/components/src/dynamo/replay/tests/test_simulation_integration.py +++ b/components/src/dynamo/replay/tests/test_simulation_integration.py @@ -235,7 +235,7 @@ def test_sweeper_runs_real_dynamo_replay_in_spawned_workers() -> None: include_router=True, ) - candidates = Sweeper( + result = Sweeper( runner_factory=DynamoReplayRunnerFactory(), providers={ "dynamo.planner": create_planner_provider(), @@ -245,7 +245,9 @@ def test_sweeper_runs_real_dynamo_replay_in_spawned_workers() -> None: show_progress=False, ).run(config) + candidates = result.candidates assert len(candidates) == 2 + assert all(candidate.status == "feasible" for candidate in candidates) assert all( candidate.metrics["output_throughput_tok_s"] > 0.0 for candidate in candidates ) diff --git a/container/deps/requirements.aisimulate.txt b/container/deps/requirements.aisimulate.txt index bf9b45abcf43..7d834b93a593 100644 --- a/container/deps/requirements.aisimulate.txt +++ b/container/deps/requirements.aisimulate.txt @@ -2,4 +2,4 @@ # SPDX-License-Identifier: Apache-2.0 # Published AISimulate wheel staged into planner and framework runtime wheelhouses. -aisimulate==0.1.0.dev2 +aisimulate==0.12.0.dev1 diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index c81a67297659..03372c8ca5a4 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -639,10 +639,22 @@ COPY container/deps/requirements.aisimulate.txt /opt/dynamo/container/deps/requi # wheel consumed by ai-dynamo instead of rebuilding it from vendored source. # Download only this distribution; runtime images own dependency installation # through their requirements files and local wheels. +# AISimulate targets glibc 2.34+, which all consuming runtime images support. +# Select that target explicitly: the manylinux_2_28 builder only stages the +# wheel and must not filter it using the builder's older glibc version. RUN --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/.cache/uv,sharing=shared \ export UV_CACHE_DIR=/root/.cache/uv && \ source ${VIRTUAL_ENV}/bin/activate && \ + case "${TARGETARCH}" in \ + amd64) AISIMULATE_WHEEL_ARCH=x86_64 ;; \ + arm64) AISIMULATE_WHEEL_ARCH=aarch64 ;; \ + *) echo "Unsupported AISimulate target architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ python -m pip download \ + --platform "manylinux_2_34_${AISIMULATE_WHEEL_ARCH}" \ + --python-version "${PYTHON_VERSION}" \ + --implementation cp \ + --abi abi3 \ --only-binary=:all: \ --no-deps \ --dest /opt/dynamo/dist \ diff --git a/docs/fern/pages/reference/components/mocker-cli-reference.mdx b/docs/fern/pages/reference/components/mocker-cli-reference.mdx index c10afd5578cc..b29b9a248884 100644 --- a/docs/fern/pages/reference/components/mocker-cli-reference.mdx +++ b/docs/fern/pages/reference/components/mocker-cli-reference.mdx @@ -167,8 +167,9 @@ For the timing-model design, see [Mocker Engine Architecture](../../developer-gu - AIC backend engine version, for example `0.12.0` for vLLM. When unset, uses the default version for - the backend. + AIC performance-database version. Use `current`, `previous`, or `next` when the slot is available + for the selected system and backend, or specify a version assigned to one of those slots. + When unset, uses the release database's `current` slot. diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 1f91c48d63e0..bbb8ce02a133 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.1.0-dev.2" +version = "0.12.0-dev.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aa0422f239fd9b9ca92d4a57fe7069771e158293d266e05a20fde477d15b9af" +checksum = "d4db6a085073ecb1c2569f796ba44742ed4681f8f4060daaed2456ef4ea9b652" dependencies = [ "anyhow", "bincode 1.3.3", @@ -45,6 +45,7 @@ dependencies = [ "derive_builder", "flate2", "indicatif 0.18.4", + "log", "parquet", "pep440_rs", "quick_cache", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index ed198a024a82..ef4c56d7adf3 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.1.0-dev.2" +version = "0.12.0-dev.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aa0422f239fd9b9ca92d4a57fe7069771e158293d266e05a20fde477d15b9af" +checksum = "d4db6a085073ecb1c2569f796ba44742ed4681f8f4060daaed2456ef4ea9b652" dependencies = [ "anyhow", "bincode 1.3.3", @@ -45,6 +45,7 @@ dependencies = [ "derive_builder", "flate2", "indicatif 0.18.4", + "log", "parquet", "pep440_rs", "pyo3", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index c3e27c53fa81..6836f8dab7a7 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -60,7 +60,7 @@ request-trace-s3 = ["dynamo-llm/request-trace-s3"] # former AIC crate surface. The `python` feature is required for # AicEngineBuilder; this extension supplies the interpreter, so the dependency # must not enable `extension-module`. -aisimulate-core = { version = "=0.1.0-dev.2", optional = true, features = ["python"] } +aisimulate-core = { version = "=0.12.0-dev.1", optional = true, features = ["python"] } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } dynamo-parsers = { version = "8.1.0" } diff --git a/lib/bindings/python/src/dynamo/_internal/aic.py b/lib/bindings/python/src/dynamo/_internal/aic.py index 532ed2868540..acb0ea883e20 100644 --- a/lib/bindings/python/src/dynamo/_internal/aic.py +++ b/lib/bindings/python/src/dynamo/_internal/aic.py @@ -15,11 +15,11 @@ # Dynamo's historical default when conditional acceptance rates are omitted. _DEFAULT_NEXTN_ACCEPT_RATES = [0.85, 0.3, 0.0, 0.0, 0.0] -# Default backend versions match the AIC-core v0.11.0 perf DB. +# Resolve defaults through the queryable slots in the pinned AISimulate perf DB. DEFAULT_BACKEND_VERSIONS = { - "vllm": "0.19.0", - "sglang": "0.5.10", - "trtllm": "1.3.0rc10", + "vllm": "current", + "sglang": "current", + "trtllm": "current", } _KV_CAPACITY_BACKENDS = frozenset(DEFAULT_BACKEND_VERSIONS) DEFAULT_STATIC_STRIDE = 32 @@ -39,7 +39,7 @@ def _validate_kv_capacity_backend(backend_name: str) -> None: def resolve_backend_version(backend_name: str, backend_version: str | None) -> str: - """Return the pinned backend version used for AIC perf lookups.""" + """Preserve explicit versions; otherwise use the release database current slot.""" if backend_version is not None: return backend_version return DEFAULT_BACKEND_VERSIONS.get(backend_name, DEFAULT_BACKEND_VERSIONS["vllm"]) diff --git a/lib/bindings/python/tests/replay/replay_utils.py b/lib/bindings/python/tests/replay/replay_utils.py index d0cebce559b9..6c4ee67ea672 100644 --- a/lib/bindings/python/tests/replay/replay_utils.py +++ b/lib/bindings/python/tests/replay/replay_utils.py @@ -36,17 +36,6 @@ {"timestamp": 3052, "input_length": 2007, "output_length": 354, "hash_ids": [0, 446, 447, 448]} """ -AIC_PARITY_MODEL = "Qwen/Qwen3-32B" -AIC_PARITY_SYSTEM = "h200_sxm" -AIC_PARITY_VERSIONS = { - "vllm": "0.14.0", - "sglang": "0.5.6.post2", -} -AIC_PARITY_BACKENDS = [ - pytest.param("vllm", id="vllm"), - pytest.param("sglang", id="sglang"), -] - def _require_aisimulate_distribution(*, allow_module_level: bool = False) -> None: try: @@ -302,95 +291,6 @@ def _planner_profile_data_npz_path() -> Path: ) -def _aic_replay_args(backend_name: str): - payload = { - "block_size": 512, - "enable_prefix_caching": True, - "enable_chunked_prefill": False, - "max_num_seqs": 16, - "max_num_batched_tokens": 65536, - "num_gpu_blocks": 100000, - "speedup_ratio": 1.0, - "aic_backend": backend_name, - "aic_system": AIC_PARITY_SYSTEM, - "aic_backend_version": AIC_PARITY_VERSIONS[backend_name], - "aic_tp_size": 1, - "aic_model_path": AIC_PARITY_MODEL, - } - if backend_name == "sglang": - payload["engine_type"] = "sglang" - payload["sglang"] = { - "page_size": 512, - "max_prefill_tokens": 65536, - "chunked_prefill_size": 65536, - } - return MockEngineArgs.from_json(json.dumps(payload)) - - -def _aic_disagg_replay_args( - backend_name: str, - *, - tp_size: int, - is_prefill: bool, - max_num_seqs: int, - max_num_batched_tokens: int, -): - payload = { - "block_size": 512, - "enable_prefix_caching": False, - "enable_chunked_prefill": False, - "max_num_seqs": max_num_seqs, - "max_num_batched_tokens": max_num_batched_tokens, - "num_gpu_blocks": 50000, - "speedup_ratio": 1.0, - "aic_backend": backend_name, - "aic_system": AIC_PARITY_SYSTEM, - "aic_backend_version": AIC_PARITY_VERSIONS[backend_name], - "aic_tp_size": tp_size, - "aic_model_path": AIC_PARITY_MODEL, - "is_prefill": is_prefill, - "is_decode": not is_prefill, - } - if backend_name == "sglang": - payload["engine_type"] = "sglang" - payload["sglang"] = { - "page_size": 512, - "max_prefill_tokens": 65536, - "chunked_prefill_size": 65536, - } - return MockEngineArgs.from_json(json.dumps(payload)) - - -def _run_aic_static_point(backend_name: str, isl: int, osl: int, batch_size: int): - aic_core = pytest.importorskip("aiconfigurator_core") - - database = aic_core.sdk.perf_database.get_database( - system=AIC_PARITY_SYSTEM, - backend=backend_name, - version=AIC_PARITY_VERSIONS[backend_name], - ) - backend = aic_core.sdk.backends.factory.get_backend(backend_name) - model = aic_core.sdk.models.get_model( - model_path=AIC_PARITY_MODEL, - model_config=aic_core.sdk.config.ModelConfig(tp_size=1), - backend_name=backend_name, - ) - summary = backend.run_static( - model=model, - database=database, - runtime_config=aic_core.sdk.config.RuntimeConfig( - batch_size=batch_size, - beam_width=1, - isl=isl, - osl=osl, - prefix=0, - ), - mode="static", - stride=32, - ) - return summary.get_summary_df().to_dict(orient="records")[0] - - def _planner_profile_data_dir_path() -> Path: return ( Path(__file__).resolve().parents[5] diff --git a/lib/bindings/python/tests/replay/test_replay_aic_capacity.py b/lib/bindings/python/tests/replay/test_replay_aic_capacity.py index 244ecb935059..12d050d37b75 100644 --- a/lib/bindings/python/tests/replay/test_replay_aic_capacity.py +++ b/lib/bindings/python/tests/replay/test_replay_aic_capacity.py @@ -33,7 +33,7 @@ def _direct_aic_replay_args() -> MockEngineArgs: { "engine_type": "trtllm", "aic_backend": "trtllm", - "aic_backend_version": "1.3.0rc10", + "aic_backend_version": "current", "aic_system": "gb200", "aic_model_path": "meta-llama/Meta-Llama-3.1-8B", "aic_tp_size": 1, @@ -99,7 +99,7 @@ def fake_estimate_num_gpu_blocks(**kwargs): "gpu_memory_utilization": 0.8, "mem_fraction_static": None, "free_gpu_memory_fraction": None, - "backend_version": None, + "backend_version": "current", "pp_size": 1, "moe_tp_size": None, "moe_ep_size": None, diff --git a/lib/bindings/python/tests/replay/test_replay_aic_integration.py b/lib/bindings/python/tests/replay/test_replay_aic_integration.py index 7941c1d5d095..fdb6ee4b3f0b 100644 --- a/lib/bindings/python/tests/replay/test_replay_aic_integration.py +++ b/lib/bindings/python/tests/replay/test_replay_aic_integration.py @@ -1,12 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Offline, network-free coverage of Dynamo replay against AIC 0.11.""" +"""Offline, network-free coverage of Dynamo replay against the pinned AISimulate release.""" from __future__ import annotations import pytest +from dynamo.replay.config import load_engine_args + pytestmark = [ pytest.mark.aiconfigurator, pytest.mark.gpu_0, @@ -17,7 +19,7 @@ AIC_MODEL = "Qwen/Qwen3-32B" AIC_SYSTEM = "h200_sxm" -AIC_BACKEND_VERSION = "0.19.0" +AIC_BACKEND_VERSION = "current" @pytest.fixture(autouse=True) @@ -61,6 +63,23 @@ def test_real_aic_memory_estimates_gpu_blocks() -> None: assert blocks > 0 +def test_default_aic_capacity_uses_queryable_version() -> None: + args = load_engine_args( + { + "aic_backend": "vllm", + "aic_system": AIC_SYSTEM, + "aic_model_path": AIC_MODEL, + "aic_tp_size": 1, + "block_size": 64, + "max_num_batched_tokens": 4096, + "max_num_seqs": 128, + } + ) + assert args is not None + assert args.num_gpu_blocks > 0 + assert args.aic_backend_version == "current" + + def test_aggregated_replay_uses_native_aic_engine() -> None: from aiconfigurator_core.sdk.engine import compile_engine diff --git a/lib/bindings/python/tests/test_aic_capacity.py b/lib/bindings/python/tests/test_aic_capacity.py index c26bf7973ab2..a4ee02f1fcce 100644 --- a/lib/bindings/python/tests/test_aic_capacity.py +++ b/lib/bindings/python/tests/test_aic_capacity.py @@ -264,6 +264,11 @@ def broken_memory_module(name, *args, **kwargs): assert exc_info.value is missing_dependency +@pytest.mark.parametrize("backend", ["vllm", "sglang", "trtllm"]) +def test_default_version_uses_queryable_slot(backend): + assert resolve_backend_version(backend, None) == "current" + + def test_trtllm_version_resolution(): assert resolve_backend_version("trtllm", "0.20.0") == "0.20.0" diff --git a/lib/mocker/src/common/protocols.rs b/lib/mocker/src/common/protocols.rs index bce94d72a27e..9524f05bec9c 100644 --- a/lib/mocker/src/common/protocols.rs +++ b/lib/mocker/src/common/protocols.rs @@ -461,7 +461,9 @@ struct MockEngineArgsSerde { enable_local_indexer: OptionalConfigValue, bootstrap_port: OptionalConfigValue, handoff_session_timeout_ms: OptionalConfigValue, + #[serde(alias = "kv_transfer_bytes_per_token")] kv_bytes_per_token: OptionalConfigValue, + kv_cache_bytes_per_token: OptionalConfigValue, kv_transfer_bandwidth: OptionalConfigValue, kv_transfer_timing_mode: OptionalConfigValue, reasoning: OptionalConfigValue, @@ -591,8 +593,9 @@ pub struct MockEngineArgs { #[builder(default = "None")] pub aic_system: Option, - /// AIC backend engine version (e.g., "0.12.0" for vLLM, "0.5.6.post2" for SGLang). - /// If None, uses the default version for the backend. + /// AIC performance-database slot ("current", "previous", or "next" when available), + /// or a version assigned to one of those slots. + /// If None, uses the release database's "current" slot. #[serde(skip)] #[builder(default = "None")] pub aic_backend_version: Option, @@ -703,11 +706,16 @@ pub struct MockEngineArgs { #[validate(range(min = 1))] pub handoff_session_timeout_ms: u64, - /// KV cache bytes per token, auto-computed from model config by Python CLI. + /// Bytes transferred per token, auto-computed from model config by Python CLI. /// Formula: num_layers * 2 * num_kv_heads * head_dim * dtype_bytes #[builder(default = "None")] pub kv_bytes_per_token: Option, + /// Physical KV-cache bytes occupied by one token, independent of transfer geometry. + #[builder(default = "None")] + #[serde(skip_serializing_if = "Option::is_none")] + pub kv_cache_bytes_per_token: Option, + /// KV cache transfer bandwidth in GB/s for disaggregated serving latency simulation. /// Default: 64.0 (inter-node InfiniBand). Set to 0 to disable KV transfer delay. /// For intra-node NVLink, typical value is ~450. @@ -1047,6 +1055,9 @@ impl TryFrom for MockEngineArgs { if let Some(kv_bytes_per_token) = compat.kv_bytes_per_token.into_nullable() { builder = builder.kv_bytes_per_token(kv_bytes_per_token); } + if let Some(kv_cache_bytes_per_token) = compat.kv_cache_bytes_per_token.into_nullable() { + builder = builder.kv_cache_bytes_per_token(kv_cache_bytes_per_token); + } if let Some(kv_transfer_bandwidth) = compat.kv_transfer_bandwidth.into_nullable() { builder = builder.kv_transfer_bandwidth(kv_transfer_bandwidth); } diff --git a/lib/mocker/src/engine_adapter.rs b/lib/mocker/src/engine_adapter.rs index 98ad4c1785b4..a92f415473f9 100644 --- a/lib/mocker/src/engine_adapter.rs +++ b/lib/mocker/src/engine_adapter.rs @@ -125,7 +125,8 @@ pub(crate) fn engine_components( preemption_mode, emit_kv_events, emit_kv_token_ids, - kv_bytes_per_token: args.kv_bytes_per_token, + kv_transfer_bytes_per_token: args.kv_bytes_per_token, + kv_cache_bytes_per_token: args.kv_cache_bytes_per_token, kv_transfer_bandwidth: args.kv_transfer_bandwidth, kv_transfer_timing_mode, timing_model, @@ -159,6 +160,7 @@ pub(crate) fn aggregated_replay_setup( let config = ReplayEngineConfig { dp_size: components.args.dp_size, tensor_parallel_size: replay_tensor_parallel_size(&components.args)?, + num_gpu_blocks_is_explicit: None, rank: components.rank, prefill: None, decode: None, @@ -181,16 +183,19 @@ pub(crate) fn disaggregated_replay_setup( let prefill_role = ReplayRoleConfig { dp_size: prefill.args.dp_size, tensor_parallel_size: replay_tensor_parallel_size(&prefill.args)?, + num_gpu_blocks_is_explicit: None, rank: prefill.rank, }; let decode_role = ReplayRoleConfig { dp_size: decode.args.dp_size, tensor_parallel_size: replay_tensor_parallel_size(&decode.args)?, + num_gpu_blocks_is_explicit: None, rank: decode.rank, }; let config = ReplayEngineConfig { dp_size: prefill_role.dp_size, tensor_parallel_size: prefill_role.tensor_parallel_size, + num_gpu_blocks_is_explicit: prefill_role.num_gpu_blocks_is_explicit, rank: prefill_role.rank.clone(), prefill: Some(prefill_role), decode: Some(decode_role), @@ -279,18 +284,34 @@ mod tests { #[test] fn vllm_defaults_materialize_once_at_the_shared_boundary() { - let args = MockEngineArgs::builder().build().unwrap(); + let mut args = MockEngineArgs::builder().build().unwrap(); + args.kv_bytes_per_token = Some(4096); + args.kv_cache_bytes_per_token = Some(1024); let components = engine_components(args, true, true).unwrap(); assert_eq!(components.args.block_size, 64); assert_eq!(components.rank.backend, Backend::Vllm); assert_eq!(components.rank.block_size, 64); + assert_eq!(components.rank.kv_transfer_bytes_per_token, Some(4096)); + assert_eq!(components.rank.kv_cache_bytes_per_token, Some(1024)); assert!(components.rank.emit_kv_events); assert!(components.rank.emit_kv_token_ids); assert_eq!(components.rank.timing_model, TimingModelConfig::Polynomial); assert!(components.timing.is_none()); } + #[test] + fn replay_json_keeps_transfer_and_cache_geometry_independent() { + let args: MockEngineArgs = serde_json::from_value(serde_json::json!({ + "kv_transfer_bytes_per_token": 4096, + "kv_cache_bytes_per_token": 1024, + })) + .unwrap(); + let components = engine_components(args, false, false).unwrap(); + assert_eq!(components.rank.kv_transfer_bytes_per_token, Some(4096)); + assert_eq!(components.rank.kv_cache_bytes_per_token, Some(1024)); + } + #[test] fn backend_specific_fields_match_the_engine_contract() { let mut sglang = MockEngineArgs::builder().build().unwrap(); diff --git a/lib/mocker/src/replay/offline/extensions/kv_router/mod.rs b/lib/mocker/src/replay/offline/extensions/kv_router/mod.rs index 5698062371c7..4ea2d4de1258 100644 --- a/lib/mocker/src/replay/offline/extensions/kv_router/mod.rs +++ b/lib/mocker/src/replay/offline/extensions/kv_router/mod.rs @@ -154,6 +154,7 @@ pub(crate) struct WorkerAdmission { uuid: Uuid, worker_idx: usize, overlap_blocks: u32, + best_available_overlap_blocks: u32, isl_blocks: u32, } @@ -169,6 +170,7 @@ pub(crate) struct RouterEffects { struct AdmitOutcome { worker_idx: usize, overlap_blocks: u32, + best_available_overlap_blocks: u32, isl_blocks: u32, } @@ -368,10 +370,12 @@ impl KvRouterPlacement { Placement { request_id: admission.uuid, scheduler_id: admission.worker_idx, + placement_replica_id: None, reported_overlap_tokens: admission.overlap_blocks as usize * self.router.block_size as usize, cache_sample: Some(PlacementCacheSample { overlap_blocks: admission.overlap_blocks, + best_available_overlap_blocks: admission.best_available_overlap_blocks, isl_blocks: admission.isl_blocks, }), } @@ -662,6 +666,7 @@ impl OfflineReplayRouter { uuid, worker_idx: outcome.worker_idx, overlap_blocks: outcome.overlap_blocks, + best_available_overlap_blocks: outcome.best_available_overlap_blocks, isl_blocks: outcome.isl_blocks, }], }) @@ -927,6 +932,19 @@ impl OfflineReplayRouter { .project_worker_loads(request.token_seq.as_deref(), decay_now); let scheduling_request = request.scheduling_request(self.block_size as usize, worker_loads); let eligibility = scheduling_request.eligibility(); + let best_available_overlap_blocks = request + .overlaps + .scores + .iter() + .filter(|(worker, _)| { + self.workers_with_configs + .get(&worker.worker_id) + .is_some_and(|config| eligibility.allows_worker(worker.worker_id, config)) + && worker.dp_rank < self.dp_size + }) + .map(|(_, overlap)| *overlap) + .max() + .unwrap_or(0); let selection = self .selector .select_worker(WorkerSelectionInput::configured( @@ -972,6 +990,7 @@ impl OfflineReplayRouter { Ok(AdmitOutcome { worker_idx, overlap_blocks, + best_available_overlap_blocks, isl_blocks, }) } @@ -993,6 +1012,7 @@ impl OfflineReplayRouter { uuid, worker_idx: outcome.worker_idx, overlap_blocks: outcome.overlap_blocks, + best_available_overlap_blocks: outcome.best_available_overlap_blocks, isl_blocks: outcome.isl_blocks, }); } @@ -1419,6 +1439,7 @@ mod tests { uuid: Uuid::from_u128(1), worker_idx: 1, overlap_blocks: 1, + best_available_overlap_blocks: 1, isl_blocks: 1, }] ); @@ -1804,11 +1825,37 @@ policy_classes: uuid: Uuid::from_u128(1), worker_idx: 3, overlap_blocks: 0, + best_available_overlap_blocks: 0, isl_blocks: 1, }] ); } + #[test] + fn cache_telemetry_excludes_removed_workers() { + let mut router = + OfflineReplayRouter::new(&replay_args(), Some(router_config()), None, 2).unwrap(); + let target = request(1, 7); + let hashes = ReplayRequestHashes::from_tokens(&target.tokens, router.block_size); + router + .on_kv_events(vec![store_event( + 1, + 1, + hashes.local_block_hashes[0], + StorageTier::Device, + )]) + .unwrap(); + router.remove_worker(1).unwrap(); + + let effects = router + .on_request_arrival(&target, Some(hashes), 0.0) + .unwrap(); + assert_eq!(effects.admissions.len(), 1); + assert_eq!(effects.admissions[0].worker_idx, 0); + assert_eq!(effects.admissions[0].overlap_blocks, 0); + assert_eq!(effects.admissions[0].best_available_overlap_blocks, 0); + } + #[test] fn finalized_worker_removal_drops_retained_topology_and_cache_state() { let mut router = @@ -1901,6 +1948,7 @@ policy_classes: uuid: Uuid::from_u128(2), worker_idx: 1, overlap_blocks: 0, + best_available_overlap_blocks: 0, isl_blocks: 1, }] ); diff --git a/pyproject.toml b/pyproject.toml index 31c19289007c..50c0f3d73ccc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ license-files = ["LICENSE"] requires-python = ">=3.10" dependencies = [ "ai-dynamo-runtime==1.5.0", - "aisimulate==0.1.0.dev2; python_version >= '3.11' and python_version < '3.14'", + "aisimulate==0.12.0.dev1; python_version >= '3.11' and python_version < '3.14'", "aiohttp>=3.14.3,<4.0", "transformers>=4.56.0", "kubernetes>=32.0.1,<33.0.0", @@ -240,6 +240,9 @@ xfail_strict = true log_cli_level = "INFO" filterwarnings = [ "error", + # DGDR still uses AISimulate 0.12's supported legacy sweep API until its + # migration to the new Sweeper API is implemented. + 'ignore:`aiconfigurator\.sdk\.sweep\.sweep_(agg|disagg)\(\)` is deprecated:DeprecationWarning', # chardet 6.x is pulled in transitively but requests 2.32.x only supports chardet <6; # charset_normalizer is used at runtime so the warning is benign. "ignore:.*doesn't match a supported version", diff --git a/tests/dependencies/test_aiconfigurator_consistency.py b/tests/dependencies/test_aiconfigurator_consistency.py index 332b3c0e59a7..80c27c7c2358 100644 --- a/tests/dependencies/test_aiconfigurator_consistency.py +++ b/tests/dependencies/test_aiconfigurator_consistency.py @@ -93,7 +93,7 @@ def test_no_manifest_installs_retired_aic_distributions() -> None: assert all(package["name"] != "aiconfigurator-core" for package in packages) assert features["aic-forward-pass"] == ["dep:aisimulate-core"] assert dependencies["aisimulate-core"] == { - "version": "=0.1.0-dev.2", + "version": "=0.12.0-dev.1", "optional": True, "features": ["python"], } diff --git a/tests/frontend/router_aic_image_smoke.py b/tests/frontend/router_aic_image_smoke.py index 8a85f48b6029..4a7769fe68f2 100644 --- a/tests/frontend/router_aic_image_smoke.py +++ b/tests/frontend/router_aic_image_smoke.py @@ -31,7 +31,7 @@ def main() -> None: "--aic-model-path", "Qwen/Qwen3-32B", "--aic-backend-version", - "0.19.0", + "current", ] from aiconfigurator_core.sdk.engine import EngineHandle diff --git a/tests/router/test_router_e2e_with_mockers.py b/tests/router/test_router_e2e_with_mockers.py index a03f083e4623..113a175046ab 100644 --- a/tests/router/test_router_e2e_with_mockers.py +++ b/tests/router/test_router_e2e_with_mockers.py @@ -97,7 +97,7 @@ ROUTER_AIC_CONFIG = { "aic_backend": "vllm", "aic_system": "h200_sxm", - "aic_backend_version": "0.14.0", + "aic_backend_version": "current", "aic_tp_size": 1, "aic_model_path": "Qwen/Qwen3-32B", } diff --git a/tests/wheels/smoke_install.py b/tests/wheels/smoke_install.py index 6f82181f8624..d645cd1c8194 100644 --- a/tests/wheels/smoke_install.py +++ b/tests/wheels/smoke_install.py @@ -406,7 +406,7 @@ def run_aic_core_import_smoke(venv_python: Path) -> None: assert (package_root / "model_configs/Qwen--Qwen3-32B_config.json").is_file() assert (package_root / "systems/h200_sxm.yaml").is_file() parquet_files = list( - (package_root / "systems/data/h200_sxm").glob("*/vllm/0.19.0/*.parquet") + (package_root / "systems/data/h200_sxm").glob("*/vllm/0.24.0/*.parquet") ) assert parquet_files for path in parquet_files: @@ -419,7 +419,7 @@ def run_aic_core_import_smoke(venv_python: Path) -> None: "model_name": "Qwen/Qwen3-32B", "system_name": "h200_sxm", "backend": "vllm", - "backend_version": "0.19.0", + "backend_version": "0.24.0", "kv_block_size": None, "tp_size": 1, "pp_size": 1,