diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cea878fa5..7f5cd9edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -642,10 +642,6 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: "3.12" - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - if: matrix.shard.suite == 'contracts' - with: - node-version: "22" - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: application-test-wheel-${{ matrix.arch }} @@ -669,12 +665,14 @@ jobs: python -m pip install pip-licenses==5.5.5 python -m uv pip install --system "${AISIMULATE_TEST_WHEEL}[dev]" -r scripts/fpm_accuracy/requirements.in python -m pip check + python scripts/check_prediction_numerics.py --fetch-baseline-only python scripts/check_application_test_inventory.py --output application-test-inventory.json python -m pytest -q \ python/aisimulate/tests/unit/cli/test_aisimulate_compatibility.py \ python/aisimulate/tests/cross_package aiconfigurator version - python -m pytest -q -c pytest.ini tests -n 4 + python -m pytest -q -c pytest.ini tests -n 4 \ + --ignore=tests/test_ci_workflow_contracts.py - name: Run unit tests if: matrix.shard.suite == 'unit' run: >- @@ -834,7 +832,9 @@ jobs: -c python/aisimulate/pytest.ini crates/core/parity_tests/perfmodel/test_compile_engine_parity.py - name: Check native prediction numerical sentinels - run: python scripts/check_prediction_numerics.py --output numerical-sentinels.json + run: | + python scripts/check_prediction_numerics.py --fetch-baseline-only + python scripts/check_prediction_numerics.py --output numerical-sentinels.json - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: @@ -880,8 +880,8 @@ jobs: release-smoke/bin/python -c "import importlib.metadata as metadata; import aisimulate, aisimulate_core, aiconfigurator, aiconfigurator_core; - assert aisimulate.__version__ == aisimulate_core.__version__ == '0.12.0'; - assert metadata.version('aisimulate') == '0.12.0'; + assert aisimulate.__version__ == aisimulate_core.__version__ == '0.13.0'; + assert metadata.version('aisimulate') == '0.13.0'; installed = {dist.metadata['Name'].lower() for dist in metadata.distributions()}; forbidden = {'aisimulate-core', 'aiconfigurator', 'aiconfigurator-core'}; assert installed.isdisjoint(forbidden), installed & forbidden" diff --git a/.github/workflows/fast-ci.yml b/.github/workflows/fast-ci.yml index 4c6e33d41..0f3544ca7 100644 --- a/.github/workflows/fast-ci.yml +++ b/.github/workflows/fast-ci.yml @@ -89,6 +89,7 @@ jobs: fi - name: Check active workflow contracts run: | + python scripts/check_prediction_numerics.py --fetch-baseline-only python -m pytest -c /dev/null tests/test_ci_workflow_contracts.py tests/test_ci_qualification.py tests/test_release_fpe.py -q \ -p no:cacheprovider \ --override-ini="addopts=" \ diff --git a/Cargo.lock b/Cargo.lock index b3f9b1447..ff928a172 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,7 +33,7 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "bincode", diff --git a/Cargo.toml b/Cargo.toml index c616fe540..adadfacdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ exclude = ["crates/tests/public-api"] resolver = "3" [workspace.package] -version = "0.12.0" +version = "0.13.0" edition = "2024" description = "Backend-neutral inference simulation and deployment configuration search" authors = ["NVIDIA Inc. "] diff --git a/README.md b/README.md index 38cf152d6..199f50ec2 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ contract. ## AIConfigurator compatibility CLI The `aisimulate` wheel preserves the established `aiconfigurator` command for -workflows that have not yet moved to the unified CLI. AISimulate 0.12.0 keeps +workflows that have not yet moved to the unified CLI. AISimulate 0.13.0 keeps this compatibility surface, while new prediction and search integrations should start with `aisimulate predict` and `aisimulate recommend`. @@ -231,7 +231,7 @@ development, releases, issues, and pull requests; open all new issues and pull requests in this repository. The `aiconfigurator` compatibility command remains available from the -`aisimulate` wheel in 0.12.0. It is targeted for removal in AISimulate 0.13.0, +`aisimulate` wheel through 0.13.0. It is targeted for removal in AISimulate 0.14.0, after every remaining AIC workflow has a verified replacement in the unified `aisimulate` CLI. Until then, use the compatibility command for the workflows identified in the migration guide. diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index aac04c6c1..fcc873e96 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "aisimulate-core" description = "Engine-neutral inference simulation, deterministic replay, and performance modeling" -version = "0.12.0" +version = "0.13.0" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/crates/core/perfmodel/README.md b/crates/core/perfmodel/README.md index 927e52860..5857e485d 100644 --- a/crates/core/perfmodel/README.md +++ b/crates/core/perfmodel/README.md @@ -43,7 +43,7 @@ crate's `embed-python` feature, which enables PyO3's `auto-initialize` support: ```toml [dependencies] -aisimulate-core = { version = "0.12.0", features = ["embed-python"] } +aisimulate-core = { version = "0.13.0", features = ["embed-python"] } ``` Applications that embed Python in an existing host may initialize the diff --git a/crates/core/src/perfmodel/engine/runtime.rs b/crates/core/src/perfmodel/engine/runtime.rs index 91a1423f5..b6cd090b1 100644 --- a/crates/core/src/perfmodel/engine/runtime.rs +++ b/crates/core/src/perfmodel/engine/runtime.rs @@ -1958,7 +1958,18 @@ mod tests { /// Build an `Engine` from the hand-built op lists over the real fixture DB. fn build_engine(nextn: Option) -> Engine { - let db = PerfDatabase::load(&systems_root(), "b200_sxm", "vllm", "0.24.0").unwrap(); + // Match SILICON's shared-layer default and honor the declared reuse + // of graph-timed FP8-block GEMM measurements from vLLM 0.25.0. + let db = PerfDatabase::load_resolved( + &systems_root(), + "b200_sxm", + "vllm", + "0.24.0", + true, + false, + false, + ) + .unwrap(); let spec = EngineSpec::new( fixture_engine_config(nextn), context_ops(), @@ -2335,9 +2346,17 @@ mod tests { for mode in [DatabaseMode::Silicon, DatabaseMode::Sol] { let db = Arc::new( - PerfDatabase::load(&systems_root(), "b200_sxm", "vllm", "0.24.0") - .unwrap() - .with_mode(mode, TransferPolicy::default()), + PerfDatabase::load_resolved( + &systems_root(), + "b200_sxm", + "vllm", + "0.24.0", + mode == DatabaseMode::Silicon, + false, + false, + ) + .unwrap() + .with_mode(mode, TransferPolicy::default()), ); let mut config = fixture_engine_config(Some(3)); config.database_mode = mode; diff --git a/crates/core/src/perfmodel/fpm/tests.rs b/crates/core/src/perfmodel/fpm/tests.rs index 1e51ebb24..78c22682c 100644 --- a/crates/core/src/perfmodel/fpm/tests.rs +++ b/crates/core/src/perfmodel/fpm/tests.rs @@ -135,10 +135,7 @@ fn fixture_engine_config() -> EngineConfig { /// public `from_native` constructors compile via Python; `from_engine` lets /// the pure-Rust tests build the native variant directly. fn native_model(options: ForwardPassPerfOptions) -> ForwardPassPerfModel { - let db = PerfDatabase::load(&systems_root(), "b200_sxm", "vllm", "0.24.0").unwrap(); - let spec = EngineSpec::new(fixture_engine_config(), context_ops(), generation_ops()); - let engine = Engine::build(spec, Arc::new(db)).unwrap(); - ForwardPassPerfModel::from_engine(Arc::new(engine), options) + ForwardPassPerfModel::from_engine(fixture_engine(), options) } fn regression_model( @@ -149,7 +146,18 @@ fn regression_model( } fn fixture_engine() -> Arc { - let db = PerfDatabase::load(&systems_root(), "b200_sxm", "vllm", "0.24.0").unwrap(); + // Match SILICON's shared-layer default: 0.24.0 declares reuse of the + // corrected, graph-timed FP8-block GEMM measurements from 0.25.0. + let db = PerfDatabase::load_resolved( + &systems_root(), + "b200_sxm", + "vllm", + "0.24.0", + true, + false, + false, + ) + .unwrap(); let spec = EngineSpec::new(fixture_engine_config(), context_ops(), generation_ops()); Arc::new(Engine::build(spec, Arc::new(db)).unwrap()) } diff --git a/crates/tests/public-api/Cargo.lock b/crates/tests/public-api/Cargo.lock index 9e14ecfb9..8b7cd4b46 100644 --- a/crates/tests/public-api/Cargo.lock +++ b/crates/tests/public-api/Cargo.lock @@ -24,7 +24,7 @@ dependencies = [ [[package]] name = "aisimulate-core" -version = "0.12.0" +version = "0.13.0" dependencies = [ "anyhow", "bincode", diff --git a/docs/artifact-contract.md b/docs/artifact-contract.md index 45a99f47d..b7f618322 100644 --- a/docs/artifact-contract.md +++ b/docs/artifact-contract.md @@ -1,6 +1,6 @@ # AISimulate artifact contract -AISimulate 0.12.0 has one product version and exactly two release artifacts: +AISimulate 0.13.0 has one product version and exactly two release artifacts: | Artifact | Build manifest | Public purpose | | --- | --- | --- | @@ -17,7 +17,7 @@ live inside the `aisimulate` wheel and therefore do not add artifacts. builds and validates the output directory afterward. A release build fails if an additional wheel, source distribution, or crate appears. -Both artifacts use version `0.12.0`. The wheel builds its native extension from +Both artifacts use version `0.13.0`. The wheel builds its native extension from the same Rust source as the published crate; it does not install a second core distribution. @@ -36,8 +36,8 @@ fails CI if either packaging copy differs byte-for-byte from its root original, and the release-artifact validator checks the bytes installed in the wheel. Nightly builds stamp a dev suffix with `scripts/apply_dev_version.py` before -building: the wheel becomes `0.12.0.devYYYYMMDD` (PEP 440) and the crate -`0.12.0-dev.YYYYMMDD` (SemVer — cargo rejects the PEP 440 spelling, and the +building: the wheel becomes `0.13.0.devYYYYMMDD` (PEP 440) and the crate +`0.13.0-dev.YYYYMMDD` (SemVer — cargo rejects the PEP 440 spelling, and the dotted date is a numeric identifier so pre-release versions order numerically). The wheel form follows the ai-dynamo/dynamo nightly convention. The release script accepts only this suffix pair and still diff --git a/docs/ci.md b/docs/ci.md index 30cea9a31..2c08deb22 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -261,7 +261,10 @@ marked it N/A. [Native numerical checks](../scripts/check_prediction_numerics.py) exercise eight frozen queries: dense Qwen3-32B and MoE MiniMax-M2.5, prefill/decode, and short/long sequences. The [manifest](../.github/prediction-numerical-sentinels.json) -records a full baseline commit that must resolve in the checkout. Tolerances +records a full baseline commit that must resolve in the checkout. CI runs +`scripts/check_prediction_numerics.py --fetch-baseline-only` before contract +tests and native qualification so historical PR commits remain available after +squash merges. This fetch preserves the recorded SHA and expected values. Tolerances are 2% relative and 0.0001 ms absolute. Missing, duplicate, failed, nonfinite, nonpositive, or out-of-tolerance results fail. Intentional modeling changes need explained before/after evidence; do not refresh goldens merely to pass CI. @@ -349,7 +352,7 @@ Main branch nightly CI builds the approved release surface: one `aisimulate` whe architecture and one `aisimulate-core` Rust source crate. A changes guard compares `main` with the last successful scheduled nightly. The build stamps a dev version using the original UTC run-creation date followed by its zero-padded ten-digit workflow run number, for example -`0.12.0.dev202609170000001234`. Scheduled and manual runs have distinct versions; +`0.13.0.dev202609170000001234`. Scheduled and manual runs have distinct versions; retries retain the same version, and later dates sort after earlier dates. Builds use pinned tooling and record checksums and provenance. diff --git a/docs/cli/migrate-from-aiconfigurator.md b/docs/cli/migrate-from-aiconfigurator.md index 0fb962629..c2b966795 100644 --- a/docs/cli/migrate-from-aiconfigurator.md +++ b/docs/cli/migrate-from-aiconfigurator.md @@ -1471,7 +1471,7 @@ aggregate-only SLA semantics, described in its feature guide. ### 6.2 Repository and release transition AISimulate is the home for ongoing development, issues, and releases. The standalone -AIConfigurator repository is scheduled to archive after its final 0.12.0 release. AISimulate 0.12.0 -keeps the AIC compatibility command; removal is targeted for 0.13.0 after all remaining workflows +AIConfigurator repository is scheduled to archive after its final 0.12.0 release. AISimulate 0.13.0 +keeps the AIC compatibility command; removal is targeted for 0.14.0 after all remaining workflows have verified unified-CLI replacements. See the [release transition policy](../../README.md#aiconfigurator-repository-transition) and [repository history](../repository-history.md). diff --git a/docs/core-api.md b/docs/core-api.md index 9ff4938c7..26c304e20 100644 --- a/docs/core-api.md +++ b/docs/core-api.md @@ -12,7 +12,7 @@ and unified native PyO3 extension. It does not depend on another core distribution or on Dynamo. The crate owns the compiled engine, forward-pass model, Replay runtime, KV-cache request/response types, and the embedded Rust-to-Python construction path. The legacy `aiconfigurator_core` Python -namespace remains available during the AIC 0.12.0 compatibility window. +namespace remains available in AISimulate 0.13.0 for compatibility. ## Stable Python facade diff --git a/docs/data/pr244/README.md b/docs/data/pr244/README.md index 66a8a31f7..48afe0460 100644 --- a/docs/data/pr244/README.md +++ b/docs/data/pr244/README.md @@ -32,6 +32,13 @@ and row count match. Rejection leaves the existing manifest unchanged. All 85 tables have zero reported null cells, duplicate physical keys, and nonfinite or nonpositive latency values. +The Collector backend-facts registry records 221 new vLLM 0.25.0 slices +from these tables (49 each for B300, GB200, and GB300; 37 each for H100 and +H200). Six direct vLLM 0.24.0 FP8-block GEMM facts are removed for the +corrected tables described below. Their replacements remain identified as +0.25.0 measurements. Existing backend/kernel mappings are unchanged; the +registry check still compares every fact against the collected labels. + Cases and rows are different units: a single MoE/GDN case can emit many rows. Zero unattempted cases means every **retained** case reached a final checkpoint outcome. It does not mean every possible shape succeeded or that capability diff --git a/docs/installation.md b/docs/installation.md index 7c8b5d5c7..35841a3a4 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -106,7 +106,7 @@ to the intended environment. ## Use an internal nightly The [nightly workflow](../.github/workflows/nightly-ci.yml) produces a wheel -version such as `0.12.0.devYYYYMMDD`, then stages artifacts to access-controlled +version such as `0.13.0.devYYYYMMDD`, then stages artifacts to access-controlled Artifactory through the protected release environment. The run subsequently checks the downloaded wheel and qualifies its FPE support matrix. Use a successful completed nightly: a successful build, staging step, or dev suffix diff --git a/docs/repository-history.md b/docs/repository-history.md index 40ad5c7a2..16d797322 100644 --- a/docs/repository-history.md +++ b/docs/repository-history.md @@ -12,20 +12,20 @@ migration is tracked by ### API mapping -| AIC 0.11 surface | AISimulate 0.12 surface | Compatibility | +| AIC 0.11 surface | AISimulate 0.13 surface | Compatibility | | --- | --- | --- | | Python distribution `aiconfigurator` | `aisimulate` | The `aiconfigurator` command and import namespace ship inside `aisimulate` during the compatibility window | | CLI `aiconfigurator ...` | `aisimulate predict` / `aisimulate recommend` for new simulation workflows; `aiconfigurator ...` for compatibility-only workflows | See the [AIC migration guide](cli/migrate-from-aiconfigurator.md); this is not a flag-compatible rename | | Python distribution `aiconfigurator-core` | included in `aisimulate` | No separate core distribution is installed | -| `aiconfigurator_core` | `aiconfigurator_core` from the `aisimulate` wheel | Existing import remains available in 0.12.0 | -| `aiconfigurator_core.sdk` | `aisimulate_core.sdk` facade in the same wheel | Both import paths remain available in 0.12.0 | +| `aiconfigurator_core` | `aiconfigurator_core` from the `aisimulate` wheel | Existing import remains available in 0.13.0 | +| `aiconfigurator_core.sdk` | `aisimulate_core.sdk` facade in the same wheel | Both import paths remain available in 0.13.0 | | Rust package/import `aiconfigurator-core` / `aiconfigurator_core` | `aisimulate-core` / `aisimulate_core` | Cargo consumers may temporarily alias the new package under the old dependency key | Temporary Cargo alias: ```toml [dependencies] -aiconfigurator-core = { package = "aisimulate-core", version = "0.12", features = ["python"] } +aiconfigurator-core = { package = "aisimulate-core", version = "0.13", features = ["python"] } ``` The former AIC crate-root types and builders remain available through this @@ -175,8 +175,8 @@ will publish its final 0.12.0 `aiconfigurator` and `aiconfigurator-core` artifacts and then be archived; ongoing development, releases, issues, and pull requests move to AISimulate. -The compatibility command remains in the AISimulate 0.12.0 wheel and is -targeted for removal in AISimulate 0.13.0. Removal is gated on verified unified +The compatibility command remains in the AISimulate 0.13.0 wheel and is +targeted for removal in AISimulate 0.14.0. Removal is gated on verified unified CLI replacements for every remaining workflow in the migration guide. For features already implemented by the standalone Sweeper, see diff --git a/python/aisimulate/README.md b/python/aisimulate/README.md index d1d4d092b..457375a76 100644 --- a/python/aisimulate/README.md +++ b/python/aisimulate/README.md @@ -6,10 +6,11 @@ SPDX-License-Identifier: Apache-2.0 # AISimulate application (AIConfigurator compatibility source) > This directory contains the complete AIConfigurator application migrated to -> the standalone AISimulate repository. It builds the `aisimulate` 0.12.0 +> the standalone AISimulate repository. It builds the `aisimulate` 0.13.0 > wheel, not a separate `aiconfigurator` wheel. The legacy import namespace and > `aiconfigurator` executable remain compatibility surfaces alongside the public -> `aisimulate` prediction CLI. The original AIC +> `aisimulate` prediction CLI. Command removal is targeted for AISimulate 0.14.0, +> after every remaining workflow has a verified unified-CLI replacement. The original AIC > documentation below is retained so existing workflows remain discoverable > during the CLI parity and deprecation window. diff --git a/python/aisimulate/collector/op_backend_facts.yaml b/python/aisimulate/collector/op_backend_facts.yaml index 36d80bca7..19d8e4293 100644 --- a/python/aisimulate/collector/op_backend_facts.yaml +++ b/python/aisimulate/collector/op_backend_facts.yaml @@ -142,6 +142,16 @@ ops: - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill", "vllm_triton_attn"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2"], kernel_sources: ["vllm_flash_attn_fa2"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3"], kernel_sources: ["vllm_flash_attn_fa3"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3"], kernel_sources: ["vllm_flash_attn_fa3"]} - op_file: context_mla_perf family: mla axes: [op_name, kv_cache_dtype, mla_dtype] @@ -648,6 +658,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - op_file: dsv4_csa_generation_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -677,6 +692,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - op_file: dsv4_csa_topk_calib_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -698,6 +718,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - op_file: dsv4_hca_context_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -727,6 +752,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - op_file: dsv4_hca_generation_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -756,6 +786,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - op_file: dsv4_megamoe_module_perf family: moe axes: [op_name, phase, moe_dtype, kernel_dtype] @@ -783,6 +818,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_paged_mqa_logits_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["deepgemm"], kernel_sources: ["vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits"]} - op_file: encoder_attention_perf family: encoder_attention axes: [op_name, attn_dtype] @@ -812,6 +852,11 @@ ops: - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa2"], kernel_sources: ["vllm_vit_flash_attn_fa2"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa2"], kernel_sources: ["vllm_vit_flash_attn_fa2"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa4"], kernel_sources: ["vllm_vit_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa4"], kernel_sources: ["vllm_vit_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa4"], kernel_sources: ["vllm_vit_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa4"], kernel_sources: ["vllm_vit_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa3"], kernel_sources: ["vllm_vit_flash_attn_fa3"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "encoder_attention", attn_dtype: "bfloat16", backends: ["fa3"], kernel_sources: ["vllm_vit_flash_attn_fa3"]} - op_file: gdn_perf family: linear_attention axes: [op_name, phase] @@ -884,6 +929,16 @@ ops: - {framework: "vllm", version: "0.27.1", system: "gb200", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} - op_file: gemm_perf family: gemm axes: [op_name, gemm_dtype] @@ -980,26 +1035,20 @@ ops: - {framework: "vllm", version: "0.26.0", system: "b60", op_name: "gemm", gemm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["vllm_default"]} - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} - {framework: "vllm", version: "0.24.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.24.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.24.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} - {framework: "vllm", version: "0.24.0", system: "gb200", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "gb200", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.24.0", system: "gb200", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.24.0", system: "gb200", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} - {framework: "vllm", version: "0.24.0", system: "gb300", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "gb300", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.24.0", system: "gb300", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.24.0", system: "gb300", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm", "flashinfer"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel", "FlashInferFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm", "flashinfer"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel", "FlashInferFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["triton"], kernel_sources: ["TritonFp8BlockScaledMMKernel"]} @@ -1023,6 +1072,24 @@ ops: - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "gemm", gemm_dtype: "nvfp4", backends: ["flashinfer_cutedsl"], kernel_sources: ["FlashInferCuteDslNvFp4LinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm", "flashinfer"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel", "FlashInferFp8BlockScaledMMKernel"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm", "flashinfer"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel", "FlashInferFp8BlockScaledMMKernel"]} - op_file: generation_attention_perf family: attention axes: [op_name, attn_dtype, kv_cache_dtype] @@ -1086,6 +1153,16 @@ ops: - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_triton_attn"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2"], kernel_sources: ["vllm_flash_attn_fa2"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3"], kernel_sources: ["vllm_flash_attn_fa3"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3"], kernel_sources: ["vllm_flash_attn_fa3"]} - op_file: generation_mla_perf family: mla axes: [op_name, kv_cache_dtype, mla_dtype] @@ -1282,6 +1359,16 @@ ops: - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "post", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "post", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "post", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "post", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "post", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "post", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "pre", architecture: "DeepseekV4ForCausalLM", backends: ["tilelang"], kernel_sources: ["vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang"]} - op_file: mla_bmm_perf family: mla_bmm axes: [op_name, bmm_dtype] @@ -1374,6 +1461,16 @@ ops: - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["vllm_torch_bmm"]} - op_file: mla_context_module_perf family: mla axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1493,6 +1590,41 @@ ops: - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_context_module", architecture: "KimiK3ForConditionalGeneration", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "bfloat16", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_context_module", architecture: "KimiK3ForConditionalGeneration", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "fp8_block", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_context_module", architecture: "KimiK3ForConditionalGeneration", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "nvfp4", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "bfloat16", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "fp8_block", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "nvfp4", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "bfloat16", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "fp8_block", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "nvfp4", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flash_attn", "flashinfer"], kernel_sources: ["FLASHINFER_MLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "bfloat16", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "fp8_block", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "fp8", gemm_type: "nvfp4", backends: ["flashinfer", "trtllm_ragged"], kernel_sources: ["FLASHINFER_MLA", "TRTLLM_RAGGED"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN", "FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN", "FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashmla"], kernel_sources: ["FLASHMLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashmla"], kernel_sources: ["FLASHMLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN", "FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN", "FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn", "flashmla"], kernel_sources: ["FLASHMLA", "FLASH_ATTN"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_context_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn", "flashmla"], kernel_sources: ["FLASHMLA", "FLASH_ATTN"]} - op_file: mla_generation_module_perf family: mla axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1591,6 +1723,32 @@ ops: - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_generation_module", architecture: "KimiK3ForConditionalGeneration", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_generation_module", architecture: "KimiK3ForConditionalGeneration", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} - {framework: "vllm", version: "0.27.0", system: "gb300", op_name: "mla_generation_module", architecture: "KimiK3ForConditionalGeneration", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "mla_generation_module", architecture: "DeepseekV3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA"]} - op_file: moe_a2a_perf family: comm axes: [op_name, phase] @@ -1825,6 +1983,37 @@ ops: - {framework: "vllm", version: "0.27.1", system: "gb200", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic"]} - {framework: "vllm", version: "0.27.1", system: "gb300", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_cutlass", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_cutlass_flashinferexperts", "vllm_fp8moe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["marlin", "triton_fused_moe"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_cutlass", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_cutlass_flashinferexperts", "vllm_fp8moe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["marlin", "triton_fused_moe"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic"]} - op_file: msa_context_module_perf family: msa axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1905,6 +2094,32 @@ ops: - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["vllm_kernel"], kernel_sources: ["MiniMaxM3SparseMSAImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} + - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "msa_context_module", architecture: "MiniMaxM3ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["triton"], kernel_sources: ["MiniMaxM3SparseTritonImpl"]} - op_file: msa_generation_module_perf family: msa axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] diff --git a/python/aisimulate/pyproject.toml b/python/aisimulate/pyproject.toml index fad9c93b7..0dd2dada9 100644 --- a/python/aisimulate/pyproject.toml +++ b/python/aisimulate/pyproject.toml @@ -3,7 +3,7 @@ [project] name = "aisimulate" -version = "0.12.0" +version = "0.13.0" authors = [ { name = "NVIDIA Inc.", email = "sw-dl-dynamo@nvidia.com" }, ] diff --git a/python/aisimulate/src/aiconfigurator/deprecation.py b/python/aisimulate/src/aiconfigurator/deprecation.py index aa37dc1f4..0548293db 100644 --- a/python/aisimulate/src/aiconfigurator/deprecation.py +++ b/python/aisimulate/src/aiconfigurator/deprecation.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -_MIGRATION_GUIDE = "https://github.com/ai-dynamo/aiconfigurator/blob/main/docs/aisimulate_migration.md" +_MIGRATION_GUIDE = "https://github.com/ai-dynamo/aisimulate/blob/main/docs/cli/migrate-from-aiconfigurator.md" _warned_entry_points: set[str] = set() _P = ParamSpec("_P") @@ -21,7 +21,7 @@ def warn_legacy_cli(mode: str | None) -> None: - """Warn once when the CLI is run from the legacy AIC distribution.""" + """Warn once when the legacy compatibility CLI is run.""" command = "aiconfigurator cli" + (f" {mode}" if mode else "") key = f"cli:{mode or ''}" @@ -29,10 +29,11 @@ def warn_legacy_cli(mode: str | None) -> None: return _warned_entry_points.add(key) message = ( - f"`{command}` is running from the deprecated AIConfigurator " - "distribution, which is planned for removal in AIConfigurator 0.13.0. " - f"Install `aisimulate==0.12.0` and keep running `{command} ...`; " - "AISimulate preserves the established command name and arguments. " + f"`{command}` is a deprecated compatibility command shipped by AISimulate 0.13.0. " + "Command removal is targeted for AISimulate 0.14.0 after every remaining " + "workflow has a verified unified-CLI replacement. " + f"Until then, keep running `{command} ...`; AISimulate preserves the " + "established command name and arguments. " f"Migration guide: {_MIGRATION_GUIDE}" ) warnings.warn(message, DeprecationWarning, stacklevel=4) @@ -51,10 +52,9 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: if entry_point not in _warned_entry_points: _warned_entry_points.add(entry_point) warnings.warn( - f"`{entry_point}()` is deprecated and will be removed in " - "AIConfigurator 0.13.0. Migrate to " - "`aisimulate.sweeper.Sweeper(...).run(config)`. Installing " - "`aisimulate==0.12.0` temporarily retains the legacy " + f"`{entry_point}()` is deprecated. Migrate to " + "`aisimulate.sweeper.Sweeper(...).run(config)`. " + "AISimulate 0.13.0 temporarily retains the legacy " f"`aiconfigurator` import namespace. Migration guide: {_MIGRATION_GUIDE}", DeprecationWarning, stacklevel=2, diff --git a/python/aisimulate/src/aisimulate_core/__init__.py b/python/aisimulate/src/aisimulate_core/__init__.py index 14d937db8..a19e30951 100644 --- a/python/aisimulate/src/aisimulate_core/__init__.py +++ b/python/aisimulate/src/aisimulate_core/__init__.py @@ -4,7 +4,7 @@ """Public AISimulate core namespace. The implementation keeps the ``aiconfigurator_core`` package importable for -the AIC 0.12.0 compatibility window. New consumers should import this module or +the AISimulate 0.13.0 compatibility window. New consumers should import this module or ``aisimulate_core.sdk``. """ diff --git a/python/aisimulate/tests/e2e/tools/test_installed_fpm_workflow.py b/python/aisimulate/tests/e2e/tools/test_installed_fpm_workflow.py index 7cd7433af..b83651d49 100644 --- a/python/aisimulate/tests/e2e/tools/test_installed_fpm_workflow.py +++ b/python/aisimulate/tests/e2e/tools/test_installed_fpm_workflow.py @@ -112,8 +112,17 @@ def test_built_application_wheel_runs_installed_fpm_plan_and_resolves_runtime_as cwd=unrelated_repository, env=env, ) + installed_version = _run( + [ + str(installed_python), + "-c", + "import importlib.metadata; print(importlib.metadata.version('aisimulate'))", + ], + cwd=unrelated_repository, + env=env, + ).stdout.strip() - assert "Verified installed AISimulate 0.12.0 FPM workflow" in completed.stdout - assert "installed:aisimulate==0.12.0:record-sha256:" in completed.stdout + assert f"Verified installed AISimulate {installed_version} FPM workflow" in completed.stdout + assert f"installed:aisimulate=={installed_version}:record-sha256:" in completed.stdout assert unrelated_head not in completed.stdout assert str(unrelated_repository) not in completed.stdout diff --git a/python/aisimulate/tests/unit/test_aisimulate_deprecation.py b/python/aisimulate/tests/unit/test_aisimulate_deprecation.py index f2a4487d3..77814c497 100644 --- a/python/aisimulate/tests/unit/test_aisimulate_deprecation.py +++ b/python/aisimulate/tests/unit/test_aisimulate_deprecation.py @@ -52,8 +52,10 @@ def test_each_legacy_cli_mode_warns_without_renaming_the_command(monkeypatch, mo assert f"`aiconfigurator cli {mode}`" in str(warning.message) assert f"`aiconfigurator cli {mode} ...`" in str(warning.message) assert "`aisimulate cli" not in str(warning.message) - assert "AIConfigurator 0.13.0" in str(warning.message) - assert "deprecated AIConfigurator distribution" in str(warning.message) + assert "AISimulate 0.13.0" in str(warning.message) + assert "removal is targeted for AISimulate 0.14.0" in str(warning.message) + assert "deprecated compatibility command" in str(warning.message) + assert "aisimulate==0.12.0" not in str(warning.message) assert "preserves the established command name" in str(warning.message) assert warning.filename == __file__ @@ -91,5 +93,6 @@ def test_each_legacy_sweeper_entry_point_warns_once_at_caller(entry_point) -> No assert warning.category is DeprecationWarning assert entry_point.__name__ in str(warning.message) assert "aisimulate.sweeper.Sweeper" in str(warning.message) - assert "AIConfigurator 0.13.0" in str(warning.message) + assert "AISimulate 0.13.0" in str(warning.message) + assert "aisimulate==0.12.0" not in str(warning.message) assert warning.filename == __file__ diff --git a/python/aisimulate/tools/verify_installed_package_layers.py b/python/aisimulate/tools/verify_installed_package_layers.py index 7a53d6863..f6c6092cc 100644 --- a/python/aisimulate/tools/verify_installed_package_layers.py +++ b/python/aisimulate/tools/verify_installed_package_layers.py @@ -135,6 +135,8 @@ def _verify_imports() -> None: expected_facade = { "AttentionBackend", "EngineHandle", + "ForwardPassPerfModelConfig", + "ForwardPassPerfOptions", "MoEBackend", "ModelConfig", "RuntimeConfig", diff --git a/python/aisimulate/uv.lock b/python/aisimulate/uv.lock index ea992bddb..015401509 100644 --- a/python/aisimulate/uv.lock +++ b/python/aisimulate/uv.lock @@ -24,7 +24,7 @@ wheels = [ [[package]] name = "aisimulate" -version = "0.12.0" +version = "0.13.0" source = { editable = "." } dependencies = [ { name = "bokeh" }, diff --git a/scripts/apply_dev_version.py b/scripts/apply_dev_version.py index 3ad0b8e23..3e2d714fe 100644 --- a/scripts/apply_dev_version.py +++ b/scripts/apply_dev_version.py @@ -8,7 +8,7 @@ - [project].version in python/aisimulate/pyproject.toml (PEP 440 form) - [package].version in crates/core/Cargo.toml and [workspace.package].version in Cargo.toml (SemVer form: dash instead of - dot, so '0.12.0-dev.20260827' -- cargo rejects the PEP 440 spelling) + dot, so '0.13.0-dev.202609170000001234' -- cargo rejects the PEP 440 spelling) The suffix is the UTC creation date followed by a ten-digit workflow run number; legacy date-only suffixes remain accepted. The crate form keeps the diff --git a/scripts/build_release_artifacts.py b/scripts/build_release_artifacts.py index e2c92e607..1ebb8b7e8 100755 --- a/scripts/build_release_artifacts.py +++ b/scripts/build_release_artifacts.py @@ -18,9 +18,9 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -VERSION = "0.12.0" +VERSION = "0.13.0" # Nightly CI stamps a dev suffix via scripts/apply_dev_version.py: -# PEP 440 `0.12.0.devYYYYMMDD` in the wheel, SemVer `0.12.0-dev.YYYYMMDD` in +# PEP 440 `0.13.0.devYYYYMMDD` in the wheel, SemVer `0.13.0-dev.YYYYMMDD` in # the crate, optionally followed by a ten-digit run number in both formats. # Cargo rejects the PEP 440 spelling. The release contract still # anchors on VERSION; only this suffix pair is additionally accepted. diff --git a/scripts/check_prediction_numerics.py b/scripts/check_prediction_numerics.py index e85658b6c..57f7ec295 100644 --- a/scripts/check_prediction_numerics.py +++ b/scripts/check_prediction_numerics.py @@ -20,18 +20,35 @@ from pathlib import Path -def validate_cases(manifest: dict) -> list[dict]: +def resolve_baseline(manifest: dict, *, fetch: bool = False, repository_root: Path | None = None) -> str: baseline = manifest.get("baseline_source_sha") if not isinstance(baseline, str) or not re.fullmatch(r"[0-9a-f]{40}", baseline): raise ValueError("baseline_source_sha must be a full commit SHA") + root = repository_root or Path(__file__).resolve().parents[1] + command = ["git", "cat-file", "-e", f"{baseline}^{{commit}}"] resolved = subprocess.run( - ["git", "cat-file", "-e", f"{baseline}^{{commit}}"], - cwd=Path(__file__).resolve().parents[1], + command, + cwd=root, capture_output=True, timeout=10, ) + if resolved.returncode and fetch: + # Squashed PR commits may be absent even after fetching all branch history. + # Fetch only the recorded source; never replace it or refresh expected values. + subprocess.run( + ["git", "fetch", "--no-tags", "--depth=1", "--no-write-fetch-head", "origin", baseline], + cwd=root, + check=True, + timeout=60, + ) + resolved = subprocess.run(command, cwd=root, capture_output=True, timeout=10) if resolved.returncode: raise ValueError(f"baseline_source_sha does not resolve to a commit: {baseline}") + return baseline + + +def validate_cases(manifest: dict) -> list[dict]: + resolve_baseline(manifest) cases = manifest["cases"] if manifest["schema_version"] != 1 or not cases: raise ValueError("sentinel manifest must have schema 1 and nonempty cases") @@ -104,9 +121,16 @@ def main() -> int: type=Path, default=Path(__file__).resolve().parents[1] / ".github/prediction-numerical-sentinels.json", ) - parser.add_argument("--output", type=Path, required=True) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--output", type=Path) + mode.add_argument( + "--fetch-baseline-only", action="store_true", help="Fetch the recorded commit without running predictions" + ) args = parser.parse_args() manifest = json.loads(args.manifest.read_text()) + if args.fetch_baseline_only: + print(f"Numerical baseline available: {resolve_baseline(manifest, fetch=True)}") + return 0 cases = validate_cases(manifest) rows = collect(cases) failures = check_results(cases, rows) diff --git a/tests/e2e/test_unified_cli_engine.py b/tests/e2e/test_unified_cli_engine.py index 2b471b8ae..a2f5e9abe 100644 --- a/tests/e2e/test_unified_cli_engine.py +++ b/tests/e2e/test_unified_cli_engine.py @@ -299,8 +299,8 @@ def test_engine_recommend_cli_cases_round_trip(config_path: Path, tmp_path: Path deployment = prediction_to_replay_spec(concrete).backend_deployment assert raw["engine"]["hardware"] == "h200_sxm" assert raw["engine"]["workers"]["decode"]["hardware"] == "gb200" - assert deployment.prefill_engine_args["aic_system"] == "h200_sxm" - assert deployment.decode_engine_args["aic_system"] == "gb200" + assert deployment.prefill_engine_args["timing_model"]["config"]["system"] == "h200_sxm" + assert deployment.decode_engine_args["timing_model"]["config"]["system"] == "gb200" assert candidate.config["prefill_hardware_sku"] == "h200_sxm" assert candidate.config["decode_hardware_sku"] == "gb200" assert candidate.used_gpus == 2 diff --git a/tests/golden/afd-qualification.json b/tests/golden/afd-qualification.json index bfc18a497..49cca6894 100644 --- a/tests/golden/afd-qualification.json +++ b/tests/golden/afd-qualification.json @@ -50,7 +50,7 @@ "backend_version": "1.3.0rc14", "deployment_mode": "afd", "replay_spec_api_version": 1, - "replay_spec_sha256": "bbbd762b1ca036a32851c2107c765761d17c980b9f30b0bc29abd4204b8f0c8f" + "replay_spec_sha256": "3f53f3aef8fccb41331b93d9130fa7581d8cb989a614e36ce0df7f4260cc6b7f" }, "kind": "aisimulate.afd.qualification", "performance_model": { diff --git a/tests/sweeper/test_packaging.py b/tests/sweeper/test_packaging.py index b3e5fb0d1..0b491e148 100644 --- a/tests/sweeper/test_packaging.py +++ b/tests/sweeper/test_packaging.py @@ -165,10 +165,9 @@ def test_aisimulate_source_versions_are_synchronized(): core = tomllib.loads((repo_root / "crates/core/Cargo.toml").read_text()) workspace = tomllib.loads((repo_root / "Cargo.toml").read_text()) - expected_python = "0.12.0" - expected_cargo = "0.12.0" - assert project["project"]["version"] == expected_python - assert core["package"]["version"] == expected_cargo + workspace_version = workspace["workspace"]["package"]["version"] + assert project["project"]["version"] == workspace_version + assert core["package"]["version"] == workspace_version assert workspace["workspace"]["members"] == ["crates/core"] assert project["tool"]["maturin"]["manifest-path"] == "../../crates/core/Cargo.toml" diff --git a/tests/test_ci_qualification.py b/tests/test_ci_qualification.py index 146790000..acfad3b9a 100644 --- a/tests/test_ci_qualification.py +++ b/tests/test_ci_qualification.py @@ -7,11 +7,12 @@ import json import os import subprocess +import sys from pathlib import Path import pytest -from scripts.check_prediction_numerics import check_results, validate_cases +from scripts.check_prediction_numerics import check_results, resolve_baseline, validate_cases ROOT = Path(__file__).resolve().parents[1] BASELINE_SHA = json.loads((ROOT / ".github/prediction-numerical-sentinels.json").read_text())["baseline_source_sha"] @@ -64,6 +65,91 @@ def test_valid_baseline_commit_is_accepted(case): assert validate_cases({"schema_version": 1, "baseline_source_sha": BASELINE_SHA, "cases": [case]}) == [case] +def test_fetch_historical_baseline_preserves_checkout_and_works_offline_afterward(tmp_path): + upstream = tmp_path / "upstream" + checkout = tmp_path / "checkout" + upstream.mkdir() + + def git(root, *args): + return subprocess.run(["git", *args], cwd=root, check=True, capture_output=True, text=True).stdout.strip() + + git(upstream, "init", "-q", "-b", "main") + git(upstream, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "--allow-empty", "-qm", "main") + git(upstream, "checkout", "-qb", "historical-baseline") + git( + upstream, + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "--allow-empty", + "-qm", + "baseline", + ) + baseline = git(upstream, "rev-parse", "HEAD") + git(upstream, "checkout", "main") + git(tmp_path, "clone", "--depth=1", "--single-branch", upstream.as_uri(), str(checkout)) + head = git(checkout, "rev-parse", "HEAD") + manifest = {"baseline_source_sha": baseline} + with pytest.raises(ValueError, match="does not resolve"): + resolve_baseline(manifest, repository_root=checkout) + assert resolve_baseline(manifest, fetch=True, repository_root=checkout) == baseline + assert git(checkout, "rev-parse", "HEAD") == head + assert git(checkout, "branch", "--show-current") == "main" + git(checkout, "remote", "remove", "origin") + assert resolve_baseline(manifest, fetch=True, repository_root=checkout) == baseline + assert manifest == {"baseline_source_sha": baseline} + + +@pytest.mark.parametrize("baseline", [None, "main", "--upload-pack=invalid", "a" * 39, "z" * 40]) +def test_fetch_baseline_rejects_non_sha_before_git(tmp_path, baseline): + with pytest.raises(ValueError, match="full commit SHA"): + resolve_baseline({"baseline_source_sha": baseline}, fetch=True, repository_root=tmp_path) + + +def test_fetch_baseline_fails_when_origin_cannot_supply_commit(tmp_path): + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + subprocess.run(["git", "remote", "add", "origin", str(tmp_path / "missing")], cwd=tmp_path, check=True) + with pytest.raises(subprocess.CalledProcessError): + resolve_baseline({"baseline_source_sha": "0" * 40}, fetch=True, repository_root=tmp_path) + + +def test_fetch_baseline_cli_does_not_require_prediction_cases_or_write_results(tmp_path): + baseline = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps({"baseline_source_sha": baseline})) + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/check_prediction_numerics.py"), + "--manifest", + str(manifest), + "--fetch-baseline-only", + ], + cwd=tmp_path, + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == f"Numerical baseline available: {baseline}" + assert set(tmp_path.iterdir()) == {manifest} + + +@pytest.mark.parametrize("arguments", [[], ["--fetch-baseline-only", "--output", "results.json"]]) +def test_numerical_cli_requires_exactly_one_mode(tmp_path, arguments): + result = subprocess.run( + [sys.executable, str(ROOT / "scripts/check_prediction_numerics.py"), *arguments], + cwd=tmp_path, + capture_output=True, + text=True, + ) + assert result.returncode == 2 + assert "--fetch-baseline-only" in result.stderr + assert "--output" in result.stderr + assert not list(tmp_path.iterdir()) + + @pytest.mark.parametrize("base", ["", "runner:latest", "runner:2.0", "runner@sha256:abc", "runner@sha256:" + "x" * 64]) def test_image_builder_rejects_unpinned_base_before_docker(tmp_path, base): result, log = _build_image(tmp_path, base) diff --git a/tests/test_ci_workflow_contracts.py b/tests/test_ci_workflow_contracts.py index 3a3009342..d8c9bc1ae 100644 --- a/tests/test_ci_workflow_contracts.py +++ b/tests/test_ci_workflow_contracts.py @@ -17,6 +17,7 @@ import shutil import subprocess import sys +import tomllib import zipfile from pathlib import Path from types import SimpleNamespace @@ -679,6 +680,14 @@ def test_full_ci_owns_migrated_expensive_suites() -> None: assert "test_core_public_api.py" not in application_commands assert "test_core_public_api.py" not in compatibility_commands + contract_steps = [ + step for step in jobs["application-tests"]["steps"] if step.get("if") == "matrix.shard.suite == 'contracts'" + ] + assert len(contract_steps) == 1 + contract_command = contract_steps[0]["run"] + assert "--ignore=tests/fpm_accuracy" not in contract_command + assert "--ignore=tests/test_ci_workflow_contracts.py" in contract_command + recommendation_path = "tests/e2e/cli/test_cli_recommend.py" recommendation_steps = [ step for step in jobs["application-tests"]["steps"] if recommendation_path in step.get("run", "") @@ -2762,6 +2771,11 @@ def _nightly_version(created, number): ) +def _current_product_version(): + manifest = tomllib.loads((REPOSITORY_ROOT / "python/aisimulate/pyproject.toml").read_text()) + return manifest["project"]["version"] + + def test_nightly_versions_are_unique_date_ordered_and_stable_across_retries(): from packaging.version import Version @@ -2778,20 +2792,26 @@ def test_nightly_versions_are_unique_date_ordered_and_stable_across_retries(): versions.append(value["dev-version"]) assert json.loads(_nightly_version(date, number).stdout) == value assert versions == ["202609170000001234", "202609170000001235", "202609180000001236"] - assert Version("0.12.0.dev20260917") < Version("0.12.0.dev" + versions[0]) - assert [Version("0.12.0.dev" + v) for v in versions] == sorted(Version("0.12.0.dev" + v) for v in versions) + base_version = _current_product_version() + assert Version(f"{base_version}.dev20260917") < Version(f"{base_version}.dev{versions[0]}") + stamped_versions = [Version(f"{base_version}.dev{version}") for version in versions] + assert stamped_versions == sorted(stamped_versions) for number in (0, -1, 10000000000, "invalid"): assert _nightly_version("2026-09-17T00:00:00Z", number).returncode != 0 @pytest.mark.parametrize("suffix", [".dev20260917", ".dev202609170000001234"]) -@pytest.mark.parametrize("base_version", ["0.12.0", "0.11.0"]) +@pytest.mark.parametrize("base_version", [None, "0.12.0"], ids=["current", "historical"]) def test_current_release_tools_stamp_and_validate_historical_manifests(tmp_path, suffix, base_version): + current_version = _current_product_version() + base_version = base_version or current_version for name in ("Cargo.toml", "crates/core/Cargo.toml", "python/aisimulate/pyproject.toml"): target = tmp_path / name target.parent.mkdir(parents=True, exist_ok=True) target.write_text( - (REPOSITORY_ROOT / name).read_text().replace('version = "0.12.0"', f'version = "{base_version}"') + (REPOSITORY_ROOT / name) + .read_text() + .replace(f'version = "{current_version}"', f'version = "{base_version}"') ) for args in ( ["init", "-q"], diff --git a/tests/test_unified_traffic_runtime.py b/tests/test_unified_traffic_runtime.py index 92b0564fe..e587c1ef2 100644 --- a/tests/test_unified_traffic_runtime.py +++ b/tests/test_unified_traffic_runtime.py @@ -440,22 +440,27 @@ def _fpm_engine() -> dict: } -def test_prediction_spec_lowers_fpm_forward_model_onto_the_rank() -> None: +def test_prediction_spec_lowers_fpm_forward_model_into_canonical_timing() -> None: parsed = CorePredictionConfig.model_validate({"engine": _fpm_engine()}) deployment = prediction_to_replay_spec(parsed).backend_deployment - assert deployment.agg_engine_args["aic_forward_model"] == "fpm" - assert "timing_model" not in deployment.agg_engine_args + timing = deployment.agg_engine_args["timing_model"] + assert timing["type"] == "external" + assert timing["provider"] == "aic" + assert timing["config"]["estimation_mode"] == "fpm_interpolation" + assert timing["config"]["worker_type"] == "aggregated" + assert "aic_forward_model" not in deployment.agg_engine_args assert deployment.performance_model_metadata["aggregated"]["config"]["forward_model"] == "fpm" -def test_prediction_spec_omits_the_forward_model_rank_field_for_op_level() -> None: +def test_prediction_spec_lowers_default_timing_into_canonical_auto_selection() -> None: engine = _fpm_engine() engine["workers"]["aggregated"]["timing"] = {"type": "default"} parsed = CorePredictionConfig.model_validate({"engine": engine}) deployment = prediction_to_replay_spec(parsed).backend_deployment assert "aic_forward_model" not in deployment.agg_engine_args + assert deployment.agg_engine_args["timing_model"]["config"]["estimation_mode"] == "auto" assert deployment.performance_model_metadata["aggregated"]["config"]["forward_model"] == "op_level" @@ -471,7 +476,13 @@ def test_prediction_spec_lowers_forward_model_per_role_in_disaggregated_mode() - deployment = prediction_to_replay_spec(parsed).backend_deployment assert "aic_forward_model" not in deployment.prefill_engine_args - assert deployment.decode_engine_args["aic_forward_model"] == "fpm" + assert "aic_forward_model" not in deployment.decode_engine_args + prefill = deployment.prefill_engine_args["timing_model"]["config"] + decode = deployment.decode_engine_args["timing_model"]["config"] + assert prefill["estimation_mode"] == "auto" + assert prefill["worker_type"] == "prefill" + assert decode["estimation_mode"] == "fpm_interpolation" + assert decode["worker_type"] == "decode" assert deployment.performance_model_metadata["prefill"]["config"]["forward_model"] == "op_level" assert deployment.performance_model_metadata["decode"]["config"]["forward_model"] == "fpm"