Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
3 changes: 1 addition & 2 deletions .buildkite/lm-eval-harness/test_lm_eval_correctness.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
LM eval harness on model to compare vs HF baseline computed offline.
"""LM eval harness on model to compare vs HF baseline computed offline.
Configs are found in configs/$MODEL.yaml

pytest -s -v test_lm_eval_correctness.py \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,7 @@ def compare_data_columns(
drop_column: str,
debug: bool = False,
):
"""
Align concatenation by keys derived from info_cols instead of row order.
"""Align concatenation by keys derived from info_cols instead of row order.
- Pick one canonical key list: subset of info_cols present in ALL files.
- For each file: set index to those keys, aggregate duplicates
(mean for metric, first for names).
Expand Down Expand Up @@ -363,8 +362,7 @@ def _apply_two_decimals(
# Export helpers (Excel + CSV)
# -----------------------------
def _sanitize_sheet_name(name: str) -> str:
"""
Excel sheet constraints:
"""Excel sheet constraints:
- max 31 chars
- cannot contain: : \ / ? * [ ]
- cannot be empty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,7 @@ def results_to_json(latency, throughput, serving):


def get_size_with_unit(bytes, suffix="B"):
"""
Scale bytes to its proper format
"""Scale bytes to its proper format
e.g:
1253656 => '1.20MB'
1253656678 => '1.17GB'
Expand Down
21 changes: 8 additions & 13 deletions .buildkite/scripts/generate-nightly-index.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@


def normalize_package_name(name: str) -> str:
"""
Normalize package name according to PEP 503.
"""Normalize package name according to PEP 503.
https://peps.python.org/pep-0503/#normalized-names
Replace runs of underscores, hyphens, and periods with a single hyphen,
Expand Down Expand Up @@ -55,17 +54,18 @@ class WheelFileInfo:


def parse_from_filename(file: str) -> WheelFileInfo:
"""
Parse wheel file name to extract metadata.
"""Parse wheel file name to extract metadata.
The format of wheel names:
{package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl
All versions could contain a variant like '+cu129' or '.cpu' or `.rocm` (or not).
Example:
vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
vllm-0.10.2rc2+cu129-cp38-abi3-manylinux2014_aarch64.whl
vllm-0.11.1rc8.dev14+gaa384b3c0-cp38-abi3-manylinux2014_aarch64.whl
vllm-0.11.1rc8.dev14+gaa384b3c0.cu130-cp38-abi3-manylinux1_x86_64.whl
"""
wheel_file_re = re.compile(
r"^(?P<package_name>.+)-(?P<version>[^-]+?)(-(?P<build_tag>[^-]+))?-(?P<python_tag>[^-]+)-(?P<abi_tag>[^-]+)-(?P<platform_tag>[^-]+)\.whl$"
Expand Down Expand Up @@ -111,9 +111,7 @@ def parse_from_filename(file: str) -> WheelFileInfo:


def generate_project_list(subdir_names: list[str], comment: str = "") -> str:
"""
Generate project list HTML content linking to each project & variant subdirectory.
"""
"""Generate project list HTML content linking to each project & variant subdirectory."""
href_tags = []
for name in sorted(subdir_names):
name = name.strip("/").strip(".")
Expand All @@ -127,9 +125,7 @@ def generate_package_index_and_metadata(
index_base_dir: Path,
comment: str = "",
) -> tuple[str, str]:
"""
Generate package index HTML content for a specific package, linking to actual wheel files.
"""
"""Generate package index HTML content for a specific package, linking to actual wheel files."""
href_tags = []
metadata = []
for file in sorted(wheel_files, key=lambda x: x.filename):
Expand All @@ -156,8 +152,7 @@ def generate_index_and_metadata(
alias_to_default: str | None = None,
comment: str = "",
):
"""
Generate index for all wheel files.
"""Generate index for all wheel files.
Args:
whl_files (list[str]): List of wheel files (must be directly under `wheel_base_dir`).
Expand Down Expand Up @@ -216,8 +211,8 @@ def generate_index_and_metadata(
},
...
]
"""
"""
parsed_files = [parse_from_filename(f) for f in whl_files]

if not parsed_files:
Expand Down
28 changes: 14 additions & 14 deletions benchmarks/attention_benchmarks/batch_spec.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""
Simplified batch specification grammar for attention benchmarks.
"""Simplified batch specification grammar for attention benchmarks.

Grammar (underscore-separated segments):
Format: (<count>?) q<q_len>(k?) (s<seq_len>(k?))?
Expand All @@ -24,6 +23,7 @@
q4s1k -> [(4, 1024)] # 4-token extend (spec decode)
2q1k_32q1s1k -> [(1024, 1024)] * 2 + [(1, 1024)] * 32 # Mixed batch
16q4s1k -> [(4, 1024)] * 16 # 16 spec decode requests

"""

from collections import Counter
Expand Down Expand Up @@ -71,8 +71,7 @@ def _parse_size(size_str: str, k_suffix: str) -> int:


def parse_batch_spec(spec: str) -> list[BatchRequest]:
"""
Parse batch specification string into list of BatchRequest objects.
"""Parse batch specification string into list of BatchRequest objects.

Grammar: (<count>?) q<q_len>(k?) (s<seq_len>(k?))?

Expand All @@ -84,6 +83,7 @@ def parse_batch_spec(spec: str) -> list[BatchRequest]:

Raises:
ValueError: If spec format is invalid

"""
requests = []

Expand All @@ -103,8 +103,7 @@ def parse_batch_spec(spec: str) -> list[BatchRequest]:


def format_batch_spec(requests: list[BatchRequest]) -> str:
"""
Format list of BatchRequest into human-readable string.
"""Format list of BatchRequest into human-readable string.

Groups requests by type and provides counts and sizes.

Expand All @@ -113,6 +112,7 @@ def format_batch_spec(requests: list[BatchRequest]) -> str:

Returns:
Formatted string describing the batch

"""
kinds = {
"prefill": [],
Expand Down Expand Up @@ -157,8 +157,7 @@ def format_batch_spec(requests: list[BatchRequest]) -> str:


def reorder_for_flashinfer(requests: list[BatchRequest]) -> list[BatchRequest]:
"""
Reorder requests for FlashInfer: decode first, then prefill.
"""Reorder requests for FlashInfer: decode first, then prefill.

FlashInfer expects decode requests before prefill requests for
optimal performance.
Expand All @@ -168,6 +167,7 @@ def reorder_for_flashinfer(requests: list[BatchRequest]) -> list[BatchRequest]:

Returns:
Reordered list with decode requests first

"""
decodes = [r for r in requests if r.is_decode]
non_decodes = [r for r in requests if not r.is_decode]
Expand All @@ -177,14 +177,14 @@ def reorder_for_flashinfer(requests: list[BatchRequest]) -> list[BatchRequest]:
def split_by_type(
requests: list[BatchRequest],
) -> dict[str, list[BatchRequest]]:
"""
Split requests by type for analysis.
"""Split requests by type for analysis.

Args:
requests: List of BatchRequest

Returns:
Dict with keys: 'decode', 'prefill', 'extend'

"""
result = {
"decode": [],
Expand All @@ -204,14 +204,14 @@ def split_by_type(


def get_batch_stats(requests: list[BatchRequest]) -> dict:
"""
Compute statistics about a batch.
"""Compute statistics about a batch.

Args:
requests: List of BatchRequest

Returns:
Dict with batch statistics

"""
by_type = split_by_type(requests)

Expand All @@ -232,15 +232,15 @@ def get_batch_stats(requests: list[BatchRequest]) -> dict:


def get_batch_type(batch_spec: str, spec_decode_threshold: int = 8) -> str:
"""
Classify a batch spec into a type string.
"""Classify a batch spec into a type string.

Args:
batch_spec: Batch specification string (e.g., "q2k", "8q1s1k", "2q2k_8q1s1k")
spec_decode_threshold: Max q_len to be considered spec-decode vs extend

Returns:
Type string: "prefill", "decode", "spec-decode", "extend", or "mixed (types...)"

"""
requests = parse_batch_spec(batch_spec)

Expand Down
20 changes: 10 additions & 10 deletions benchmarks/attention_benchmarks/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""
Universal vLLM Attention Benchmark
"""Universal vLLM Attention Benchmark.

Benchmark any attention backend with the extended grammar.
Supports standard attention (Flash/Triton/FlashInfer) and MLA backends.
Expand All @@ -23,6 +22,7 @@

# Parameter sweep (YAML config - recommended)
python benchmark.py --config configs/cutlass_numsplits.yaml

"""

import argparse
Expand Down Expand Up @@ -84,15 +84,15 @@ def run_mla_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:


def run_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
"""
Run a single benchmark with proper backend selection.
"""Run a single benchmark with proper backend selection.

Args:
config: BenchmarkConfig with backend, batch_spec, and model params
**kwargs: Additional arguments passed to MLA benchmarks

Returns:
BenchmarkResult (may have error field set on failure)

"""
try:
if is_mla_backend(config.backend):
Expand All @@ -119,8 +119,7 @@ def run_model_parameter_sweep(
sweep: ModelParameterSweep,
console: Console,
) -> list[BenchmarkResult]:
"""
Run model parameter sweep for given backends and batch specs.
"""Run model parameter sweep for given backends and batch specs.

Args:
backends: List of backend names
Expand All @@ -131,6 +130,7 @@ def run_model_parameter_sweep(

Returns:
List of BenchmarkResult objects

"""
all_results = []

Expand Down Expand Up @@ -304,8 +304,7 @@ def run_parameter_sweep(
sweep: ParameterSweep,
console: Console,
) -> list[BenchmarkResult]:
"""
Run parameter sweep for given backends and batch specs.
"""Run parameter sweep for given backends and batch specs.

Args:
backends: List of backend names
Expand All @@ -316,6 +315,7 @@ def run_parameter_sweep(

Returns:
List of BenchmarkResult objects

"""
all_results = []

Expand Down Expand Up @@ -395,8 +395,7 @@ def load_config_from_yaml(config_path: str) -> dict:


def generate_batch_specs_from_ranges(ranges: list[dict]) -> list[str]:
"""
Generate batch specs from range specifications.
"""Generate batch specs from range specifications.

Args:
ranges: List of range specifications, each containing:
Expand All @@ -420,6 +419,7 @@ def generate_batch_specs_from_ranges(ranges: list[dict]) -> list[str]:
}
]
Returns: ["q1kv1k", "q2kv1k", ..., "q16kv1k"]

"""
all_specs = []

Expand Down
15 changes: 7 additions & 8 deletions benchmarks/attention_benchmarks/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@


def batch_spec_sort_key(spec: str) -> tuple[int, int, int]:
"""
Extract sorting key from batch spec: (batch_size, max_q_len, max_kv_len).
"""Extract sorting key from batch spec: (batch_size, max_q_len, max_kv_len).

This ensures results are sorted by batch size first, then query length,
then sequence length, rather than alphabetically.
Expand Down Expand Up @@ -354,13 +353,13 @@ def print_table(
backends: list[str],
compare_to_fastest: bool = True,
):
"""
Print results as a rich table.
"""Print results as a rich table.

Args:
results: List of BenchmarkResult
backends: List of backend names being compared
compare_to_fastest: Show percentage comparison to fastest

"""
# Group by batch spec, preserving first-occurrence order
by_spec = {}
Expand Down Expand Up @@ -489,14 +488,14 @@ def save_json(self, results: list[BenchmarkResult], path: str):


def setup_mla_dims(model_name: str = "deepseek-v3") -> dict:
"""
Get MLA dimensions for known models.
"""Get MLA dimensions for known models.

Args:
model_name: Model identifier

Returns:
Dict with MLA dimension configuration

"""
configs = {
"deepseek-v2": {
Expand Down Expand Up @@ -542,15 +541,15 @@ def get_attention_scale(head_dim: int) -> float:


def is_mla_backend(backend: str) -> bool:
"""
Check if backend is an MLA backend using the AttentionBackendEnum.
"""Check if backend is an MLA backend using the AttentionBackendEnum.

Args:
backend: Backend name matching AttentionBackendEnum exactly
(e.g., "FLASHMLA_SPARSE")

Returns:
True if the backend is an MLA backend, False otherwise

"""
from vllm.v1.attention.backends.registry import AttentionBackendEnum

Expand Down
Loading
Loading