Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions flashinfer/attention/prims_ts/_block_sparse/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ def _compile_block_sparse(key: _BlockSparseCompileKey) -> Callable[..., object]:
"use_proxy_routes": key.use_proxy_routes,
"use_causal_mask": key.mask_type == "causal",
"apply_token_mask": key.use_kv_valid_bits,
"store_score_words": config.uses_prepared_score_keep_words,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does this mean ?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It means we have prepared mask words before attention kernel so kernel can just read accroding mask word to do masking with no need to resolve this on the fly.

}
if key.page_size is not None:
if key.sparse_format != "bsr" or key.use_proxy_routes:
Expand Down
32 changes: 18 additions & 14 deletions flashinfer/attention/prims_ts/_block_sparse/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ class _BlockSparseLaunchSpec:

policy: tuple[tuple[str, object], ...]
compile_key: _BlockSparseCompileKey
# Whether prepared routes carry K32 score-validity words; the decode
# config owns this rule and the plan sizes its route storage from it.
prepares_score_words: bool


_CAPACITY_UNSET = object()
Expand Down Expand Up @@ -195,21 +198,20 @@ def _select_block_sparse_scheduler(
mask_type: Literal["dense", "causal"],
use_kv_valid_bits: bool,
max_row_route_capacity: int,
use_proxy_routes: bool,
) -> tuple[int, bool]:
"""Select the Q tile and scheduler without depending on KV storage."""
"""Select the Q tile and scheduler without depending on KV storage.

Proxy routes add one summary route per row on top of the exact routes and
see the same per-tile fixed cost the persistent scheduler amortizes, so
the selection does not depend on the route kind.
"""

heads_q_per_kv = num_qo_heads // num_kv_heads
q_tile_size = _select_block_sparse_q_tile_size(
q_block_size=q_block_size,
heads_q_per_kv=heads_q_per_kv,
kv_block_size=kv_block_size,
)
# Reusable planning sees route capacity, not the live exact-route work.
# Keep proxy execution on the direct grid as a conservative workload-level
# default until runtime work can participate in scheduling.
if use_proxy_routes:
return q_tile_size, False
if not _should_consider_clc(
q_tile_size=q_tile_size,
kv_block_size=kv_block_size,
Expand Down Expand Up @@ -360,11 +362,13 @@ def _validate_block_sparse_static_profile(
kv_block_size=kv_block_size,
)
if page_size is not None:
# Validate the paged route geometry with a capacity-free layout; the
# score-word slots do not take part in the page/atom checks.
_BlockSparseRouteLayout.create(
kv_route_size=kv_route_size,
kv_block_size=kv_block_size,
page_size=page_size,
has_token_bits=use_kv_valid_bits,
has_token_bits=False,
route_metadata_capacity=0,
num_rows=1,
)
Expand Down Expand Up @@ -473,9 +477,9 @@ def _resolve_block_sparse_launch_spec(

``max_row_route_capacity`` is a conservative prepared-route bound. Live
index values and physical-tail morphology never specialize this cache
entry. Proxy routes use the direct grid; exact routes retain the common
automatic scheduler. An unsupported persistent profile falls back to its
valid static counterpart.
entry. Proxy and exact routes share one scheduler selection. An
unsupported persistent profile falls back to its valid static
counterpart.
"""

q_tile_size, use_persistent_scheduler = _select_block_sparse_scheduler(
Expand All @@ -490,7 +494,6 @@ def _resolve_block_sparse_launch_spec(
mask_type=mask_type,
use_kv_valid_bits=use_kv_valid_bits,
max_row_route_capacity=max_row_route_capacity,
use_proxy_routes=use_proxy_routes,
)
compile_key = _BlockSparseCompileKey(
device_index=device_index,
Expand Down Expand Up @@ -518,7 +521,7 @@ def _resolve_block_sparse_launch_spec(
page_size=page_size,
)
try:
_make_block_sparse_config(compile_key)
config = _make_block_sparse_config(compile_key)
except ValueError:
if not compile_key.use_persistent_scheduler:
raise
Expand All @@ -532,7 +535,7 @@ def _resolve_block_sparse_launch_spec(
use_persistent_scheduler=False,
),
)
_make_block_sparse_config(compile_key)
config = _make_block_sparse_config(compile_key)

policy_entries: list[tuple[str, object]] = [
("tile_size_q", q_tile_size),
Expand All @@ -558,6 +561,7 @@ def _resolve_block_sparse_launch_spec(
return _BlockSparseLaunchSpec(
policy=tuple(policy_entries),
compile_key=compile_key,
prepares_score_words=config.uses_prepared_score_keep_words,
)


Expand Down
16 changes: 8 additions & 8 deletions flashinfer/attention/prims_ts/_block_sparse/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,14 +207,6 @@ def _build_block_sparse_plan_state(
static.kv_block_size,
)
max_row_route_capacity += ceil_div(num_summaries, static.kv_route_size)
route_layout = _BlockSparseRouteLayout.create(
kv_route_size=static.kv_route_size,
kv_block_size=static.kv_block_size,
page_size=static.page_size,
has_token_bits=static.use_kv_valid_bits or use_proxy_routes,
route_metadata_capacity=num_rows * max_row_route_capacity,
num_rows=num_rows,
)
with torch.cuda.device(device_index), torch.cuda.stream(plan_stream):
spec = _resolve_block_sparse_launch_spec(
device_index=device_index,
Expand All @@ -239,6 +231,14 @@ def _build_block_sparse_plan_state(
*spec.policy,
("max_blocks_per_row", static.max_blocks_per_row),
)
route_layout = _BlockSparseRouteLayout.create(
kv_route_size=static.kv_route_size,
kv_block_size=static.kv_block_size,
page_size=static.page_size,
has_token_bits=spec.prepares_score_words,
route_metadata_capacity=num_rows * max_row_route_capacity,
num_rows=num_rows,
)
compiled = _get_compiled_block_sparse(spec.compile_key)
dummy_kv_valid_bits = (
None
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -604,20 +604,25 @@ def __init__(
use_proxy_routes: bool,
use_causal_mask: bool = False,
apply_token_mask: bool = False,
store_score_words: bool = False,
page_size: int | None = None,
) -> None:
if not isinstance(use_proxy_routes, bool):
raise TypeError("use_proxy_routes must be a bool")
if not isinstance(apply_token_mask, bool):
raise TypeError("apply_token_mask must be a bool")
if not isinstance(store_score_words, bool):
raise TypeError("store_score_words must be a bool")
if not isinstance(use_causal_mask, bool):
raise TypeError("use_causal_mask must be a bool")
if use_proxy_routes and page_size is not None:
raise ValueError("paged KV does not support proxy routes")

num_q_blocks = (seq_len_q + q_block_size - 1) // q_block_size
num_rows = batch_size * num_kv_heads * num_q_blocks
stores_score_words = use_proxy_routes or apply_token_mask
# Structural score words (sequence tail, invalid atoms) can be stored
# without a caller token mask; proxy routes and token masks require them.
stores_score_words = use_proxy_routes or apply_token_mask or store_score_words
layout = _BlockSparseRouteLayout.create(
kv_route_size=kv_route_size,
kv_block_size=kv_block_size,
Expand Down Expand Up @@ -893,6 +898,7 @@ def __init__(
use_proxy_routes: bool,
use_causal_mask: bool = False,
apply_token_mask: bool = False,
store_score_words: bool = False,
) -> None:
super().__init__(
batch_size=batch_size,
Expand All @@ -905,6 +911,7 @@ def __init__(
use_proxy_routes=use_proxy_routes,
use_causal_mask=use_causal_mask,
apply_token_mask=apply_token_mask,
store_score_words=store_score_words,
)

@cute.jit
Expand Down
Loading
Loading