Skip to content

fix(gdn): address remaining CodeRabbit feedback from #3001 #3165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kahyunnam merged 1 commit into from
Apr 27, 2026
Merged

fix(gdn): address remaining CodeRabbit feedback from #3001 #3165

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 20 additions & 18 deletions flashinfer/gdn_prefill.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,23 +269,8 @@ def chunk_gated_delta_rule(
device=q.device,
)

# Allocate output_state if needed
if output_final_state and output_state is None:
output_state = torch.empty(
(num_seqs, num_sab_heads, head_size, head_size),
dtype=torch.float32,
device=q.device,
)
elif not output_final_state and output_state is None:
# Still need to allocate since kernel always writes state
output_state = torch.empty(
(num_seqs, num_sab_heads, head_size, head_size),
dtype=torch.float32,
device=q.device,
)

device = q.device
_scale = scale if scale is not None else 1.0 / math.sqrt(head_size)
_scale = scale if scale is not None and scale != 0.0 else 1.0 / math.sqrt(head_size)

_cuda_major = int(torch.version.cuda.split(".")[0]) if torch.version.cuda else 0
if _has_blackwell_prefill and is_sm100a_supported(device) and _cuda_major >= 13:
Expand All @@ -294,6 +279,16 @@ def chunk_gated_delta_rule(
f"Blackwell GDN prefill requires head_size=128, got {head_size}"
)

# Allocate output_state only when needed
if not output_final_state:
output_state = None
elif output_state is None:
output_state = torch.empty(
(num_seqs, num_sab_heads, head_size, head_size),
dtype=torch.float32,
device=device,
)

_g = (
g
if g is not None
Expand Down Expand Up @@ -323,14 +318,21 @@ def chunk_gated_delta_rule(
output,
cu_seqlens.to(torch.int32),
initial_state,
output_state if output_final_state else None,
output_state,
_scale,
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
cu_checkpoints=_cu_checkpoints,
output_checkpoints=state_checkpoints,
)
else:
# SM90 Hopper path (C++ JIT kernel)
if output_state is None:
output_state = torch.empty(
(num_seqs, num_sab_heads, head_size, head_size),
dtype=torch.float32,
device=device,
)

workspace_size = get_device_sm_count(device) * 128
workspace_buffer = _get_cache_buf(
"gdn_prefill_workspace", workspace_size, device
Expand All @@ -346,7 +348,7 @@ def chunk_gated_delta_rule(
initial_state,
g,
beta,
scale if scale is not None else 0.0,
_scale,
workspace_buffer,
state_checkpoints,
checkpoint_cu_starts.to(torch.int64)
Expand Down
Loading