Skip to content
65 changes: 65 additions & 0 deletions tests/kernels/attention/test_flash_attn.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,3 +215,68 @@ def test_varlen_with_paged_kv(
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol),
f"{torch.max(torch.abs(output - ref_output))}",
)


@torch.inference_mode()
def test_fa4_fp8_per_head_descales() -> None:
"""FA4 must apply non-unity per-KV-head FP8 Q/K/V descales."""
torch.set_default_device("cuda")
if not is_fa_version_supported(4):
pytest.skip(
f"Flash attention version 4 is not supported due "
f'to: "{fa_version_unsupported_reason(4)}"'
)

set_random_seed(0)
num_query_heads = 4
num_kv_heads = 2
head_size = 128
query_len = 3
kv_len = 7
scale = head_size**-0.5

query = torch.randn(query_len, num_query_heads, head_size)
key = torch.randn(kv_len, num_kv_heads, head_size)
value = torch.randn_like(key)
q_descale = torch.tensor([[0.125, 2.0]], dtype=torch.float32)
k_descale = torch.tensor([[0.25, 1.5]], dtype=torch.float32)
v_descale = torch.tensor([[0.5, 2.0]], dtype=torch.float32)

query_scale = torch.repeat_interleave(q_descale[0], num_query_heads // num_kv_heads)
query_fp8 = (query / query_scale[None, :, None]).to(torch.float8_e4m3fn)
key_fp8 = (key / k_descale[0, :, None]).to(torch.float8_e4m3fn)
value_fp8 = (value / v_descale[0, :, None]).to(torch.float8_e4m3fn)

output = flash_attn_varlen_func(
q=query_fp8,
k=key_fp8,
v=value_fp8,
cu_seqlens_q=torch.tensor([0, query_len], dtype=torch.int32),
cu_seqlens_k=torch.tensor([0, kv_len], dtype=torch.int32),
max_seqlen_q=query_len,
max_seqlen_k=kv_len,
softmax_scale=scale,
causal=True,
fa_version=4,
q_descale=q_descale,
k_descale=k_descale,
v_descale=v_descale,
)

query_head_to_kv = torch.arange(num_query_heads) // (
num_query_heads // num_kv_heads
)
key_broadcast = key[:, query_head_to_kv]
value_broadcast = value[:, query_head_to_kv]
scores = torch.einsum("qhd,khd->hqk", query, key_broadcast) * scale
scores = scores.masked_fill(
torch.triu(
torch.ones(query_len, kv_len), diagonal=kv_len - query_len + 1
).bool(),
float("-inf"),
)
reference = torch.einsum(
"hqk,khd->qhd", torch.softmax(scores.float(), dim=-1), value_broadcast
)

torch.testing.assert_close(output.float(), reference, atol=0.2, rtol=0.2)
Comment thread
yiliu30 marked this conversation as resolved.
Resolved
3 changes: 3 additions & 0 deletions vllm/vllm_flash_attn/flash_attn_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,9 @@ def flash_attn_varlen_func(
block_sparse_tensors=block_sparse_tensors,
aux_tensors=aux_tensors,
aux_tensor_leading_dims=aux_tensor_leading_dims,
q_descale=q_descale,
k_descale=k_descale,
v_descale=v_descale,
output_scale=output_scale,
)
else:
Expand Down
Loading