Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions flashinfer/gemm/gemm_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
OptimizationProfile,
TunableRunner,
TuningConfig,
is_in_profile_measurement,
)
from ..fused_moe.utils import (
get_hybrid_num_tokens_buckets,
Expand Down Expand Up @@ -2393,6 +2394,17 @@ def _get_cudnn_plan_index_for_tactic(graph, tactic) -> int:
plan_index = candidate_plan_index
break
if plan_index < 0:
# During autotuner profiling a requested tactic that resolves to
# no built plan must be reported as FAILED, not silently timed as
# tactic=-1 (raised in #3707 review; else the default's timing is
# attributed to the requested tactic and a tactic that never ran
# can win). Serving keeps the warn+fallback for robustness.
if is_in_profile_measurement():
raise RuntimeError(
"cuDNN GEMM engine/knob tactic matched no built execution "
"plan during profiling; reporting as unsupported instead "
"of silently using tactic=-1."
)
warnings.warn(
"cuDNN GEMM engine/knob tactic did not match any built execution "
"plan; falling back to default tactic=-1.",
Expand All @@ -2402,6 +2414,12 @@ def _get_cudnn_plan_index_for_tactic(graph, tactic) -> int:
plan_index = tactic

if plan_index >= graph.get_execution_plan_count():
if is_in_profile_measurement():
raise RuntimeError(
f"cuDNN GEMM plan index {plan_index} is out of range "
f"({graph.get_execution_plan_count()} plans) during profiling; "
f"reporting as unsupported instead of silently using tactic=-1."
)
warnings.warn(
f"cuDNN GEMM plan index {plan_index} is out of range "
f"(execution plan count: {graph.get_execution_plan_count()}); "
Expand Down Expand Up @@ -3605,6 +3623,14 @@ def forward(
tactic=tactic,
)
except Exception as exc:
# During autotuner profiling, report the failed tactic as
# unsupported (the autotuner marks it inf and disqualifies it)
# instead of silently timing tactic=-1 -- otherwise the
# fallback's time is attributed to the requested tactic and a
# tactic that never ran can win (raised in #3707 review).
# Serving keeps the warn+fallback for robustness.
if is_in_profile_measurement():
raise
warnings.warn(
"cuDNN fp8 GEMM tactic failed; falling back to default "
f"tactic=-1. ({exc})",
Expand Down Expand Up @@ -4121,6 +4147,14 @@ def forward(
else:
_cudnn_gemm_bf16(workspace_buffer, a, b, bias, out, tactic=tactic)
except Exception as exc:
# During autotuner profiling, report the failed tactic as
# unsupported (the autotuner marks it inf and disqualifies it)
# instead of silently timing tactic=-1 -- otherwise the
# fallback's time is attributed to the requested tactic and a
# tactic that never ran can win (raised in #3707 review).
# Serving keeps the warn+fallback for robustness.
if is_in_profile_measurement():
raise
warnings.warn(
"cuDNN bf16 GEMM tactic failed; falling back to default "
f"tactic=-1. ({exc})",
Expand Down Expand Up @@ -5052,6 +5086,14 @@ def forward(
tactic=tactic,
)
except Exception as exc:
# During autotuner profiling, report the failed tactic as
# unsupported (the autotuner marks it inf and disqualifies it)
# instead of silently timing tactic=-1 -- otherwise the
# fallback's time is attributed to the requested tactic and a
# tactic that never ran can win (raised in #3707 review).
# Serving keeps the warn+fallback for robustness.
if is_in_profile_measurement():
raise
warnings.warn(
"cuDNN mxfp8 GEMM tactic failed; falling back to default "
f"tactic=-1. ({exc})",
Expand Down Expand Up @@ -5524,6 +5566,14 @@ def forward(
tactic=tactic,
)
except Exception as exc:
# During autotuner profiling, report the failed tactic as
# unsupported (the autotuner marks it inf and disqualifies it)
# instead of silently timing tactic=-1 -- otherwise the
# fallback's time is attributed to the requested tactic and a
# tactic that never ran can win (raised in #3707 review).
# Serving keeps the warn+fallback for robustness.
if is_in_profile_measurement():
raise
warnings.warn(
"cuDNN fp4 GEMM tactic failed; falling back to default "
f"tactic=-1. ({exc})",
Expand Down Expand Up @@ -8915,6 +8965,14 @@ def forward(
tactic=tactic,
)
except Exception as exc:
# During autotuner profiling, report the failed tactic as
# unsupported (the autotuner marks it inf and disqualifies it)
# instead of silently timing tactic=-1 -- otherwise the
# fallback's time is attributed to the requested tactic and a
# tactic that never ran can win (raised in #3707 review).
# Serving keeps the warn+fallback for robustness.
if is_in_profile_measurement():
raise
warnings.warn(
"cuDNN mxfp8 GEMM tactic failed; falling back to default "
f"tactic=-1. ({exc})",
Expand Down
6 changes: 6 additions & 0 deletions flashinfer/gemm/gemm_bf16_fp4_cudnn.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
OptimizationProfile,
TunableRunner,
TuningConfig,
is_in_profile_measurement,
)
from ..fused_moe.utils import (
get_hybrid_num_tokens_buckets,
Expand Down Expand Up @@ -505,6 +506,11 @@ def forward(
tactic=tactic,
)
except Exception as exc:
# During autotuner profiling, report the failed tactic as
# unsupported (marked inf) instead of silently timing
# tactic=-1 -- see the note in gemm_base.py (#3707 review).
if is_in_profile_measurement():
raise
warnings.warn(
"cuDNN bf16-fp4 GEMM tactic failed; falling back to default "
f"tactic=-1. ({exc})",
Expand Down
63 changes: 63 additions & 0 deletions tests/autotuner/test_cudnn_profiling_fallback.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""A cuDNN GEMM tactic that cannot be used must be reported as FAILED during
autotuner profiling (so the autotuner disqualifies it), while still falling
back to tactic=-1 outside profiling for serving robustness.

Regression for the silent-fallback-during-profiling issue raised in the
#3707 review: otherwise the fallback's timing is attributed to the requested
tactic and a tactic that never ran can win the autotune.
"""

import warnings

import pytest
import torch

from flashinfer.autotuner.autotuner import _profile_measurement_scope
from flashinfer.utils import get_compute_capability


def _fp8_inputs(m=8, n=4096, k=4096):
from tests.utils_fp8 import to_float8

a = torch.randn([1, m, k], device="cuda", dtype=torch.bfloat16)
b = torch.randn([1, n, k], device="cuda", dtype=torch.bfloat16).transpose(-2, -1)
a8, a_scale = to_float8(a)
b8, b_scale = to_float8(b)
out = torch.empty([1, m, n], device="cuda", dtype=torch.bfloat16)
ws = torch.empty(32 * 1024 * 1024, device="cuda", dtype=torch.uint8)
Comment on lines +26 to +27

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

Initialize the output with NaNs before asserting fallback output.

torch.empty may already contain finite values, allowing this assertion to pass if fallback never writes out. Initialize out to NaNs so isfinite() proves the fallback populated every result element.

Suggested fix
-    out = torch.empty([1, m, n], device="cuda", dtype=torch.bfloat16)
+    out = torch.full(
+        [1, m, n], float("nan"), device="cuda", dtype=torch.bfloat16
+    )

Also applies to: 54-55

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@tests/autotuner/test_cudnn_profiling_fallback.py` around lines 26 - 27,
Update the output allocation in the test setup around the existing out tensor to
initialize every element to NaN instead of leaving it uninitialized, while
preserving its CUDA device and bfloat16 dtype. Apply the same initialization to
the additional output allocation noted in the comment so the subsequent isfinite
assertions reliably detect whether fallback populated the results.

return [a8, b8, a_scale, b_scale, out, ws]


@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
def test_cudnn_fp8_tactic_failure_raises_in_profiling_but_falls_back_serving():
if get_compute_capability(torch.device("cuda"))[0] < 9:
pytest.skip("cuDNN FP8 GEMM requires SM90+")
from flashinfer.gemm.gemm_base import (
_check_cudnn_availability,
_cudnn_gemm_fp8_runner,
)

try:
_check_cudnn_availability()
except Exception:
pytest.skip("cuDNN not available")

runner = _cudnn_gemm_fp8_runner()
inputs = _fp8_inputs()
bad_tactic = 9999 # out-of-range plan index -> deterministic fallback path

# Serving (not profiling): warn + fall back to -1 + produce a valid result.
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
runner.forward(inputs, tactic=bad_tactic)
assert any("falling back to default tactic=-1" in str(x.message) for x in w)
torch.cuda.synchronize()
assert torch.isfinite(inputs[4]).all()

# Profiling: the failed tactic must raise so the autotuner marks it inf,
# rather than silently timing the fallback under the requested tactic.
with (
pytest.raises(RuntimeError, match="during profiling"),
_profile_measurement_scope(),
):
runner.forward(inputs, tactic=bad_tactic)
Loading