diff --git a/flashinfer/gemm/gemm_base.py b/flashinfer/gemm/gemm_base.py index 3d9abd7b791..13a563d0b15 100755 --- a/flashinfer/gemm/gemm_base.py +++ b/flashinfer/gemm/gemm_base.py @@ -48,6 +48,7 @@ OptimizationProfile, TunableRunner, TuningConfig, + is_in_profile_measurement, ) from ..fused_moe.utils import ( get_hybrid_num_tokens_buckets, @@ -2393,6 +2394,17 @@ def _get_cudnn_plan_index_for_tactic(graph, tactic) -> int: plan_index = candidate_plan_index break if plan_index < 0: + # During autotuner profiling a requested tactic that resolves to + # no built plan must be reported as FAILED, not silently timed as + # tactic=-1 (raised in #3707 review; else the default's timing is + # attributed to the requested tactic and a tactic that never ran + # can win). Serving keeps the warn+fallback for robustness. + if is_in_profile_measurement(): + raise RuntimeError( + "cuDNN GEMM engine/knob tactic matched no built execution " + "plan during profiling; reporting as unsupported instead " + "of silently using tactic=-1." + ) warnings.warn( "cuDNN GEMM engine/knob tactic did not match any built execution " "plan; falling back to default tactic=-1.", @@ -2402,6 +2414,12 @@ def _get_cudnn_plan_index_for_tactic(graph, tactic) -> int: plan_index = tactic if plan_index >= graph.get_execution_plan_count(): + if is_in_profile_measurement(): + raise RuntimeError( + f"cuDNN GEMM plan index {plan_index} is out of range " + f"({graph.get_execution_plan_count()} plans) during profiling; " + f"reporting as unsupported instead of silently using tactic=-1." + ) warnings.warn( f"cuDNN GEMM plan index {plan_index} is out of range " f"(execution plan count: {graph.get_execution_plan_count()}); " @@ -3605,6 +3623,14 @@ def forward( tactic=tactic, ) except Exception as exc: + # During autotuner profiling, report the failed tactic as + # unsupported (the autotuner marks it inf and disqualifies it) + # instead of silently timing tactic=-1 -- otherwise the + # fallback's time is attributed to the requested tactic and a + # tactic that never ran can win (raised in #3707 review). + # Serving keeps the warn+fallback for robustness. + if is_in_profile_measurement(): + raise warnings.warn( "cuDNN fp8 GEMM tactic failed; falling back to default " f"tactic=-1. ({exc})", @@ -4121,6 +4147,14 @@ def forward( else: _cudnn_gemm_bf16(workspace_buffer, a, b, bias, out, tactic=tactic) except Exception as exc: + # During autotuner profiling, report the failed tactic as + # unsupported (the autotuner marks it inf and disqualifies it) + # instead of silently timing tactic=-1 -- otherwise the + # fallback's time is attributed to the requested tactic and a + # tactic that never ran can win (raised in #3707 review). + # Serving keeps the warn+fallback for robustness. + if is_in_profile_measurement(): + raise warnings.warn( "cuDNN bf16 GEMM tactic failed; falling back to default " f"tactic=-1. ({exc})", @@ -5052,6 +5086,14 @@ def forward( tactic=tactic, ) except Exception as exc: + # During autotuner profiling, report the failed tactic as + # unsupported (the autotuner marks it inf and disqualifies it) + # instead of silently timing tactic=-1 -- otherwise the + # fallback's time is attributed to the requested tactic and a + # tactic that never ran can win (raised in #3707 review). + # Serving keeps the warn+fallback for robustness. + if is_in_profile_measurement(): + raise warnings.warn( "cuDNN mxfp8 GEMM tactic failed; falling back to default " f"tactic=-1. ({exc})", @@ -5524,6 +5566,14 @@ def forward( tactic=tactic, ) except Exception as exc: + # During autotuner profiling, report the failed tactic as + # unsupported (the autotuner marks it inf and disqualifies it) + # instead of silently timing tactic=-1 -- otherwise the + # fallback's time is attributed to the requested tactic and a + # tactic that never ran can win (raised in #3707 review). + # Serving keeps the warn+fallback for robustness. + if is_in_profile_measurement(): + raise warnings.warn( "cuDNN fp4 GEMM tactic failed; falling back to default " f"tactic=-1. ({exc})", @@ -8915,6 +8965,14 @@ def forward( tactic=tactic, ) except Exception as exc: + # During autotuner profiling, report the failed tactic as + # unsupported (the autotuner marks it inf and disqualifies it) + # instead of silently timing tactic=-1 -- otherwise the + # fallback's time is attributed to the requested tactic and a + # tactic that never ran can win (raised in #3707 review). + # Serving keeps the warn+fallback for robustness. + if is_in_profile_measurement(): + raise warnings.warn( "cuDNN mxfp8 GEMM tactic failed; falling back to default " f"tactic=-1. ({exc})", diff --git a/flashinfer/gemm/gemm_bf16_fp4_cudnn.py b/flashinfer/gemm/gemm_bf16_fp4_cudnn.py index ab231d6bef3..749c2821603 100644 --- a/flashinfer/gemm/gemm_bf16_fp4_cudnn.py +++ b/flashinfer/gemm/gemm_bf16_fp4_cudnn.py @@ -15,6 +15,7 @@ OptimizationProfile, TunableRunner, TuningConfig, + is_in_profile_measurement, ) from ..fused_moe.utils import ( get_hybrid_num_tokens_buckets, @@ -505,6 +506,11 @@ def forward( tactic=tactic, ) except Exception as exc: + # During autotuner profiling, report the failed tactic as + # unsupported (marked inf) instead of silently timing + # tactic=-1 -- see the note in gemm_base.py (#3707 review). + if is_in_profile_measurement(): + raise warnings.warn( "cuDNN bf16-fp4 GEMM tactic failed; falling back to default " f"tactic=-1. ({exc})", diff --git a/tests/autotuner/test_cudnn_profiling_fallback.py b/tests/autotuner/test_cudnn_profiling_fallback.py new file mode 100644 index 00000000000..b42f72d7e53 --- /dev/null +++ b/tests/autotuner/test_cudnn_profiling_fallback.py @@ -0,0 +1,63 @@ +"""A cuDNN GEMM tactic that cannot be used must be reported as FAILED during +autotuner profiling (so the autotuner disqualifies it), while still falling +back to tactic=-1 outside profiling for serving robustness. + +Regression for the silent-fallback-during-profiling issue raised in the +#3707 review: otherwise the fallback's timing is attributed to the requested +tactic and a tactic that never ran can win the autotune. +""" + +import warnings + +import pytest +import torch + +from flashinfer.autotuner.autotuner import _profile_measurement_scope +from flashinfer.utils import get_compute_capability + + +def _fp8_inputs(m=8, n=4096, k=4096): + from tests.utils_fp8 import to_float8 + + a = torch.randn([1, m, k], device="cuda", dtype=torch.bfloat16) + b = torch.randn([1, n, k], device="cuda", dtype=torch.bfloat16).transpose(-2, -1) + a8, a_scale = to_float8(a) + b8, b_scale = to_float8(b) + out = torch.empty([1, m, n], device="cuda", dtype=torch.bfloat16) + ws = torch.empty(32 * 1024 * 1024, device="cuda", dtype=torch.uint8) + return [a8, b8, a_scale, b_scale, out, ws] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_cudnn_fp8_tactic_failure_raises_in_profiling_but_falls_back_serving(): + if get_compute_capability(torch.device("cuda"))[0] < 9: + pytest.skip("cuDNN FP8 GEMM requires SM90+") + from flashinfer.gemm.gemm_base import ( + _check_cudnn_availability, + _cudnn_gemm_fp8_runner, + ) + + try: + _check_cudnn_availability() + except Exception: + pytest.skip("cuDNN not available") + + runner = _cudnn_gemm_fp8_runner() + inputs = _fp8_inputs() + bad_tactic = 9999 # out-of-range plan index -> deterministic fallback path + + # Serving (not profiling): warn + fall back to -1 + produce a valid result. + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + runner.forward(inputs, tactic=bad_tactic) + assert any("falling back to default tactic=-1" in str(x.message) for x in w) + torch.cuda.synchronize() + assert torch.isfinite(inputs[4]).all() + + # Profiling: the failed tactic must raise so the autotuner marks it inf, + # rather than silently timing the fallback under the requested tactic. + with ( + pytest.raises(RuntimeError, match="during profiling"), + _profile_measurement_scope(), + ): + runner.forward(inputs, tactic=bad_tactic)