From 15e7bf6c2b8a0f3f6124153c6e4a4a83acc6a7a9 Mon Sep 17 00:00:00 2001 From: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:35:07 -0700 Subject: [PATCH 1/3] dflash2: greedy selector walk picks the first unbeaten candidate so an all-NaN score row stays in range (#105) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sglang/kernels/ops/speculative/dflash.py | 7 ++- .../test_dflash_selector_walk_nan_row.py | 63 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py diff --git a/python/sglang/kernels/ops/speculative/dflash.py b/python/sglang/kernels/ops/speculative/dflash.py index c86b894c05ad..c248a454c672 100644 --- a/python/sglang/kernels/ops/speculative/dflash.py +++ b/python/sglang/kernels/ops/speculative/dflash.py @@ -271,8 +271,11 @@ def _selector_walk_kernel( tl.float32 ) if greedy: - best = tl.max(scores, axis=0) - index = tl.min(tl.where(scores == best, offsets, top_k), axis=0) + # First candidate that no other candidate strictly beats. Total over + # any row (there is always at least one), so index < top_k and the + # row never resolves to a neighbour's candidates. + beaten = tl.sum((scores[None, :] > scores[:, None]).to(tl.int32), axis=1) + index = tl.min(tl.where(beaten == 0, offsets, top_k), axis=0) probabilities = tl.where(offsets == index, 1.0, 0.0) else: scaled = scores / temperature diff --git a/test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py b/test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py new file mode 100644 index 000000000000..c2644f46b2b8 --- /dev/null +++ b/test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py @@ -0,0 +1,63 @@ +import os + +os.environ.setdefault("TRITON_INTERPRET", "1") + +import torch + +from sglang.kernels.ops.speculative.dflash import selector_walk_triton +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=30, suite="base-a-test-cpu") + + +def _walk(scores, greedy): + batch, slots, top_k, _ = scores.shape + candidate_ids = torch.arange(batch * slots * top_k).view(batch, slots, top_k) + tokens, q_rows = selector_walk_triton( + candidate_ids=candidate_ids, + scores=scores, + uniforms=torch.full((batch, slots), 0.5), + temperatures=torch.ones(batch), + greedy_mask=torch.full((batch,), greedy, dtype=torch.bool), + ) + return candidate_ids, tokens, q_rows + + +class TestDFlashSelectorWalkNanRow(CustomTestCase): + def test_all_nan_greedy_row_stays_inside_its_candidate_row(self): + batch, slots, top_k = 2, 3, 4 + scores = torch.randn(batch, slots, top_k, top_k) + scores[0] = float("nan") + candidate_ids, tokens, q_rows = _walk(scores, greedy=True) + + for slot in range(slots): + self.assertIn(int(tokens[0, slot]), candidate_ids[0, slot].tolist()) + self.assertEqual(float(q_rows[0, slot].sum()), 1.0) + # Matches torch.argmax on the same rows (index 0 for all-NaN). + self.assertEqual(tokens[0].tolist(), candidate_ids[0, :, 0].tolist()) + + def test_finite_greedy_row_matches_torch_argmax(self): + batch, slots, top_k = 3, 4, 8 + scores = torch.randn(batch, slots, top_k, top_k) + scores[1, 2, :, :] = 0.0 # ties break to the left + candidate_ids, tokens, q_rows = _walk(scores, greedy=True) + + previous = torch.zeros(batch, dtype=torch.int64) + for slot in range(slots): + rows = scores[torch.arange(batch), slot, previous] + expected = rows.argmax(dim=-1) + self.assertEqual( + tokens[:, slot].tolist(), + candidate_ids[torch.arange(batch), slot, expected].tolist(), + ) + torch.testing.assert_close( + q_rows[:, slot], torch.nn.functional.one_hot(expected, top_k).float() + ) + previous = expected + + +if __name__ == "__main__": + import unittest + + unittest.main() From 03d047046d568226722c6302e105111c723e13f3 Mon Sep 17 00:00:00 2001 From: rdamani Date: Wed, 30 Sep 2026 01:41:17 +0000 Subject: [PATCH 2/3] test: run DFlash2 selector-walk NaN-row test on CUDA instead of the Triton interpreter Setting TRITON_INTERPRET inside the test module fails in CI's CPU suite once triton is already imported (tl.sum raises 'Cannot call @triton.jit'd outside of the scope of a kernel'). Move it next to the other speculative kernel tests and register it for base-b-kernel-unit on 1-gpu-large. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_dflash_selector_walk_nan_row.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) rename test/registered/{unit/kernels => kernels/ops/speculative}/test_dflash_selector_walk_nan_row.py (70%) diff --git a/test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py b/test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py similarity index 70% rename from test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py rename to test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py index c2644f46b2b8..e321cfec4ee2 100644 --- a/test/registered/unit/kernels/test_dflash_selector_walk_nan_row.py +++ b/test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py @@ -1,33 +1,35 @@ -import os - -os.environ.setdefault("TRITON_INTERPRET", "1") +import unittest import torch from sglang.kernels.ops.speculative.dflash import selector_walk_triton -from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=30, suite="base-a-test-cpu") +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") def _walk(scores, greedy): batch, slots, top_k, _ = scores.shape - candidate_ids = torch.arange(batch * slots * top_k).view(batch, slots, top_k) + device = scores.device + candidate_ids = torch.arange(batch * slots * top_k, device=device).view( + batch, slots, top_k + ) tokens, q_rows = selector_walk_triton( candidate_ids=candidate_ids, scores=scores, - uniforms=torch.full((batch, slots), 0.5), - temperatures=torch.ones(batch), - greedy_mask=torch.full((batch,), greedy, dtype=torch.bool), + uniforms=torch.full((batch, slots), 0.5, device=device), + temperatures=torch.ones(batch, device=device), + greedy_mask=torch.full((batch,), greedy, dtype=torch.bool, device=device), ) - return candidate_ids, tokens, q_rows + return candidate_ids.cpu(), tokens.cpu(), q_rows.cpu() +@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") class TestDFlashSelectorWalkNanRow(CustomTestCase): def test_all_nan_greedy_row_stays_inside_its_candidate_row(self): batch, slots, top_k = 2, 3, 4 - scores = torch.randn(batch, slots, top_k, top_k) + scores = torch.randn(batch, slots, top_k, top_k, device="cuda") scores[0] = float("nan") candidate_ids, tokens, q_rows = _walk(scores, greedy=True) @@ -39,9 +41,10 @@ def test_all_nan_greedy_row_stays_inside_its_candidate_row(self): def test_finite_greedy_row_matches_torch_argmax(self): batch, slots, top_k = 3, 4, 8 - scores = torch.randn(batch, slots, top_k, top_k) + scores = torch.randn(batch, slots, top_k, top_k, device="cuda") scores[1, 2, :, :] = 0.0 # ties break to the left candidate_ids, tokens, q_rows = _walk(scores, greedy=True) + scores = scores.cpu() previous = torch.zeros(batch, dtype=torch.int64) for slot in range(slots): @@ -58,6 +61,4 @@ def test_finite_greedy_row_matches_torch_argmax(self): if __name__ == "__main__": - import unittest - unittest.main() From c06d4b7f3e9efa4586c210ffc6516b3dd2066a68 Mon Sep 17 00:00:00 2001 From: rdamani Date: Mon, 5 Oct 2026 18:33:03 +0000 Subject: [PATCH 3/3] [DFlash2] Match torch.argmax NaN semantics in greedy selector walk Pick the first NaN if the row has one, else the first maximum, so mixed-NaN rows agree with _selector_walk_torch. Move NaN coverage into test_lilicorr_cuda.py as reference comparisons. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sglang/kernels/ops/speculative/dflash.py | 11 ++-- .../test_dflash_selector_walk_nan_row.py | 64 ------------------- .../ops/speculative/test_lilicorr_cuda.py | 23 +++++++ 3 files changed, 29 insertions(+), 69 deletions(-) delete mode 100644 test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py diff --git a/python/sglang/kernels/ops/speculative/dflash.py b/python/sglang/kernels/ops/speculative/dflash.py index c248a454c672..d5c7e33542f1 100644 --- a/python/sglang/kernels/ops/speculative/dflash.py +++ b/python/sglang/kernels/ops/speculative/dflash.py @@ -271,11 +271,12 @@ def _selector_walk_kernel( tl.float32 ) if greedy: - # First candidate that no other candidate strictly beats. Total over - # any row (there is always at least one), so index < top_k and the - # row never resolves to a neighbour's candidates. - beaten = tl.sum((scores[None, :] > scores[:, None]).to(tl.int32), axis=1) - index = tl.min(tl.where(beaten == 0, offsets, top_k), axis=0) + # Same pick as torch.argmax: the first NaN if any, else the first max. + is_nan = scores != scores + best = tl.max(tl.where(is_nan, -float("inf"), scores), axis=0) + has_nan = tl.max(is_nan.to(tl.int32), axis=0) > 0 + hit = tl.where(has_nan, is_nan, scores == best) + index = tl.min(tl.where(hit, offsets, top_k), axis=0) probabilities = tl.where(offsets == index, 1.0, 0.0) else: scaled = scores / temperature diff --git a/test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py b/test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py deleted file mode 100644 index e321cfec4ee2..000000000000 --- a/test/registered/kernels/ops/speculative/test_dflash_selector_walk_nan_row.py +++ /dev/null @@ -1,64 +0,0 @@ -import unittest - -import torch - -from sglang.kernels.ops.speculative.dflash import selector_walk_triton -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") - - -def _walk(scores, greedy): - batch, slots, top_k, _ = scores.shape - device = scores.device - candidate_ids = torch.arange(batch * slots * top_k, device=device).view( - batch, slots, top_k - ) - tokens, q_rows = selector_walk_triton( - candidate_ids=candidate_ids, - scores=scores, - uniforms=torch.full((batch, slots), 0.5, device=device), - temperatures=torch.ones(batch, device=device), - greedy_mask=torch.full((batch,), greedy, dtype=torch.bool, device=device), - ) - return candidate_ids.cpu(), tokens.cpu(), q_rows.cpu() - - -@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") -class TestDFlashSelectorWalkNanRow(CustomTestCase): - def test_all_nan_greedy_row_stays_inside_its_candidate_row(self): - batch, slots, top_k = 2, 3, 4 - scores = torch.randn(batch, slots, top_k, top_k, device="cuda") - scores[0] = float("nan") - candidate_ids, tokens, q_rows = _walk(scores, greedy=True) - - for slot in range(slots): - self.assertIn(int(tokens[0, slot]), candidate_ids[0, slot].tolist()) - self.assertEqual(float(q_rows[0, slot].sum()), 1.0) - # Matches torch.argmax on the same rows (index 0 for all-NaN). - self.assertEqual(tokens[0].tolist(), candidate_ids[0, :, 0].tolist()) - - def test_finite_greedy_row_matches_torch_argmax(self): - batch, slots, top_k = 3, 4, 8 - scores = torch.randn(batch, slots, top_k, top_k, device="cuda") - scores[1, 2, :, :] = 0.0 # ties break to the left - candidate_ids, tokens, q_rows = _walk(scores, greedy=True) - scores = scores.cpu() - - previous = torch.zeros(batch, dtype=torch.int64) - for slot in range(slots): - rows = scores[torch.arange(batch), slot, previous] - expected = rows.argmax(dim=-1) - self.assertEqual( - tokens[:, slot].tolist(), - candidate_ids[torch.arange(batch), slot, expected].tolist(), - ) - torch.testing.assert_close( - q_rows[:, slot], torch.nn.functional.one_hot(expected, top_k).float() - ) - previous = expected - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py b/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py index bbc7c7bedf1b..9e305bf096c0 100644 --- a/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py +++ b/test/registered/kernels/ops/speculative/test_lilicorr_cuda.py @@ -113,6 +113,29 @@ def test_greedy_path_breaks_ties_toward_the_lower_candidate_on_device(): assert actual[:, 0].cpu().equal(tokens[:, 0, 0].cpu()) +@pytest.mark.parametrize("nan_rows", ["all", "mixed"]) +def test_greedy_path_matches_the_reference_on_nan_scores(nan_rows): + torch.manual_seed(5) + bs, slots, topk = 3, 6, 8 + log_start = torch.randn(bs, topk, device="cuda") + log_pair = torch.randn(bs, slots - 1, topk, topk, device="cuda") + if nan_rows == "all": + log_start[0] = float("nan") + log_pair[0] = float("nan") + else: + log_pair[torch.rand_like(log_pair) < 0.3] = float("nan") + candidate_tokens = torch.arange(bs * slots * topk, device="cuda").view( + bs, slots, topk + ) + state = _greedy_state(bs, slots, candidate_tokens.device) + + tokens, q = lilicorr_sample_path(log_start, log_pair, candidate_tokens, **state) + ref_tokens, ref_q = _walk_reference(log_start, log_pair, candidate_tokens, **state) + + assert torch.equal(tokens.cpu(), ref_tokens) + assert torch.equal(q.cpu(), ref_q) + + def _sampled_inputs(bs, slots, k, *, seed=0): torch.manual_seed(seed) return dict(