Skip to content
7 changes: 5 additions & 2 deletions python/sglang/kernels/ops/speculative/dflash.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,8 +271,11 @@ def _selector_walk_kernel(
tl.float32
)
if greedy:
best = tl.max(scores, axis=0)
index = tl.min(tl.where(scores == best, offsets, top_k), axis=0)
# First candidate that no other candidate strictly beats. Total over
# any row (there is always at least one), so index < top_k and the
# row never resolves to a neighbour's candidates.
beaten = tl.sum((scores[None, :] > scores[:, None]).to(tl.int32), axis=1)
Comment thread
rodamani marked this conversation as resolved.
Resolved
index = tl.min(tl.where(beaten == 0, offsets, top_k), axis=0)
probabilities = tl.where(offsets == index, 1.0, 0.0)
else:
scaled = scores / temperature
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import unittest

import torch

from sglang.kernels.ops.speculative.dflash import selector_walk_triton
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase

register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large")


def _walk(scores, greedy):
batch, slots, top_k, _ = scores.shape
device = scores.device
candidate_ids = torch.arange(batch * slots * top_k, device=device).view(
batch, slots, top_k
)
tokens, q_rows = selector_walk_triton(
candidate_ids=candidate_ids,
scores=scores,
uniforms=torch.full((batch, slots), 0.5, device=device),
temperatures=torch.ones(batch, device=device),
greedy_mask=torch.full((batch,), greedy, dtype=torch.bool, device=device),
)
return candidate_ids.cpu(), tokens.cpu(), q_rows.cpu()


@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required")
class TestDFlashSelectorWalkNanRow(CustomTestCase):
def test_all_nan_greedy_row_stays_inside_its_candidate_row(self):
batch, slots, top_k = 2, 3, 4
scores = torch.randn(batch, slots, top_k, top_k, device="cuda")
scores[0] = float("nan")
candidate_ids, tokens, q_rows = _walk(scores, greedy=True)

for slot in range(slots):
self.assertIn(int(tokens[0, slot]), candidate_ids[0, slot].tolist())
self.assertEqual(float(q_rows[0, slot].sum()), 1.0)
Comment thread
rodamani marked this conversation as resolved.
Resolved
# Matches torch.argmax on the same rows (index 0 for all-NaN).
self.assertEqual(tokens[0].tolist(), candidate_ids[0, :, 0].tolist())

def test_finite_greedy_row_matches_torch_argmax(self):
batch, slots, top_k = 3, 4, 8
scores = torch.randn(batch, slots, top_k, top_k, device="cuda")
scores[1, 2, :, :] = 0.0 # ties break to the left
candidate_ids, tokens, q_rows = _walk(scores, greedy=True)
scores = scores.cpu()

previous = torch.zeros(batch, dtype=torch.int64)
for slot in range(slots):
rows = scores[torch.arange(batch), slot, previous]
expected = rows.argmax(dim=-1)
self.assertEqual(
tokens[:, slot].tolist(),
candidate_ids[torch.arange(batch), slot, expected].tolist(),
)
torch.testing.assert_close(
q_rows[:, slot], torch.nn.functional.one_hot(expected, top_k).float()
)
previous = expected


if __name__ == "__main__":
unittest.main()
Loading