From 02e4d11e0b9cd2317ab1ef0b783f9a06cbab2871 Mon Sep 17 00:00:00 2001 From: narutolhy <582909902@qq.com> Date: Wed, 3 Sep 2025 13:09:37 -0700 Subject: [PATCH 1/3] fix probs name which without temp scaling name --- python/sglang/srt/layers/sampler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index 56a831f2dafd..fb3c842bca2a 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -82,7 +82,7 @@ def forward( else: # Post process original logits. if temperatures are all 1.0, no need to rescale if return_logprob and RETURN_ORIGINAL_LOGPROB: - logprobs = torch.softmax(logits, dim=-1) + probs_without_temp_scaling = torch.softmax(logits, dim=-1) # Post process logits logits.div_(sampling_info.temperatures) @@ -123,8 +123,8 @@ def forward( if return_logprob: # clamp to avoid -inf if RETURN_ORIGINAL_LOGPROB: - logprobs = torch.log(logprobs).clamp( - min=torch.finfo(logprobs.dtype).min + logprobs = torch.log(probs_without_temp_scaling).clamp( + min=torch.finfo(probs_without_temp_scaling.dtype).min ) else: logprobs = torch.log(probs).clamp(min=torch.finfo(probs.dtype).min) From 36bbe99e57b41f24754edfd740b7db020a3f0cba Mon Sep 17 00:00:00 2001 From: narutolhy <582909902@qq.com> Date: Wed, 3 Sep 2025 13:51:02 -0700 Subject: [PATCH 2/3] edit comment --- python/sglang/srt/layers/sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index fb3c842bca2a..783124886669 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -80,7 +80,7 @@ def forward( logprobs = torch.nn.functional.log_softmax(logits, dim=-1) else: - # Post process original logits. if temperatures are all 1.0, no need to rescale + # If requested, cache probabilities from original logits before temperature scaling. if return_logprob and RETURN_ORIGINAL_LOGPROB: probs_without_temp_scaling = torch.softmax(logits, dim=-1) From 7b11ca5cc8f7e9bd2cc8b85de298b0839f378bb4 Mon Sep 17 00:00:00 2001 From: narutolhy <582909902@qq.com> Date: Mon, 8 Sep 2025 23:11:02 -0700 Subject: [PATCH 3/3] delete probs_without_temp_scaling after usage --- python/sglang/srt/layers/sampler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index 783124886669..565ce106e883 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -126,6 +126,7 @@ def forward( logprobs = torch.log(probs_without_temp_scaling).clamp( min=torch.finfo(probs_without_temp_scaling.dtype).min ) + del probs_without_temp_scaling else: logprobs = torch.log(probs).clamp(min=torch.finfo(probs.dtype).min)