diff --git a/docs/basic_usage/sampling_params.md b/docs/basic_usage/sampling_params.md index 23415f9af555..9508c049259f 100644 --- a/docs/basic_usage/sampling_params.md +++ b/docs/basic_usage/sampling_params.md @@ -62,7 +62,7 @@ python -m sglang.launch_server --model-path --sampling-defaults openai |--------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| | frequency_penalty | `float = 0.0` | Penalizes tokens based on their frequency in generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of penalization grows linearly with each appearance of a token. | | presence_penalty | `float = 0.0` | Penalizes tokens if they appeared in the generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of the penalization is constant if a token occurred. | -| repetition_penalty | `float = 1.0` | Scales the logits of previously generated tokens to discourage (values > 1) or encourage (values < 1) repetition. Valid range is `[0, 2]`; `1.0` leaves probabilities unchanged. | +| repetition_penalty | `float = 1.0` | Scales the logits of previously generated tokens to discourage (values > 1) or encourage (values < 1) repetition. Valid range is `(0, 2]`; `1.0` leaves probabilities unchanged. | | min_new_tokens | `int = 0` | Forces the model to generate at least `min_new_tokens` until a stop word or EOS token is sampled. Note that this might lead to unintended behavior, for example, if the distribution is highly skewed towards these tokens. | ### Constrained decoding diff --git a/python/sglang/srt/sampling/sampling_params.py b/python/sglang/srt/sampling/sampling_params.py index f84b0b0d5ebe..0744a57fdd82 100644 --- a/python/sglang/srt/sampling/sampling_params.py +++ b/python/sglang/srt/sampling/sampling_params.py @@ -139,10 +139,10 @@ def verify(self, vocab_size): raise ValueError( "presence_penalty must be in [-2, 2], got " f"{self.presence_penalty}." ) - if not 0.0 <= self.repetition_penalty <= 2.0: + if not 0.0 < self.repetition_penalty <= 2.0: raise ValueError( - "repetition_penalty must be in [0, 2], got " - f"{self.repetition_penalty}." + "repetition_penalty must be in (0, 2] (1.0 = no penalty), " + f"got {self.repetition_penalty}." ) if not 0 <= self.min_new_tokens: raise ValueError( diff --git a/test/registered/unit/sampling/test_sampling_params.py b/test/registered/unit/sampling/test_sampling_params.py index b849e6e852c5..67c51a1a2760 100644 --- a/test/registered/unit/sampling/test_sampling_params.py +++ b/test/registered/unit/sampling/test_sampling_params.py @@ -178,22 +178,36 @@ def test_presence_penalty_out_of_range_raises(self): # --- repetition_penalty --- def test_repetition_penalty_negative_raises(self): - """Test that verify() rejects negative repetition_penalty (valid range is [0, 2]).""" + """Test that verify() rejects negative repetition_penalty (valid range is (0, 2]).""" sp = self._make(repetition_penalty=-0.1) with self.assertRaises(ValueError): sp.verify(self.VOCAB_SIZE) + def test_repetition_penalty_zero_raises(self): + """Test that verify() rejects repetition_penalty=0. + + A value of 0 makes the sampling kernel divide logits by 0, producing + inf/NaN in the probability tensor and crashing every TP rank with a + device-side assert. + """ + sp = self._make(repetition_penalty=0.0) + with self.assertRaises(ValueError): + sp.verify(self.VOCAB_SIZE) + def test_repetition_penalty_above_two_raises(self): """Test that verify() rejects repetition_penalty > 2.0.""" sp = self._make(repetition_penalty=2.1) with self.assertRaises(ValueError): sp.verify(self.VOCAB_SIZE) - def test_repetition_penalty_boundaries_valid(self): - """Test that boundary values 0.0 and 2.0 are both accepted.""" - self._make(repetition_penalty=0.0).verify(self.VOCAB_SIZE) + def test_repetition_penalty_boundary_two_valid(self): + """Test that the upper boundary value 2.0 is accepted.""" self._make(repetition_penalty=2.0).verify(self.VOCAB_SIZE) + def test_repetition_penalty_small_positive_valid(self): + """Test that a small positive repetition_penalty (e.g. 1e-3) is accepted.""" + self._make(repetition_penalty=1e-3).verify(self.VOCAB_SIZE) + # --- min_new_tokens / max_new_tokens --- def test_negative_min_new_tokens_raises(self): """Test that verify() rejects negative min_new_tokens."""