diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx
index 0ccb28dbd10b..78681d399d9b 100644
--- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx
+++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx
@@ -777,6 +777,37 @@ Tool Call: None
Arguments: }
```
+#### 4.3.3 Semantic Decisions
+
+Qwen3.5-35B-A3B can answer typed questions about an input with a probability for every option through `/v1/decisions`, without generating text. The server turns thinking off for these requests, so the answer is read after a closed think block, and the same server keeps serving chat traffic. No launch flag is needed. Until a release contains `/v1/decisions`, install a [nightly build](/docs/get-started/install#nightly-builds). With a server started by `sglang serve --model-path Qwen/Qwen3.5-35B-A3B --host 127.0.0.1 --port 30000`:
+
+```python Example
+import requests
+
+response = requests.post(
+ "http://127.0.0.1:30000/v1/decisions",
+ json={
+ "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.",
+ "questions": [
+ {
+ "id": "team",
+ "type": "choice",
+ "question": "Which team should handle this ticket?",
+ "options": [{"name": "billing"}, {"name": "technical"}, {"name": "sales"}],
+ },
+ {"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."},
+ ],
+ },
+ timeout=60,
+)
+response.raise_for_status()
+answers = response.json()["answers"]
+print(answers["team"]["choice"], answers["team"]["probabilities"])
+print(answers["urgent"]["probabilities"]["yes"])
+```
+
+See [Decision models](/docs/supported-models/decision_models) for the request reference, how the probabilities are computed, and the limits.
+
## 5. Benchmark
### 5.1 Accuracy Benchmark
diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
index 30cd5eac5e53..e99d6986f347 100644
--- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
+++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
@@ -541,3 +541,34 @@ providers:
```
+
+## 4. Semantic Decisions
+
+Qwen3.8-27B can answer typed questions about an input with a probability for every option through `/v1/decisions`, without generating text. The server turns thinking off for these requests, so the answer is read after a closed think block, and the same server keeps serving chat traffic. No launch flag is needed. Until a release contains `/v1/decisions`, install a [nightly build](/docs/get-started/install#nightly-builds). With a server started by `sglang serve --model-path Qwen/Qwen3.8-27B --host 127.0.0.1 --port 30000`:
+
+```python Example
+import requests
+
+response = requests.post(
+ "http://127.0.0.1:30000/v1/decisions",
+ json={
+ "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.",
+ "questions": [
+ {
+ "id": "team",
+ "type": "choice",
+ "question": "Which team should handle this ticket?",
+ "options": [{"name": "billing"}, {"name": "technical"}, {"name": "sales"}],
+ },
+ {"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."},
+ ],
+ },
+ timeout=60,
+)
+response.raise_for_status()
+answers = response.json()["answers"]
+print(answers["team"]["choice"], answers["team"]["probabilities"])
+print(answers["urgent"]["probabilities"]["yes"])
+```
+
+See [Decision models](/docs/supported-models/decision_models) for the request reference, how the probabilities are computed, and the limits.
diff --git a/docs/docs.json b/docs/docs.json
index 0cf88a03b152..fc942d10650b 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -998,7 +998,8 @@
{
"group": "Specialized Models",
"pages": [
- "docs/supported-models/reward_models"
+ "docs/supported-models/reward_models",
+ "docs/supported-models/decision_models"
]
},
{
diff --git a/docs/docs/basic_usage/native_api.mdx b/docs/docs/basic_usage/native_api.mdx
index eff2293166d9..1673efc5b5d3 100644
--- a/docs/docs/basic_usage/native_api.mdx
+++ b/docs/docs/basic_usage/native_api.mdx
@@ -15,6 +15,7 @@ Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its nati
- `/encode`(embedding model)
- `/v1/rerank`(cross encoder rerank model)
- `/v1/score`(decoder-only scoring)
+- `/v1/decisions`(typed decisions on a chat model)
- `/classify`(reward model)
- `/start_expert_distribution_record`
- `/stop_expert_distribution_record`
@@ -256,6 +257,8 @@ Parameters:
- `item_first`: Whether items come first in concatenation order (default: False)
- `model`: Model name
+`query` and `items` also accept token ids, with `query` set to `[]` when each item is a complete prompt. `label_token_ids` can be one list per item. `temperature` divides the label logits before the softmax and needs `apply_softmax`, and `return_token_logprobs` adds `token_logprobs`, the full-vocabulary log-probabilities of the labels.
+
The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`.
```python Example
@@ -293,6 +296,20 @@ for item, scores in zip(items, response_json["scores"]):
terminate_process(score_process)
```
+## v1/decisions (typed decisions on a chat model)
+
+Answer typed questions about an input with per-option probabilities, without generating. The server renders each question with the model's chat template and thinking turned off, assigns one-token answer labels, and scores them through the same path as `v1/score`.
+
+Parameters:
+- `input`: Text, object, or array the questions are about
+- `questions`: List of questions, each with a unique `id` and a `type` of `choice` (with `options`), `score` (with `levels`), or `yes_no`
+- `temperature`: Divides the label logits before the softmax (default: 1)
+- `chat_template_kwargs`: Extra chat template arguments
+- `prompt_format_version`: Optional pin of the server-owned prompt wording
+- `return_prompt_token_ids`: Return the scored prompt and label token ids for replay through `v1/score` (default: False)
+
+The response contains `answers` keyed by question id, each with `type`, `probabilities`, and `label_mass`, plus `choice` for a choice question or `score` for a score question. A yes or no answer is read from `probabilities["yes"]`. See [Decision models](/docs/supported-models/decision_models) for examples, the full reference, and error cases.
+
## Classify (reward model)
SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations.
diff --git a/docs/docs/supported-models.mdx b/docs/docs/supported-models.mdx
index bcfcf89a75ca..33ae19614b5b 100644
--- a/docs/docs/supported-models.mdx
+++ b/docs/docs/supported-models.mdx
@@ -84,4 +84,13 @@ SGLang supports model families across text generation, retrieval, and reward wor
>
RLHF and reward scoring pipelines optimized for production latency.
+
+ Typed choice, score, and yes or no answers with a probability for every option.
+
diff --git a/docs/docs/supported-models/decision_models.mdx b/docs/docs/supported-models/decision_models.mdx
new file mode 100644
index 000000000000..122e5876d714
--- /dev/null
+++ b/docs/docs/supported-models/decision_models.mdx
@@ -0,0 +1,227 @@
+---
+title: Decision models
+description: Answer typed choice, score, and yes or no questions with a probability for every option from a chat model, without generating text.
+---
+
+`/v1/decisions` turns a chat model into a decision model. You send an input and a list of typed questions, and each answer comes back with the probability of every option, read from the model's next-token scores at the answer position. No text is generated and no output is parsed. It needs no special checkpoint: any generation model served with a Jinja chat template can answer, as long as the checks below pass for it.
+
+
+ `/v1/decisions` is an SGLang extension under `/v1`, like `/v1/score` and `/v1/rerank`. It is not part of the OpenAI API, so call it over HTTP rather than through an OpenAI SDK method. Until a release contains it, install a [nightly build](/docs/get-started/install#nightly-builds), which is built from the main branch.
+
+
+## Supported models
+
+| Model | Example HuggingFace identifier | Notes |
+| --- | --- | --- |
+| **Qwen3.8-27B** | `Qwen/Qwen3.8-27B` | Validated on one H200 in BF16, with and without NEXTN speculative decoding. See the [model page](/cookbook/autoregressive/Qwen/Qwen3.8-27B). |
+| **Qwen3.5-35B-A3B** | `Qwen/Qwen3.5-35B-A3B` | Validated on one H200 in BF16. See the [model page](/cookbook/autoregressive/Qwen/Qwen3.5). |
+
+Other chat models are served when the answer labels are single tokens at the answer position and the answer does not start inside a reasoning block. The server checks these per request and returns a 400 that names the reason when one fails, see [Errors](#errors). It does not check how the chat template renders the question or where the template itself would put an answer, so inspect `prompt_token_ids` from `return_prompt_token_ids` before relying on a new model.
+
+## Launch command
+
+```bash Command
+python -m sglang.launch_server \
+ --model-path Qwen/Qwen3.8-27B \
+ --host 127.0.0.1 --port 30000
+```
+
+No extra flag is needed. The same server keeps serving `/v1/chat/completions` and `/generate`, and decisions can run alongside that traffic. A server launched with `--enable-mis` or `--dllm-algorithm`, or one that uses a built-in conversation template instead of the tokenizer's Jinja template, refuses decisions.
+
+## Example request
+
+```python Example
+import requests
+
+URL = "http://127.0.0.1:30000"
+
+response = requests.post(
+ f"{URL}/v1/decisions",
+ json={
+ "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales.",
+ "questions": [
+ {
+ "id": "team",
+ "type": "choice",
+ "question": "Which team should handle this ticket?",
+ "options": [
+ {"name": "billing", "description": "Payment or subscription issues"},
+ {"name": "technical", "description": "Bugs or integration problems"},
+ {"name": "sales", "description": "Pricing or account questions"},
+ ],
+ },
+ {
+ "id": "frustration",
+ "type": "score",
+ "question": "How frustrated is the customer?",
+ "levels": ["Calm", "Frustrated but civil", "Very angry"],
+ },
+ {
+ "id": "urgent",
+ "type": "yes_no",
+ "question": "The customer needs an answer today.",
+ },
+ ],
+ },
+ timeout=60,
+)
+response.raise_for_status()
+for question_id, answer in response.json()["answers"].items():
+ print(question_id, answer)
+```
+
+Each answer contains:
+
+- `type`: the question type, `choice`, `score`, or `yes_no`.
+- `probabilities`: one value per option name, level index, or `yes` and `no`, summing to 1. For a yes or no question, `probabilities["yes"]` is the answer.
+- `choice` for a choice question (the most probable option) or `score` for a score question (the probability-weighted mean level index).
+- `label_mass`: the full-vocabulary probability of the answer labels at the answer position. A low value means the model puts most of its probability outside the offered answers.
+
+Questions are answered independently and scored together in one batch. To route the ticket in this example, send it to the chosen team, and hand it to a person when the top probability falls below a threshold you validate on your own labeled tickets.
+
+## How answers are computed
+
+The server renders each question as one user message with the model's chat template and thinking turned off, and labels the answers `A` to `Z` for options, `0` to `9` for levels, and `yes` and `no`. It checks that each label is one distinct token at the answer position, then runs one prefill pass per question through the scoring path of `/v1/score` and reads the next-token log-probabilities of the labels over the full vocabulary. For a yes or no question with log-probabilities `lp_yes` and `lp_no` and request temperature `T`:
+
+- `probabilities["yes"] = exp(lp_yes / T) / (exp(lp_yes / T) + exp(lp_no / T))`, which equals a softmax of the two label logits divided by `T`, because the vocabulary normalizer cancels.
+- `label_mass = exp(lp_yes) + exp(lp_no)`, which does not depend on `T`.
+
+Choice and score questions work the same way over their labels. None of these values is a calibrated probability that the decision is correct. Validate any threshold on labeled data from your workload.
+
+## Request and response reference
+
+Request fields:
+
+| Field | Type | Meaning |
+| --- | --- | --- |
+| `input` | string, object, or array | What the questions are about. Objects and arrays are rendered as compact JSON. Must not be blank. |
+| `questions` | list of questions | At least one. Each question has a unique `id`, which keys its answer. |
+| `temperature` | number above 0, default 1 | Divides the label logits before the softmax over the labels. It does not change `label_mass`. |
+| `chat_template_kwargs` | object, default empty | Extra chat template arguments, applied after `--default-chat-template-kwargs`. The thinking toggle that the chat template or its reasoning parser names stays off in either case. |
+| `prompt_format_version` | integer, optional | Pins the prompt wording. A server that serves another version returns 400. |
+| `return_prompt_token_ids` | boolean, default false | Adds `prompt_token_ids` and `label_token_ids` to each answer. |
+| `model` | string, optional | Echoed in the response, as `default` when omitted. It does not select a LoRA adapter, and the `base:adapter` form returns 400. |
+
+Question fields:
+
+| Type | Fields | Labels |
+| --- | --- | --- |
+| `choice` | `id`, `question`, and `options`: 2 to 26 objects with a `name` and an optional `description` | `A` to `Z` in list order |
+| `score` | `id`, `question`, and `levels`: 2 to 10 level descriptions, lowest first | `0` to `9` |
+| `yes_no` | `id`, `question`, and optional `yes` and `no` descriptions | `yes` and `no` |
+
+`question`, descriptions, and levels can also be JSON objects or arrays. Each question becomes one user message: the input, a blank line, the question line, one line per option, level, or described yes or no answer, and a closing instruction to answer with one label only. An option renders as `A: billing` or `A: billing - `, a level as `0: `, and a described yes or no answer as `yes: `.
+
+The response has `object` set to `decisions`, `model`, `prompt_format_version`, `answers` keyed by question id, and `usage` with `prompt_tokens` for all questions and `completion_tokens` 0.
+
+## Errors
+
+The request fails with HTTP 400 for:
+
+- an unknown question type or an unknown field anywhere in the request
+- option or level counts outside the ranges above
+- a blank input, question, question id, or level
+- a repeated question id, or option names that are blank, contain control or line break characters, or repeat another name after trimming and case folding
+- a label that is not one distinct token at the answer position for the served tokenizer
+- a prompt that does not fit the model's context length
+- a chat template that always thinks before answering or starts every answer with a reasoning block, `chat_template_kwargs` that set the thinking toggle to anything but `false`, a rendered prompt that leaves a reasoning block open, or a model whose reasoning parser expects answers to start inside a reasoning block when the rendered prompt does not close one
+- a `prompt_format_version` other than the served one, or a `model` that names a LoRA adapter
+- a server launched with `--enable-mis` or `--dllm-algorithm`, or one that uses a built-in conversation template, whether named with `--chat-template`, loaded from a JSON template file, or inferred from the model path
+- a model whose chat route uses a built-in encoder instead of a chat template, a tokenizer that does not encode the rendered chat text back to the same ids, or a model that is not a generation model
+- a chat template that raises an error for the question message, or a server launched with `--skip-tokenizer-init`
+
+Errors about one question name its id or its position in `questions`.
+
+## Pin and replay the prompt
+
+The server owns the prompt wording of `/v1/decisions` and versions it. Every response carries `prompt_format_version`, and a change to the wording ships as a new version. Send `prompt_format_version` to fail loudly instead of receiving answers from different wording after a server upgrade.
+
+To keep an answer reproducible independently of the server's wording, ask for the scored ids and replay them through `/v1/score`, which only scores the ids you send:
+
+```python Example
+import requests
+
+URL = "http://127.0.0.1:30000"
+
+body = {
+ "input": "The integration keeps failing.",
+ "questions": [{"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."}],
+ "prompt_format_version": 1,
+ "return_prompt_token_ids": True,
+}
+answer = requests.post(f"{URL}/v1/decisions", json=body, timeout=60).json()["answers"]["urgent"]
+replay = requests.post(
+ f"{URL}/v1/score",
+ json={
+ "query": [],
+ "items": [answer["prompt_token_ids"]],
+ "label_token_ids": [answer["label_token_ids"]],
+ "apply_softmax": True,
+ "return_token_logprobs": True,
+ },
+ timeout=60,
+).json()
+print(answer["probabilities"], replay["scores"][0])
+```
+
+The replayed scores equal the answer's probabilities when both requests use the same `temperature` and run in the same cache state. If the decision set `temperature`, send the same value in the `/v1/score` body. A replay right after the decision reuses its cached prefix, so it can differ slightly, as described in [Reproducibility](#reproducibility).
+
+## Thinking models
+
+Qwen3.8-27B and Qwen3.5-35B-A3B think by default. `/v1/decisions` turns thinking off for every question, refuses a request that turns it back on, and refuses a rendered prompt that leaves a reasoning block open, so the answer is never read inside the reasoning. Chat requests to the same server keep the model's default, and launching with `--reasoning-parser qwen3` affects chat only.
+
+## Reproducibility
+
+These models mix full and linear attention. Answer probabilities can move by up to several hundredths (0.07 in our checks), and `label_mass` by up to about 0.14, between cold and prefix-cached requests and across batch compositions, while the chosen option stayed the same in our checks. On Qwen3.5-35B-A3B, `--disable-radix-cache` gave identical values across sequential repeats, at the cost of prefix reuse.
+
+## Build decisions without `/v1/decisions`
+
+When you need your own prompt or labels, build the prompt on the client and score it through `/v1/score`, which takes token ids and one label list per item. This needs `transformers` and `jinja2` on the client. Every label must be exactly one token at the answer position:
+
+```python Example
+import requests
+from transformers import AutoTokenizer
+
+MODEL = "Qwen/Qwen3.8-27B"
+URL = "http://127.0.0.1:30000"
+tokenizer = AutoTokenizer.from_pretrained(MODEL)
+
+options = ["billing", "technical", "sales"]
+letters = ["A", "B", "C"]
+prompt = (
+ "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.\n\n"
+ "Question: Which team should handle this ticket?\n"
+ + "\n".join(f"{letter}: {name}" for letter, name in zip(letters, options))
+ + "\nAnswer with the letter of one option only."
+)
+text = tokenizer.apply_chat_template(
+ [{"role": "user", "content": prompt}], add_generation_prompt=True, enable_thinking=False, tokenize=False
+)
+input_ids = tokenizer.encode(text, add_special_tokens=False)
+label_ids = []
+for letter in letters:
+ ids = tokenizer.encode(text + letter, add_special_tokens=False)
+ if ids[: len(input_ids)] != input_ids or len(ids) != len(input_ids) + 1:
+ raise ValueError(f"Label {letter} is not one token at the answer position")
+ label_ids.append(ids[-1])
+
+scores = requests.post(
+ f"{URL}/v1/score",
+ json={"query": [], "items": [input_ids], "label_token_ids": [label_ids], "apply_softmax": True},
+ timeout=60,
+).json()["scores"][0]
+print(dict(zip(options, scores)))
+```
+
+`enable_thinking=False` renders a closed think block, so the answer token follows it directly. Do not score a prompt that leaves the think block open, because the scored position then falls inside the reasoning.
+
+Structured output gives the same decision through generation. A one-token regex such as `(A|B|C)` with `temperature` 0 and logprobs returns probabilities renormalized over the labels, equal to the `/v1/score` values for one-character labels in the same cache state. A JSON schema with `enum` fields fills several fields in one response, but gives no clean per-option probability, because an enum value can span several tokens. See [Structured outputs](/docs/advanced_features/structured_outputs) for the request formats.
+
+## Limitations
+
+- Labels must be one token at the answer position. `/v1/decisions` assigns one-token labels for you, and a client-built prompt must do the same.
+- For `yes_no`, `label_mass` counts only the lowercase `yes` and `no` tokens. The model also puts probability on `Yes` and `No`, so this value reads lower than for choice and score questions even on clear cases, while `probabilities["yes"]` is unaffected.
+- The server tells from the chat template and its reasoning parser whether answers could start inside a reasoning block, using the parser's reasoning tags in the generation prompt and in the template's own rendering of a finished answer. Reasoning that shows neither is not detected, so check `prompt_token_ids` for such a model. Some templates of models that do not reason still contain reasoning tags without ever closing a block in the generation prompt, and those models are refused.
+- A default temperature in `--preferred-sampling-params` can reach scoring requests and scale both the probabilities and `label_mass` when a decision shares a batch with generation or runs under speculative decoding. Leave it unset for decisions.
+- The server tokenizes each question's full prompt once on the HTTP event loop, as `/v1/score` does for its items, and lets other requests run between questions. Tokenizers whose label check falls back to the full prompt encode it once more per label.
+- A decision takes a single `input`, not a chat history. To decide about a conversation, pass the history as the input text or as a JSON array.
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index ec0cacf01a7c..5045e033d2ba 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -88,6 +88,7 @@
ChatCompletionRequest,
ClassifyRequest,
CompletionRequest,
+ DecisionRequest,
DetokenizeRequest,
EmbeddingRequest,
ErrorResponse,
@@ -100,6 +101,7 @@
)
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion
+from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
from sglang.srt.entrypoints.openai.serving_rerank import OpenAIServingRerank
from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore
@@ -323,6 +325,9 @@ async def lifespan(fast_api_app: FastAPI):
fast_api_app.state.openai_serving_score = OpenAIServingScore(
_global_state.tokenizer_manager
)
+ fast_api_app.state.openai_serving_decisions = OpenAIServingDecisions(
+ fast_api_app.state.openai_serving_chat
+ )
fast_api_app.state.openai_serving_rerank = OpenAIServingRerank(
_global_state.tokenizer_manager, _global_state.template_manager
)
@@ -1933,6 +1938,14 @@ async def v1_score_request(request: ScoringRequest, raw_request: Request):
)
+@app.post("/v1/decisions", dependencies=[Depends(validate_json_request)])
+async def v1_decisions_request(request: DecisionRequest, raw_request: Request):
+ """Answer typed choice, score, and yes or no questions about an input by scoring single-token answer labels through the scoring API, without generation."""
+ return await raw_request.app.state.openai_serving_decisions.handle_request(
+ request, raw_request
+ )
+
+
@app.post("/v1/responses", dependencies=[Depends(validate_json_request)])
async def v1_responses_request(request: ResponsesRequest, raw_request: Request):
"""Endpoint for the responses API with reasoning support."""
diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py
index 243b40e6ddba..2b197c1d3554 100644
--- a/python/sglang/srt/entrypoints/openai/protocol.py
+++ b/python/sglang/srt/entrypoints/openai/protocol.py
@@ -17,6 +17,7 @@
import logging
import time
+import unicodedata
import uuid
from dataclasses import dataclass
from typing import (
@@ -1420,6 +1421,128 @@ class ScoringResponse(BaseModel):
object: str = "scoring"
+def _nonblank_decision_text(value):
+ if not (value.strip() if isinstance(value, str) else value):
+ raise ValueError("must not be blank")
+ return value
+
+
+# Objects and arrays are rendered into the prompt as compact JSON.
+DecisionText = Union[str, Dict[str, Any], List[Any]]
+RequiredDecisionText = Annotated[DecisionText, AfterValidator(_nonblank_decision_text)]
+
+
+class DecisionOption(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ name: str
+ description: Optional[DecisionText] = None
+
+
+class DecisionChoiceQuestion(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: Annotated[str, AfterValidator(_nonblank_decision_text)]
+ type: Literal["choice"]
+ question: RequiredDecisionText
+ # Options are labeled A to Z in order, so at most 26.
+ options: List[DecisionOption] = Field(min_length=2, max_length=26)
+
+ @model_validator(mode="after")
+ def _option_names_distinct(self):
+ seen = set()
+ for option in self.options:
+ key = option.name.strip().casefold()
+ if not key:
+ raise ValueError(f"question {self.id!r}: option names must be nonempty")
+ # Each option is rendered as one prompt line.
+ if any(unicodedata.category(c) in ("Cc", "Zl", "Zp") for c in option.name):
+ raise ValueError(
+ f"question {self.id!r}: option name {option.name!r} must not "
+ "contain control or line break characters"
+ )
+ if key in seen:
+ raise ValueError(
+ f"question {self.id!r}: option name {option.name!r} repeats "
+ "another option"
+ )
+ seen.add(key)
+ return self
+
+
+class DecisionScoreQuestion(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: Annotated[str, AfterValidator(_nonblank_decision_text)]
+ type: Literal["score"]
+ question: RequiredDecisionText
+ # Levels are labeled 0 to 9 in order, so at most 10.
+ levels: List[RequiredDecisionText] = Field(min_length=2, max_length=10)
+
+
+class DecisionYesNoQuestion(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: Annotated[str, AfterValidator(_nonblank_decision_text)]
+ type: Literal["yes_no"]
+ question: RequiredDecisionText
+ yes: Optional[DecisionText] = None
+ no: Optional[DecisionText] = None
+
+
+DecisionQuestion = Annotated[
+ Union[DecisionChoiceQuestion, DecisionScoreQuestion, DecisionYesNoQuestion],
+ Field(discriminator="type"),
+]
+
+
+class DecisionRequest(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ input: RequiredDecisionText
+ questions: List[DecisionQuestion] = Field(min_length=1)
+ # Scales option probabilities only, not label_mass.
+ temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
+ # Applied over the server defaults, with the template reasoning toggle off.
+ chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
+ # Pins the server-owned prompt wording. A different served version is refused.
+ prompt_format_version: Optional[int] = None
+ return_prompt_token_ids: bool = False
+ model: str = DEFAULT_MODEL_NAME
+
+ @field_validator("questions")
+ @classmethod
+ def _question_ids_distinct(cls, questions):
+ seen = set()
+ for question in questions:
+ if question.id in seen:
+ raise ValueError(
+ f"question id {question.id!r} repeats another question"
+ )
+ seen.add(question.id)
+ return questions
+
+
+class DecisionAnswer(BaseModel):
+ type: Literal["choice", "score", "yes_no"]
+ probabilities: Dict[str, float]
+ # Full-vocabulary probability of all answer labels at the answer position.
+ label_mass: float
+ choice: Optional[str] = None
+ score: Optional[float] = None
+ # The exact /v1/score inputs, with return_prompt_token_ids.
+ prompt_token_ids: Optional[List[int]] = None
+ label_token_ids: Optional[List[int]] = None
+
+
+class DecisionResponse(BaseModel):
+ object: str = "decisions"
+ model: str
+ prompt_format_version: int
+ answers: Dict[str, DecisionAnswer]
+ usage: UsageInfo
+
+
class V1RerankReqInput(BaseModel):
query: RerankContent = Field(
...,
@@ -1545,6 +1668,7 @@ class DetokenizeResponse(BaseModel):
EmbeddingRequest,
ClassifyRequest,
ScoringRequest,
+ DecisionRequest,
V1RerankReqInput,
TokenizeRequest,
DetokenizeRequest,
diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py
index 92842acff2b7..38fd20470e3e 100644
--- a/python/sglang/srt/entrypoints/openai/serving_chat.py
+++ b/python/sglang/srt/entrypoints/openai/serving_chat.py
@@ -2445,7 +2445,7 @@ def _process_response_logprobs(self, ret_item: dict[str, Any]) -> ChoiceLogprobs
output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"]
output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None)
token_logprobs = self._build_token_logprobs_from_raw(
- output_token_logprobs, output_top_logprobs, use_token_index=True
+ output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
@@ -2453,7 +2453,6 @@ def _build_token_logprobs_from_raw(
self,
output_token_logprobs: list[Any],
output_top_logprobs: list[Any] | None,
- use_token_index: bool = False,
) -> list[ChatCompletionTokenLogprob]:
"""Build OpenAI ChatCompletionTokenLogprob from the engine's raw
``(logprob, token_id, token_text)`` triples.
@@ -2487,13 +2486,9 @@ def _build_token_logprobs_from_raw(
top_logprobs: list[TopLogprob] = []
if output_top_logprobs:
- # - Non-streaming (use_token_index=True): output_top_logprobs is
- # the full per-position list; take the row for this token.
- # - Streaming (use_token_index=False): rows are pre-sliced so the
- # current chunk holds exactly one row at index 0.
- top_row_idx = token_idx if use_token_index else 0
- if top_row_idx < len(output_top_logprobs):
- top_row = output_top_logprobs[top_row_idx]
+ # Both callers pass rows aligned with output_token_logprobs.
+ if token_idx < len(output_top_logprobs):
+ top_row = output_top_logprobs[token_idx]
if top_row is not None:
for top_logprob, top_id, top_text in top_row:
if is_byte_level:
@@ -2686,7 +2681,7 @@ def _process_streaming_logprobs(
n_prev_token:total_output_logprobs
]
token_logprobs = self._build_token_logprobs_from_raw(
- output_token_logprobs, output_top_logprobs, use_token_index=False
+ output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
diff --git a/python/sglang/srt/entrypoints/openai/serving_decisions.py b/python/sglang/srt/entrypoints/openai/serving_decisions.py
new file mode 100644
index 000000000000..bfd8c0be9249
--- /dev/null
+++ b/python/sglang/srt/entrypoints/openai/serving_decisions.py
@@ -0,0 +1,428 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import math
+import string
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
+
+from fastapi import Request
+from fastapi.responses import ORJSONResponse
+from transformers import PreTrainedTokenizerBase
+
+from sglang.srt.entrypoints.openai.protocol import (
+ DecisionAnswer,
+ DecisionChoiceQuestion,
+ DecisionQuestion,
+ DecisionRequest,
+ DecisionResponse,
+ DecisionScoreQuestion,
+ DecisionText,
+ UsageInfo,
+)
+from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
+from sglang.srt.entrypoints.openai.serving_chat import _CHAT_TEMPLATE_CLIENT_ERRORS
+from sglang.srt.parser.reasoning_parser import ReasoningParser
+from sglang.srt.runtime_context import get_exec
+
+if TYPE_CHECKING:
+ from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
+
+logger = logging.getLogger(__name__)
+
+# Version of the server-owned prompt wording and answer labels.
+# Any change to _render_question or _answer_labels needs a new version.
+PROMPT_FORMAT_VERSION = 1
+
+# Parser defaults that name the chat template kwarg toggling reasoning.
+_PARSER_TOGGLE_MODES = (
+ "thinking",
+ "enable_thinking",
+ "explicit_thinking",
+ "explicit_enable_thinking",
+)
+# Answer text of a finished reply, rendered only to see what precedes an answer.
+_REPLY_SENTINEL = "DECISION_ANSWER"
+
+
+class OpenAIServingDecisions(OpenAIServingBase):
+ """Handler for /v1/decisions requests, answered by candidate scoring without generation"""
+
+ def __init__(self, chat_serving: OpenAIServingChat):
+ super().__init__(chat_serving.tokenizer_manager)
+ # Render the way the chat route does, and refuse where it renders differently.
+ self.template_manager = chat_serving.template_manager
+ self.default_chat_template_kwargs = chat_serving.default_chat_template_kwargs
+ self.chat_encoding_spec = chat_serving.chat_encoding_spec
+ self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
+ tokenizer = self.tokenizer_manager.tokenizer
+ # Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
+ self.added_tokens = (
+ {i: token for token, i in tokenizer.get_added_vocab().items()}
+ if isinstance(tokenizer, PreTrainedTokenizerBase)
+ else {}
+ )
+ # The configured reasoning parser, or the one the chat template suggests,
+ # tells where reasoning blocks start and end and whether answers open one.
+ parser = (
+ chat_serving.reasoning_parser
+ or self.template_manager.suggested_reasoning_parser
+ )
+ self.reasoning_markers = None
+ self.answers_open_reasoning = False
+ # The kwarg that turns reasoning on and off: the template's, else, when
+ # detection finds no config, the one the configured or suggested parser names.
+ config = self.template_manager.reasoning_config
+ self.reasoning_toggle = config.toggle_param if config is not None else None
+ if parser is not None:
+ try:
+ detector = ReasoningParser(
+ model_type=parser, tokenizer=tokenizer
+ ).detector
+ except ValueError as e:
+ logger.warning(
+ "No reasoning block check for /v1/decisions with parser '%s': %s",
+ parser,
+ e,
+ )
+ else:
+ mode = detector.reasoning_default
+ if config is None and mode in _PARSER_TOGGLE_MODES:
+ self.reasoning_toggle = mode.removeprefix("explicit_")
+ if detector.think_start_token and detector.think_end_token:
+ self.reasoning_markers = (
+ detector.think_start_token,
+ detector.think_end_token,
+ )
+ self.answers_open_reasoning = detector.reasoning_default == "always"
+
+ def _request_id_prefix(self) -> str:
+ return "decision-"
+
+ def _validate_request(self, request: DecisionRequest) -> Optional[str]:
+ if not self.tokenizer_manager.is_generation:
+ return "/v1/decisions requires a generation model"
+ if self.tokenizer_manager.tokenizer is None:
+ return "/v1/decisions requires the server tokenizer"
+ if self.chat_encoding_spec is not None:
+ return (
+ "/v1/decisions requires a chat template, but this model's chat "
+ f"route uses the {self.chat_encoding_spec!r} encoder"
+ )
+ if self.prompt_text_is_lossy:
+ return (
+ "/v1/decisions places answer labels on the rendered chat text, "
+ "which this tokenizer does not encode back to the same ids"
+ )
+ if self.template_manager.chat_template_name is not None:
+ return (
+ "/v1/decisions renders the tokenizer's Jinja chat template, but "
+ "this server uses the built-in chat template "
+ f"{self.template_manager.chat_template_name!r}"
+ )
+ if get_exec().features.enable_mis:
+ return "/v1/decisions does not support --enable-mis"
+ if get_exec().dllm.dllm_algorithm is not None:
+ return (
+ "/v1/decisions does not support diffusion language models "
+ "served with --dllm-algorithm"
+ )
+ _, adapter = self._parse_model_parameter(request.model)
+ if adapter is not None:
+ return (
+ f"model names the LoRA adapter {adapter!r}, which /v1/decisions "
+ "does not support"
+ )
+ version = request.prompt_format_version
+ if version is not None and version != PROMPT_FORMAT_VERSION:
+ return (
+ f"prompt_format_version {version} is not served, this server "
+ f"uses version {PROMPT_FORMAT_VERSION}"
+ )
+ # The answer position must follow the reasoning block, not sit inside it.
+ config = self.template_manager.reasoning_config
+ if config is not None and config.always_on:
+ return (
+ "/v1/decisions does not support chat templates that always "
+ "reason before answering"
+ )
+ toggle = self.reasoning_toggle
+ kwargs = request.chat_template_kwargs
+ if toggle in kwargs and kwargs[toggle] is not False:
+ return (
+ f"chat_template_kwargs sets {toggle!r} to {kwargs[toggle]!r}, "
+ "but decisions need it false or unset"
+ )
+ return None
+
+ def _chat_template_kwargs(self, request: DecisionRequest) -> Dict[str, Any]:
+ """Reasoning off, then the server defaults, then the request kwargs."""
+ kwargs = {}
+ if self.reasoning_toggle is not None:
+ kwargs[self.reasoning_toggle] = False
+ for key, value in self.default_chat_template_kwargs.items():
+ kwargs.setdefault(key, value)
+ kwargs.update(request.chat_template_kwargs)
+ return kwargs
+
+ def _convert_to_internal_request(
+ self,
+ request: DecisionRequest,
+ raw_request: Request = None,
+ ) -> Tuple[Iterator[Tuple[List[int], List[int]]], DecisionRequest]:
+ # Lazy, so the handler can let other requests run between questions.
+ return self._encoded_questions(request), request
+
+ def _encoded_questions(
+ self, request: DecisionRequest
+ ) -> Iterator[Tuple[List[int], List[int]]]:
+ """Prompt and label ids for each question, in request order."""
+ text = _render_text(request.input)
+ chat_template_kwargs = self._chat_template_kwargs(request)
+ for question in request.questions:
+ try:
+ encoded = self._encode_question(
+ text=text,
+ question=question,
+ chat_template_kwargs=chat_template_kwargs,
+ )
+ except ValueError as e:
+ raise ValueError(f"question {question.id!r}: {e}") from e
+ yield encoded
+
+ def _encode_question(
+ self,
+ text: str,
+ question: DecisionQuestion,
+ chat_template_kwargs: Dict[str, Any],
+ ) -> Tuple[List[int], List[int]]:
+ tokenizer = self.tokenizer_manager.tokenizer
+ _, labels = _answer_labels(question)
+ content = _render_question(text=text, question=question, labels=labels)
+ try:
+ prompt = tokenizer.apply_chat_template(
+ [{"role": "user", "content": content}],
+ tokenize=False,
+ add_generation_prompt=True,
+ **chat_template_kwargs,
+ )
+ except _CHAT_TEMPLATE_CLIENT_ERRORS as e:
+ raise ValueError(f"the chat template failed: {e}") from e
+ if self.reasoning_markers is not None:
+ # Look only after the message, whose last line is fixed text.
+ closing = content.rsplit("\n", 1)[-1]
+ cut = prompt.rfind(closing)
+ generation_prompt = prompt if cut < 0 else prompt[cut + len(closing) :]
+ start, end = self.reasoning_markers
+ opened = generation_prompt.rfind(start)
+ closed = generation_prompt.rfind(end)
+ if opened > closed:
+ raise ValueError(
+ "the chat template leaves a reasoning block open at the "
+ "answer position, so this model is not supported with these "
+ "chat_template_kwargs"
+ )
+ if self.answers_open_reasoning and closed < 0:
+ raise ValueError(
+ "the reasoning parser for this model expects answers to start "
+ "with a reasoning block, and the chat template does not close "
+ "one. Send chat_template_kwargs that turn thinking off, if the "
+ "template supports it"
+ )
+ # The template's own finished reply shows whether answers start with
+ # a reasoning block that the generation prompt leaves out.
+ reply = self._render_reply(closing, chat_template_kwargs)
+ begin = reply.rfind(closing) if reply is not None else -1
+ answer = reply.find(_REPLY_SENTINEL, begin) if begin >= 0 else -1
+ if answer >= 0 and reply[begin:answer].count(start) > (
+ generation_prompt.count(start)
+ ):
+ raise ValueError(
+ "the chat template starts every answer with a reasoning "
+ "block, so this model is not supported"
+ )
+ prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+ # Refuse here because --allow-auto-truncate would cut off the answer position.
+ context_len = self.tokenizer_manager.context_len
+ if len(prompt_ids) + self.tokenizer_manager.num_reserved_tokens >= context_len:
+ raise ValueError(
+ f"the prompt has {len(prompt_ids)} tokens, which does not fit "
+ f"the context length of {context_len} tokens"
+ )
+ label_ids = _encode_labels(
+ tokenizer=tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=labels,
+ added_tokens=self.added_tokens,
+ )
+ return prompt_ids, label_ids
+
+ def _render_reply(
+ self, message: str, chat_template_kwargs: Dict[str, Any]
+ ) -> Optional[str]:
+ try:
+ return self.tokenizer_manager.tokenizer.apply_chat_template(
+ [
+ {"role": "user", "content": message},
+ {"role": "assistant", "content": _REPLY_SENTINEL},
+ ],
+ tokenize=False,
+ **chat_template_kwargs,
+ )
+ except _CHAT_TEMPLATE_CLIENT_ERRORS:
+ # The generation prompt checks above still apply.
+ return None
+
+ async def _handle_non_streaming_request(
+ self,
+ adapted_request: Iterator[Tuple[List[int], List[int]]],
+ request: DecisionRequest,
+ raw_request: Request,
+ ) -> ORJSONResponse:
+ prompts, label_token_ids = [], []
+ for prompt_ids, label_ids in adapted_request:
+ prompts.append(prompt_ids)
+ label_token_ids.append(label_ids)
+ # Each question renders and tokenizes the whole input on the event loop.
+ await asyncio.sleep(0)
+ result = await self.tokenizer_manager.score_prompts(
+ prompts=prompts,
+ label_token_ids=label_token_ids,
+ apply_softmax=True,
+ request=raw_request,
+ temperature=request.temperature,
+ return_token_logprobs=True,
+ )
+ answers = {}
+ for i, question in enumerate(request.questions):
+ answer = _build_answer(
+ question=question,
+ probabilities=result.scores[i],
+ token_logprobs=result.token_logprobs[i],
+ )
+ if request.return_prompt_token_ids:
+ answer.prompt_token_ids = prompts[i]
+ answer.label_token_ids = label_token_ids[i]
+ answers[question.id] = answer
+ response = DecisionResponse(
+ model=request.model,
+ prompt_format_version=PROMPT_FORMAT_VERSION,
+ answers=answers,
+ usage=UsageInfo(
+ prompt_tokens=result.prompt_tokens,
+ total_tokens=result.prompt_tokens,
+ ),
+ )
+ return ORJSONResponse(content=response.model_dump(exclude_none=True))
+
+
+def _render_text(value: Optional[DecisionText]) -> str:
+ if value is None:
+ return ""
+ if isinstance(value, str):
+ return value
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+def _answer_labels(question: DecisionQuestion) -> Tuple[List[str], List[str]]:
+ """Answer names in the response and their single-token labels, in candidate order."""
+ if isinstance(question, DecisionChoiceQuestion):
+ names = [option.name for option in question.options]
+ return names, list(string.ascii_uppercase[: len(names)])
+ if isinstance(question, DecisionScoreQuestion):
+ levels = [str(level) for level in range(len(question.levels))]
+ return levels, levels
+ return ["yes", "no"], ["yes", "no"]
+
+
+def _render_question(text: str, question: DecisionQuestion, labels: List[str]) -> str:
+ """Prompt wording of PROMPT_FORMAT_VERSION."""
+ question_text = _render_text(question.question)
+ if isinstance(question, DecisionChoiceQuestion):
+ lines = [f"Question: {question_text}"]
+ for label, option in zip(labels, question.options):
+ detail = _render_text(option.description)
+ lines.append(
+ f"{label}: {option.name} - {detail}"
+ if detail
+ else f"{label}: {option.name}"
+ )
+ lines.append("Answer with the letter of one option only.")
+ elif isinstance(question, DecisionScoreQuestion):
+ lines = [f"Question: {question_text}"]
+ lines += [
+ f"{label}: {_render_text(level)}"
+ for label, level in zip(labels, question.levels)
+ ]
+ lines.append("Answer with the number of one level only.")
+ else:
+ lines = [f"Is the following true? {question_text}"]
+ for label, description in zip(labels, (question.yes, question.no)):
+ detail = _render_text(description)
+ if detail:
+ lines.append(f"{label}: {detail}")
+ lines.append("Answer with yes or no only.")
+ return "\n".join([text, "", *lines])
+
+
+def _encode_labels(
+ tokenizer: Any,
+ prompt: str,
+ prompt_ids: List[int],
+ labels: List[str],
+ added_tokens: Dict[int, str],
+) -> List[int]:
+ """Check that each label adds exactly one distinct token after the prompt."""
+ # Added tokens are split off before tokenization.
+ # The text after the last one tokenizes on its own,
+ # so the check does not grow with the input.
+ # When the prompt ends with an added token, each label starts a new segment,
+ # which is how the model continues after that token.
+ text, text_ids = prompt, prompt_ids
+ last = next(
+ (i for i in reversed(range(len(prompt_ids))) if prompt_ids[i] in added_tokens),
+ None,
+ )
+ if last is not None:
+ token = added_tokens[prompt_ids[last]]
+ start = prompt.rfind(token)
+ suffix = prompt[start + len(token) :]
+ if start >= 0 and (
+ tokenizer.encode(suffix, add_special_tokens=False) == prompt_ids[last + 1 :]
+ ):
+ text, text_ids = suffix, prompt_ids[last + 1 :]
+ label_ids = []
+ for label in labels:
+ ids = tokenizer.encode(text + label, add_special_tokens=False)
+ if (
+ len(ids) != len(text_ids) + 1
+ or ids[:-1] != text_ids
+ or ids[-1] in label_ids
+ ):
+ raise ValueError(
+ f"the answer label {label!r} is not one distinct token after the "
+ "chat prompt for this tokenizer, so this model is not supported"
+ )
+ label_ids.append(ids[-1])
+ return label_ids
+
+
+def _build_answer(
+ question: DecisionQuestion,
+ probabilities: List[float],
+ token_logprobs: List[float],
+) -> DecisionAnswer:
+ names, _ = _answer_labels(question)
+ value = {}
+ if isinstance(question, DecisionChoiceQuestion):
+ value["choice"] = names[probabilities.index(max(probabilities))]
+ elif isinstance(question, DecisionScoreQuestion):
+ value["score"] = math.fsum(i * p for i, p in enumerate(probabilities))
+ return DecisionAnswer(
+ type=question.type,
+ probabilities=dict(zip(names, probabilities)),
+ label_mass=math.fsum(math.exp(logprob) for logprob in token_logprobs),
+ **value,
+ )
diff --git a/python/sglang/srt/layers/logprob_processor.py b/python/sglang/srt/layers/logprob_processor.py
index 7844ed4e427d..65f945024ea7 100644
--- a/python/sglang/srt/layers/logprob_processor.py
+++ b/python/sglang/srt/layers/logprob_processor.py
@@ -155,15 +155,17 @@ def get_token_ids_logprobs_raw(
if stage == LogprobStage.DECODE:
for i, token_ids in enumerate(token_ids_logprobs_list):
if token_ids is None:
- vals.append([])
- idxs.append([])
+ # The CPU copy expects a tensor in every entry under no_copy_to_cpu.
+ # A new empty one does not keep the full logprobs alive like a view.
+ row = logprobs.new_empty((0,))
+ token_ids = []
else:
token_ids_tensor = torch.tensor(
token_ids, dtype=torch.long, pin_memory=pin_memory
).to(logprobs.device, non_blocking=True)
row = logprobs[i, token_ids_tensor]
- vals.append(row if no_copy_to_cpu else row.tolist())
- idxs.append(token_ids)
+ vals.append(row if no_copy_to_cpu else row.tolist())
+ idxs.append(token_ids)
else: # prefill
pt = 0
for i, (token_ids, pruned_len) in enumerate(
diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py
index f6164c6d0172..3dc5b4647f46 100644
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -2852,7 +2852,10 @@ def convert_logprob_style(
)
if top_logprobs_num > 0:
- if len(recv_obj.input_top_logprobs_val) > 0:
+ if (
+ recv_obj.input_top_logprobs_val is not None
+ and len(recv_obj.input_top_logprobs_val) > 0
+ ):
state.input_top_logprobs_val.extend(
recv_obj.input_top_logprobs_val[recv_obj_index]
)
@@ -2868,27 +2871,32 @@ def convert_logprob_style(
recv_obj.input_top_logprobs_idx_flat[recv_obj_index],
recv_obj.input_top_logprobs_flat_null_prefix[recv_obj_index],
)
- state.output_top_logprobs_val.extend(
- recv_obj.output_top_logprobs_val[recv_obj_index]
- )
- state.output_top_logprobs_idx.extend(
- recv_obj.output_top_logprobs_idx[recv_obj_index]
- )
+ if recv_obj.output_top_logprobs_val is not None:
+ state.output_top_logprobs_val.extend(
+ recv_obj.output_top_logprobs_val[recv_obj_index]
+ )
+ state.output_top_logprobs_idx.extend(
+ recv_obj.output_top_logprobs_idx[recv_obj_index]
+ )
if token_ids_logprob is not None:
- if len(recv_obj.input_token_ids_logprobs_val) > 0:
+ if (
+ recv_obj.input_token_ids_logprobs_val is not None
+ and len(recv_obj.input_token_ids_logprobs_val) > 0
+ ):
state.input_token_ids_logprobs_val.extend(
recv_obj.input_token_ids_logprobs_val[recv_obj_index]
)
state.input_token_ids_logprobs_idx.extend(
recv_obj.input_token_ids_logprobs_idx[recv_obj_index]
)
- state.output_token_ids_logprobs_val.extend(
- recv_obj.output_token_ids_logprobs_val[recv_obj_index]
- )
- state.output_token_ids_logprobs_idx.extend(
- recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
- )
+ if recv_obj.output_token_ids_logprobs_val is not None:
+ state.output_token_ids_logprobs_val.extend(
+ recv_obj.output_token_ids_logprobs_val[recv_obj_index]
+ )
+ state.output_token_ids_logprobs_idx.extend(
+ recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
+ )
self.add_logprob_to_meta_info(
meta_info,
diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py
index 42115543cc56..17a4cc47f9ba 100644
--- a/test/registered/unit/entrypoints/openai/test_serving_chat.py
+++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py
@@ -2956,6 +2956,31 @@ async def _collect_stream_content(self, content, choice_logprobs, req):
chunks.append(chunk)
return chunks
+ def test_streaming_top_logprobs_follow_each_token_in_chunk(self):
+ """Each token of a multi-token streaming chunk keeps its own alternatives."""
+ content = {
+ "meta_info": {
+ "output_token_logprobs": [
+ (-0.1, 1, "a"),
+ (-0.2, 2, "b"),
+ (-0.3, 3, "c"),
+ ],
+ "output_top_logprobs": [
+ [(-0.1, 1, "a"), (-2.0, 9, "x")],
+ [(-0.2, 2, "b"), (-3.0, 8, "y")],
+ [(-0.3, 3, "c"), (-4.0, 7, "z")],
+ ],
+ },
+ }
+ choice_logprobs = self.chat._process_streaming_logprobs(content, 1, 3)
+ tokens = [entry.token for entry in choice_logprobs.content]
+ alternatives = [
+ [top.token for top in entry.top_logprobs]
+ for entry in choice_logprobs.content
+ ]
+ self.assertEqual(tokens, ["b", "c"])
+ self.assertEqual(alternatives, [["b", "y"], ["c", "z"]])
+
def test_streaming_logprobs_attached_with_reasoning_parser(self):
"""Logprobs must ride on the reasoning chunk when a reasoning parser is active."""
self.chat.reasoning_parser = "qwen3"
diff --git a/test/registered/unit/entrypoints/openai/test_serving_decisions.py b/test/registered/unit/entrypoints/openai/test_serving_decisions.py
new file mode 100644
index 000000000000..3b8ec38c556b
--- /dev/null
+++ b/test/registered/unit/entrypoints/openai/test_serving_decisions.py
@@ -0,0 +1,717 @@
+"""Unit tests for /v1/decisions: request limits, prompt text, answer labels, and scoring."""
+
+import asyncio
+import json
+import string
+import unittest
+from types import SimpleNamespace
+
+import torch
+from pydantic import ValidationError
+from transformers import AddedToken, AutoTokenizer
+
+from sglang.srt.entrypoints.openai import chat_encoding
+from sglang.srt.entrypoints.openai.protocol import DecisionRequest
+from sglang.srt.entrypoints.openai.serving_decisions import (
+ PROMPT_FORMAT_VERSION,
+ OpenAIServingDecisions,
+ _encode_labels,
+ _render_question,
+)
+from sglang.srt.managers.tokenizer_manager_score_mixin import TokenizerManagerScoreMixin
+from sglang.srt.parser.template_detection import (
+ ReasoningToggleConfig,
+ detect_reasoning_parser,
+ detect_reasoning_pattern,
+)
+from sglang.srt.runtime_context import publish, restore_context, snapshot_context
+from sglang.srt.server_args import ServerArgs
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=12, suite="base-a-test-cpu")
+
+# Tokenizer files only, of a chat template that thinks by default.
+TOKENIZER = "Qwen/Qwen3.5-35B-A3B"
+
+
+def _question(kind, entries=None, question="Q"):
+ """Options as a name to description map, levels as a list, or yes and no text."""
+ body = {"type": kind, "question": question}
+ if kind == "choice" and entries is not None:
+ body["options"] = [
+ {"name": name} if text is None else {"name": name, "description": text}
+ for name, text in entries.items()
+ ]
+ elif kind == "score" and entries is not None:
+ body["levels"] = entries
+ elif entries is not None:
+ body.update(entries)
+ return body
+
+
+def _request(input_, questions, **kwargs):
+ """Questions as an id to question map, in order."""
+ return DecisionRequest(
+ input=input_,
+ questions=[{"id": qid, **question} for qid, question in questions.items()],
+ **kwargs,
+ )
+
+
+def _by_id(request, question_id):
+ return next(q for q in request.questions if q.id == question_id)
+
+
+def _encoded(handler, request):
+ """Prompt and label ids for every question, as the handler scores them."""
+ encoded, _ = handler._convert_to_internal_request(request)
+ prompts, label_ids = zip(*encoded)
+ return list(prompts), list(label_ids)
+
+
+def _handler(manager, reasoning_config=None, reasoning_parser=None, lossy=False):
+ """Build the handler over the chat serving state the server builds at startup."""
+ template = manager.tokenizer.chat_template
+ force_reasoning, detected = detect_reasoning_pattern(template)
+ if reasoning_config is None:
+ reasoning_config = detected
+ template_manager = SimpleNamespace(
+ chat_template_name=None,
+ reasoning_config=reasoning_config,
+ suggested_reasoning_parser=detect_reasoning_parser(
+ template, manager.tokenizer, detected, force_reasoning
+ ),
+ )
+ chat_serving = SimpleNamespace(
+ tokenizer_manager=manager,
+ template_manager=template_manager,
+ default_chat_template_kwargs={},
+ chat_encoding_spec=chat_encoding.resolve_chat_encoding_spec(
+ hf_config=manager.model_config.hf_config,
+ tokenizer=manager.tokenizer,
+ tool_call_parser=None,
+ ),
+ _prompt_text_round_trip_is_lossy=lossy,
+ reasoning_parser=reasoning_parser,
+ )
+ return OpenAIServingDecisions(chat_serving)
+
+
+class UnknownTokenizer:
+ """Encodes every character as the same unknown token."""
+
+ def encode(self, text, add_special_tokens=False):
+ return [0] * len(text)
+
+
+class PlainTokenizer:
+ """A tokenizer outside transformers, without added-token metadata."""
+
+ def __init__(self, tokenizer):
+ self.chat_template = tokenizer.chat_template
+ self.encode = tokenizer.encode
+ self.apply_chat_template = tokenizer.apply_chat_template
+ self.vocab_size = len(tokenizer)
+
+ def __len__(self):
+ return self.vocab_size
+
+
+class ScoringManager(TokenizerManagerScoreMixin):
+ """Replace only model execution with fixed full-vocabulary logprobs."""
+
+ def __init__(
+ self,
+ tokenizer,
+ is_generation=True,
+ context_len=4096,
+ architecture=None,
+ **server_args,
+ ):
+ self.server_args = ServerArgs(model_path="dummy", **server_args)
+ publish(self.server_args, role="test")
+ self.tokenizer = tokenizer
+ self.model_config = SimpleNamespace(
+ hf_config=SimpleNamespace(
+ architectures=[architecture or "Qwen3_5MoeForConditionalGeneration"],
+ model_type="qwen3_5_moe",
+ )
+ )
+ self.is_generation = is_generation
+ self.context_len = context_len
+ self.num_reserved_tokens = 0
+ self.request_logger = SimpleNamespace(log_requests=False)
+ generator = torch.Generator().manual_seed(0)
+ logits = torch.randn(len(tokenizer), generator=generator, dtype=torch.float64)
+ self.logprobs = torch.log_softmax(logits * 4, dim=0)
+ self.requests = []
+
+ def config_value(self, name):
+ return None
+
+ async def generate_request(self, request, raw_request):
+ self.requests.append(request)
+ request.normalize_batch_and_arguments()
+ results = []
+ for ids, labels in zip(request.input_ids, request.token_ids_logprob):
+ logprobs = [(self.logprobs[token].item(), token, None) for token in labels]
+ meta = {"prompt_tokens": len(ids), "output_token_ids_logprobs": [logprobs]}
+ results.append({"meta_info": meta})
+ yield results
+
+
+# Rendered lines of test_prompt_text by prompt format version.
+PROMPT_FIXTURES = {
+ 1: {
+ "choice": [
+ 'Question: {"question":"Which team?"}',
+ "A: billing - Payments",
+ "B: sales",
+ 'C: other - {"k":1}',
+ "Answer with the letter of one option only.",
+ ],
+ "score": [
+ "Question: Mood?",
+ "0: Calm",
+ "1: Angry",
+ "Answer with the number of one level only.",
+ ],
+ "yes_no": [
+ "Is the following true? Urgent",
+ "no: Can wait",
+ "Answer with yes or no only.",
+ ],
+ },
+}
+
+
+class TestDecisions(unittest.IsolatedAsyncioTestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+
+ def setUp(self):
+ self.addCleanup(restore_context, snapshot_context())
+
+ def test_request_limits_follow_the_label_alphabets(self):
+ options = {name: None for name in string.ascii_lowercase}
+ rejected = {
+ "unknown type": {"type": "rank", "question": "Q"},
+ "one option": _question("choice", {"a": None}),
+ "27 options": _question("choice", {**options, "extra": None}),
+ "blank option name": _question("choice", {"a": None, " ": None}),
+ "repeated option name": _question("choice", {"a": None, " A": None}),
+ "option name with a line break": _question(
+ "choice", {"a\nB: b": None, "c": None}
+ ),
+ "one level": _question("score", ["low"]),
+ "11 levels": _question("score", [str(i) for i in range(11)]),
+ "blank level": _question("score", ["low", " "]),
+ "no question text": {"type": "yes_no"},
+ "blank question text": _question("yes_no", question=" "),
+ "empty object question text": _question("yes_no", question={}),
+ "misspelled yes_no description": _question("yes_no", {"true": "Urgent"}),
+ }
+ for case, question in rejected.items():
+ with self.subTest(case), self.assertRaises(ValidationError) as caught:
+ _request("s", {"q1": question})
+ self.assertIn("questions.0", str(caught.exception))
+ question = [{"id": "q1", **_question("yes_no")}]
+ for payload in (
+ {"input": "s", "questions": []},
+ {"input": "s", "questions": question, "temperature": 0},
+ {"input": " ", "questions": question},
+ {"input": [], "questions": question},
+ {"input": "s", "questions": question, "top_p": 0.5},
+ {"input": "s", "questions": [{"id": " ", **_question("yes_no")}]},
+ {"input": "s", "questions": question * 2},
+ ):
+ with self.subTest(payload), self.assertRaises(ValidationError):
+ DecisionRequest(**payload)
+ request = _request(
+ "s",
+ {
+ "choice": _question("choice", options),
+ "score": _question("score", [str(i) for i in range(10)]),
+ "yes_no": _question("yes_no"),
+ },
+ )
+ self.assertEqual(request.chat_template_kwargs, {})
+
+ def test_prompt_text(self):
+ request = _request(
+ {"ticket": "Refund please", "tags": ["billing"]},
+ {
+ "choice": _question(
+ "choice",
+ {"billing": "Payments", "sales": None, "other": {"k": 1}},
+ question={"question": "Which team?"},
+ ),
+ "score": _question("score", ["Calm", "Angry"], question="Mood?"),
+ "yes_no": _question("yes_no", {"no": "Can wait"}, question="Urgent"),
+ },
+ )
+ text = '{"ticket":"Refund please","tags":["billing"]}'
+ # A wording change needs a new PROMPT_FORMAT_VERSION and its own fixture.
+ self.assertIn(PROMPT_FORMAT_VERSION, PROMPT_FIXTURES)
+ labels = {
+ "choice": ["A", "B", "C"],
+ "score": ["0", "1"],
+ "yes_no": ["yes", "no"],
+ }
+ for question_id, lines in PROMPT_FIXTURES[PROMPT_FORMAT_VERSION].items():
+ rendered = _render_question(
+ text=text,
+ question=_by_id(request, question_id),
+ labels=labels[question_id],
+ )
+ self.assertEqual(rendered, "\n".join([text, "", *lines]))
+
+ def test_labels_are_vocabulary_tokens_after_the_non_thinking_prompt(self):
+ handler = _handler(ScoringManager(self.tokenizer))
+ questions = {
+ "choice": _question("choice", {c: None for c in string.ascii_lowercase}),
+ "score": _question("score", [str(i) for i in range(10)]),
+ "yes_no": _question("yes_no"),
+ }
+ labels = [
+ list(string.ascii_uppercase),
+ [str(i) for i in range(10)],
+ ["yes", "no"],
+ ]
+ label_ids = [self.tokenizer.convert_tokens_to_ids(t) for t in labels]
+ # Other kwargs and a server default that thinks keep reasoning off.
+ for kwargs, server_kwargs in (
+ ({}, {}),
+ ({"chat_template_kwargs": {}}, {}),
+ ({"chat_template_kwargs": {"custom_flag": 1}}, {}),
+ ({}, {"enable_thinking": True}),
+ ):
+ with self.subTest(kwargs=kwargs, server_kwargs=server_kwargs):
+ handler.default_chat_template_kwargs = server_kwargs
+ request = _request("s", questions, **kwargs)
+ prompts, ids = _encoded(handler, request)
+ self.assertEqual(ids, label_ids)
+ for prompt in prompts:
+ self.assertTrue(
+ self.tokenizer.decode(prompt).endswith("\n\n")
+ )
+ # The suffix check agrees with the whole-prompt check.
+ text = self.tokenizer.decode(prompt)
+ for question_labels, question_ids in zip(labels, label_ids):
+ self.assertEqual(
+ _encode_labels(
+ tokenizer=self.tokenizer,
+ prompt=text,
+ prompt_ids=prompt,
+ labels=question_labels,
+ added_tokens={},
+ ),
+ question_ids,
+ )
+
+ def test_request_kwargs_override_the_server_defaults(self):
+ handler = _handler(ScoringManager(self.tokenizer))
+ handler.default_chat_template_kwargs = {"note": "server"}
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ self.tokenizer.chat_template = "{{ messages[0]['content'] }} {{ note }}\n\n"
+ question = {"q": _question("choice", {"a": None, "b": None})}
+ for kwargs, note in (({}, "server"), ({"note": "request"}, "request")):
+ with self.subTest(kwargs):
+ request = _request("s", question, chat_template_kwargs=kwargs)
+ prompts, _ = _encoded(handler, request)
+ self.assertTrue(
+ self.tokenizer.decode(prompts[0]).endswith(f" {note}\n")
+ )
+
+ def test_label_check_skips_the_text_before_the_last_added_token(self):
+ encoded = []
+ tokenizer = SimpleNamespace(
+ encode=lambda text, **kwargs: (
+ encoded.append(text) or self.tokenizer.encode(text, **kwargs)
+ )
+ )
+ added_tokens = {i: t for t, i in self.tokenizer.get_added_vocab().items()}
+ prompt = self.tokenizer.apply_chat_template(
+ [{"role": "user", "content": "state " * 2000}],
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False,
+ )
+ prompt_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ label_ids = _encode_labels(
+ tokenizer=tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=["A", "B"],
+ added_tokens=added_tokens,
+ )
+ self.assertEqual(label_ids, self.tokenizer.convert_tokens_to_ids(["A", "B"]))
+ self.assertLess(max(len(text) for text in encoded), 16)
+
+ def test_label_check_falls_back_when_the_suffix_does_not_split(self):
+ # An added token that absorbs the following space tokenizes differently
+ # from the suffix on its own, so the whole prompt is checked.
+ tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+ tokenizer.add_tokens([AddedToken("", rstrip=True)])
+ prompt = "state "
+ prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+ added_tokens = {i: t for t, i in tokenizer.get_added_vocab().items()}
+ self.assertEqual(
+ _encode_labels(
+ tokenizer=tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=["A"],
+ added_tokens=added_tokens,
+ ),
+ tokenizer.encode(prompt + "A", add_special_tokens=False)[-1:],
+ )
+
+ def test_labels_must_be_distinct_tokens(self):
+ with self.assertRaisesRegex(ValueError, "label 'B' is not one distinct"):
+ _encode_labels(
+ tokenizer=UnknownTokenizer(),
+ prompt="p",
+ prompt_ids=[0],
+ labels=["A", "B"],
+ added_tokens={},
+ )
+
+ async def test_questions_yield_to_other_requests(self):
+ handler = _handler(ScoringManager(self.tokenizer))
+ request = _request("s", {q: _question("yes_no") for q in "abc"})
+ events = []
+ encode = handler._encode_question
+
+ def recorded(**kwargs):
+ events.append("encode")
+ return encode(**kwargs)
+
+ async def other_request():
+ for _ in range(3):
+ events.append("other")
+ await asyncio.sleep(0)
+
+ handler._encode_question = recorded
+ other = asyncio.create_task(other_request())
+ response = await handler.handle_request(request, None)
+ await other
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(events, ["encode", "other"] * 3)
+
+ async def test_all_questions_are_scored_in_one_call(self):
+ manager = ScoringManager(self.tokenizer)
+ request = _request(
+ "The integration keeps failing and I am losing sales.",
+ {
+ "department": _question(
+ "choice", {"billing": None, "technical": None, "sales": None}
+ ),
+ "frustration": _question("score", ["Calm", "Civil", "Angry"]),
+ "is_urgent": _question("yes_no"),
+ },
+ temperature=2.0,
+ prompt_format_version=PROMPT_FORMAT_VERSION,
+ )
+ response = await _handler(manager).handle_request(request, None)
+ body = json.loads(response.body)
+ self.assertEqual(len(manager.requests), 1)
+ self.assertEqual(body["object"], "decisions")
+ self.assertEqual(body["prompt_format_version"], PROMPT_FORMAT_VERSION)
+ prompt_tokens = sum(len(ids) for ids in manager.requests[0].input_ids)
+ self.assertEqual(body["usage"]["prompt_tokens"], prompt_tokens)
+ self.assertEqual(body["usage"]["total_tokens"], prompt_tokens)
+ self.assertEqual(body["usage"]["completion_tokens"], 0)
+ cases = {
+ "department": (["billing", "technical", "sales"], ["A", "B", "C"]),
+ "frustration": (["0", "1", "2"], ["0", "1", "2"]),
+ "is_urgent": (["yes", "no"], ["yes", "no"]),
+ }
+ self.assertEqual(list(body["answers"]), list(cases))
+ for question_id, (names, tokens) in cases.items():
+ answer = body["answers"][question_id]
+ logprobs = manager.logprobs[self.tokenizer.convert_tokens_to_ids(tokens)]
+ probabilities = torch.softmax(logprobs / 2.0, dim=0)
+ self.assertEqual(list(answer["probabilities"]), names)
+ torch.testing.assert_close(
+ torch.tensor(
+ list(answer["probabilities"].values()), dtype=torch.float64
+ ),
+ probabilities,
+ )
+ # Relative only, label mass under these logits is far below 1e-7.
+ torch.testing.assert_close(
+ torch.tensor(answer["label_mass"], dtype=torch.float64),
+ logprobs.exp().sum(),
+ rtol=1e-9,
+ atol=0,
+ )
+ self.assertEqual(answer["type"], _by_id(request, question_id).type)
+ self.assertNotIn("prompt_token_ids", answer)
+ department = body["answers"]["department"]
+ self.assertEqual(
+ department["choice"],
+ max(department["probabilities"], key=department["probabilities"].get),
+ )
+ self.assertNotIn("score", department)
+ frustration = body["answers"]["frustration"]["probabilities"]
+ self.assertAlmostEqual(
+ body["answers"]["frustration"]["score"],
+ sum(int(level) * p for level, p in frustration.items()),
+ )
+ self.assertEqual(
+ set(body["answers"]["is_urgent"]),
+ {"type", "probabilities", "label_mass"},
+ )
+
+ async def test_returned_ids_are_the_scored_ids(self):
+ manager = ScoringManager(self.tokenizer)
+ request = _request(
+ "s",
+ {
+ "team": _question("choice", {"a": None, "b": None}),
+ "urgent": _question("yes_no"),
+ },
+ return_prompt_token_ids=True,
+ )
+ response = await _handler(manager).handle_request(request, None)
+ answers = json.loads(response.body)["answers"]
+ scored = manager.requests[0]
+ self.assertEqual(
+ [answers[q]["prompt_token_ids"] for q in ("team", "urgent")],
+ scored.input_ids,
+ )
+ self.assertEqual(
+ [answers[q]["label_token_ids"] for q in ("team", "urgent")],
+ scored.token_ids_logprob,
+ )
+
+ async def test_refusals_name_the_question_and_skip_scoring(self):
+ request = _request("s", {"first": _question("choice", {"a": None, "b": None})})
+ prompt_len = len(
+ _encoded(_handler(ScoringManager(self.tokenizer)), request)[0][0]
+ )
+ at_limit = ScoringManager(self.tokenizer, context_len=prompt_len + 3)
+ at_limit.num_reserved_tokens = 3
+ # The label merges with the trailing space, or follows a split double space.
+ trailing_space = "{{ messages[0]['content'] }}\nAnswer: "
+ double_space = "{{ messages[0]['content'] }}\nAnswer: "
+ cases = {
+ "generation model": (
+ ScoringManager(self.tokenizer, is_generation=False),
+ None,
+ ),
+ "context length": (at_limit, None),
+ "label 'A' is not one distinct token": (
+ ScoringManager(self.tokenizer),
+ trailing_space,
+ ),
+ "label 'A' is not one distinct token after": (
+ ScoringManager(self.tokenizer),
+ double_space,
+ ),
+ }
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ for message, (manager, template) in cases.items():
+ with self.subTest(message):
+ self.tokenizer.chat_template = template or original_template
+ response = await _handler(manager).handle_request(request, None)
+ self.assertEqual(response.status_code, 400)
+ error = json.loads(response.body)["message"]
+ self.assertIn(message, error)
+ if message != "generation model":
+ self.assertIn("question 'first'", error)
+ self.assertEqual(manager.requests, [])
+ self.tokenizer.chat_template = original_template
+ below_limit = ScoringManager(self.tokenizer, context_len=prompt_len + 4)
+ below_limit.num_reserved_tokens = 3
+ response = await _handler(below_limit).handle_request(request, None)
+ self.assertEqual(response.status_code, 200)
+
+ async def test_refusals_keep_the_answer_outside_reasoning(self):
+ question = {"first": _question("yes_no")}
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ # Reasoning that the detected toggle does not control.
+ open_block = "{{ messages[0]['content'] }}\nassistant\n\n"
+ cases = {
+ "always reason": (None, ReasoningToggleConfig(special_case="always"), {}),
+ "sets 'enable_thinking' to True": (None, None, {"enable_thinking": True}),
+ "sets 'enable_thinking' to None": (None, None, {"enable_thinking": None}),
+ "sets 'enable_thinking' to 0": (None, None, {"enable_thinking": 0}),
+ "leaves a reasoning block open": (open_block, None, {}),
+ }
+ for message, (template, config, kwargs) in cases.items():
+ with self.subTest(message):
+ manager = ScoringManager(self.tokenizer)
+ handler = _handler(manager, config)
+ self.tokenizer.chat_template = template or original_template
+ request = _request("s", question, chat_template_kwargs=kwargs)
+ response = await handler.handle_request(request, None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(message, json.loads(response.body)["message"])
+ self.assertEqual(manager.requests, [])
+ # A parser whose answers start inside reasoning needs a closed block, and
+ # its advice comes first when the template's own replies also open one.
+ no_block = (
+ "{% for m in messages %}{% if m['role'] == 'user' %}"
+ "{{ m['content'] }}\nassistant:\n\n{% else %}"
+ "{{ m['content'] }}{% endif %}{% endfor %}"
+ )
+ closed_block = (
+ "{{ messages[0]['content'] }}\nassistant:\n\n\n\n\n"
+ )
+ for template, status in ((no_block, 400), (closed_block, 200)):
+ with self.subTest(template=template[-24:]):
+ manager = ScoringManager(self.tokenizer)
+ handler = _handler(manager, reasoning_parser="deepseek-r1")
+ self.tokenizer.chat_template = template
+ request = _request(
+ "s", {"first": _question("choice", {"a": None, "b": None})}
+ )
+ response = await handler.handle_request(request, None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, status)
+ if status == 400:
+ self.assertIn(
+ "expects answers to start with a reasoning block",
+ json.loads(response.body)["message"],
+ )
+ # A toggle that only the parser names, in its plain or explicit form, is
+ # turned off and checked like a detected one.
+ choice = {"first": _question("choice", {"a": None, "b": None})}
+ for parser, toggle in (
+ ("qwen3", "enable_thinking"),
+ ("deepseek-v3", "thinking"),
+ ):
+ with self.subTest(parser=parser):
+ self.tokenizer.chat_template = (
+ f"{{% if {toggle} is defined and {toggle} %}}/think\n{{% endif %}}"
+ "{{ messages[0]['content'] }}\nassistant:\n\n"
+ )
+ handler = _handler(
+ ScoringManager(self.tokenizer), reasoning_parser=parser
+ )
+ handler.default_chat_template_kwargs = {toggle: True}
+ prompts, _ = _encoded(handler, _request("s", choice))
+ thinking = _request("s", choice, chat_template_kwargs={toggle: True})
+ response = await handler.handle_request(thinking, None)
+ self.tokenizer.chat_template = original_template
+ self.assertNotIn("/think", self.tokenizer.decode(prompts[0]))
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(
+ f"sets '{toggle}' to True", json.loads(response.body)["message"]
+ )
+ # A template that opens reasoning before every answer is refused.
+ for reply, status in (("", 400), ("", 200)):
+ with self.subTest(reply=reply):
+ self.tokenizer.chat_template = (
+ "{% for m in messages %}{% if m['role'] == 'user' %}"
+ "{{ m['content'] }}\nassistant:\n\n{% else %}"
+ + reply
+ + "{{ m['content'] }}\n{% endif %}{% endfor %}"
+ )
+ manager = ScoringManager(self.tokenizer)
+ handler = _handler(manager, reasoning_parser="qwen3")
+ response = await handler.handle_request(_request("s", choice), None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, status)
+ if status == 400:
+ self.assertIn(
+ "starts every answer with a reasoning block",
+ json.loads(response.body)["message"],
+ )
+ self.assertEqual(manager.requests, [])
+ # A reasoning tag inside the input does not count, with or without a
+ # reasoning block after the message.
+ request = _request(
+ " draft", {"first": _question("choice", {"a": None, "b": None})}
+ )
+ no_block = "{{ messages[0]['content'] }}\nassistant:\n\n"
+ for template in (original_template, no_block):
+ with self.subTest(template=template[-20:]):
+ handler = _handler(ScoringManager(self.tokenizer))
+ self.tokenizer.chat_template = template
+ response = await handler.handle_request(request, None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, 200)
+
+ async def test_refusals_for_unsupported_serving_setups(self):
+ def named_template():
+ handler = _handler(ScoringManager(self.tokenizer))
+ handler.template_manager.chat_template_name = "chatml"
+ return handler
+
+ # Each handler is built in its own case, since ScoringManager publishes
+ # its server args globally.
+ cases = {
+ "model names the LoRA adapter 'adapter'": (
+ lambda: _handler(ScoringManager(self.tokenizer)),
+ {"model": "base:adapter"},
+ ),
+ "uses version 1": (
+ lambda: _handler(ScoringManager(self.tokenizer)),
+ {"prompt_format_version": 2},
+ ),
+ "'dsv4' encoder": (
+ lambda: _handler(
+ ScoringManager(self.tokenizer, architecture="DeepseekV4ForCausalLM")
+ ),
+ {},
+ ),
+ "built-in chat template 'chatml'": (named_template, {}),
+ "does not encode back to the same ids": (
+ lambda: _handler(ScoringManager(self.tokenizer), lossy=True),
+ {},
+ ),
+ "--enable-mis": (
+ lambda: _handler(ScoringManager(self.tokenizer, enable_mis=True)),
+ {},
+ ),
+ "--dllm-algorithm": (
+ lambda: _handler(
+ ScoringManager(self.tokenizer, dllm_algorithm="LowConfidence")
+ ),
+ {},
+ ),
+ }
+ for message, (build, kwargs) in cases.items():
+ with self.subTest(message):
+ handler = build()
+ request = _request("s", {"q": _question("yes_no")}, **kwargs)
+ response = await handler.handle_request(request, None)
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(message, json.loads(response.body)["message"])
+ self.assertEqual(handler.tokenizer_manager.requests, [])
+
+ async def test_tokenizers_without_added_tokens_use_the_full_prompt(self):
+ request = _request("s", {"q": _question("choice", {"a": None, "b": None})})
+ plain = _handler(ScoringManager(PlainTokenizer(self.tokenizer)))
+ self.assertEqual(plain.added_tokens, {})
+ _, plain_ids = _encoded(plain, request)
+ _, ids = _encoded(_handler(ScoringManager(self.tokenizer)), request)
+ self.assertEqual(plain_ids, ids)
+
+ def test_label_after_a_final_added_token_starts_a_new_segment(self):
+ added_tokens = {i: t for t, i in self.tokenizer.get_added_vocab().items()}
+ prompt = "state"
+ prompt_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ self.assertIn(prompt_ids[-1], added_tokens)
+ self.assertEqual(
+ _encode_labels(
+ tokenizer=self.tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=["A"],
+ added_tokens=added_tokens,
+ ),
+ self.tokenizer.encode("A", add_special_tokens=False),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/layers/test_logprob_chunk_stitching.py b/test/registered/unit/layers/test_logprob_chunk_stitching.py
index 51e5cafd98cb..c4bdaa7b8da0 100644
--- a/test/registered/unit/layers/test_logprob_chunk_stitching.py
+++ b/test/registered/unit/layers/test_logprob_chunk_stitching.py
@@ -3,7 +3,8 @@
Regression for the cross-chunk stitching accounting: zero-logprob-row
sequences (logprob opt-outs in mixed batches, mid-chunked-prefill segments)
were skipped or double-emitted, drifting the per-request entry counts that
-the scheduler asserts on.
+the scheduler asserts on. Next-token token-ids logprobs of mixed batches
+must also copy to the CPU like the eager path.
"""
import unittest
@@ -13,7 +14,13 @@
import torch
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
-from sglang.srt.layers.logprob_processor import InputLogprobProcessor
+from sglang.srt.layers.logprob_processor import (
+ InputLogprobProcessor,
+ get_token_ids_logprobs,
+)
+from sglang.srt.managers.scheduler_components.batch_result_processor import (
+ SchedulerBatchResultProcessor,
+)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.logprob_test_utils import coverage_cases
from sglang.test.test_utils import CustomTestCase
@@ -158,6 +165,34 @@ def test_finalizing_input_logprobs_preserves_request_boundaries(self):
output.finalize_input_logprobs()
self.assertIs(output.input_token_ids_logprobs_val, rows)
+ def test_mixed_batch_token_ids_copy_to_cpu_like_the_eager_path(self):
+ # One request with token ids, one without, and one with an empty probe set.
+ token_ids = [[1, 3], None, []]
+ logprobs = torch.log_softmax(torch.randn(len(token_ids), 5), dim=-1)
+ vals, idxs = get_token_ids_logprobs(logprobs, token_ids, no_copy_to_cpu=True)
+ output = LogitsProcessorOutput(
+ next_token_logits=None,
+ next_token_token_ids_logprobs_val=vals,
+ next_token_token_ids_logprobs_idx=idxs,
+ )
+ SchedulerBatchResultProcessor.move_logprobs_to_cpu(
+ None,
+ batch=SimpleNamespace(return_logprob=True),
+ logits_output=output,
+ )
+ # The empty entry must not be a view that keeps the full logprobs alive.
+ self.assertIsNone(vals[1]._base)
+ expected = get_token_ids_logprobs(logprobs, token_ids, no_copy_to_cpu=False)
+ self.assertEqual(
+ (
+ output.next_token_token_ids_logprobs_val,
+ output.next_token_token_ids_logprobs_idx,
+ ),
+ expected,
+ )
+ self.assertEqual(expected[0][1:], [[], []])
+ self.assertEqual(expected[1], [[1, 3], [], []])
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/managers/test_flat_raw_top_logprobs.py b/test/registered/unit/managers/test_flat_raw_top_logprobs.py
index f7e4e09e64a6..646ccb9d2823 100644
--- a/test/registered/unit/managers/test_flat_raw_top_logprobs.py
+++ b/test/registered/unit/managers/test_flat_raw_top_logprobs.py
@@ -569,6 +569,41 @@ def test_output_logprobs_without_input_logprobs(self):
self.assertEqual(meta_info["output_token_logprobs"], [(-0.25, 42, None)])
self.assertEqual(meta_info["output_token_logprobs_length"], 1)
+ def test_top_and_token_ids_logprobs_without_logprob_lists(self):
+ """An output without logprob lists, as sent for a request aborted before
+ prefill, must not crash a request that asked for top or token id logprobs."""
+ state = _make_state(
+ return_logprob=True, top_logprobs_num=2, token_ids_logprob=[42]
+ )
+ recv_obj = SimpleNamespace(
+ input_token_logprobs_val=None,
+ output_token_logprobs_val=None,
+ input_top_logprobs_val=None,
+ input_top_logprobs_val_flat=None,
+ output_top_logprobs_val=None,
+ input_token_ids_logprobs_val=None,
+ output_token_ids_logprobs_val=None,
+ )
+ meta_info = {}
+
+ _TokenizerManagerStub().convert_logprob_style(
+ meta_info,
+ state,
+ top_logprobs_num=2,
+ token_ids_logprob=[42],
+ return_text_in_logprobs=False,
+ recv_obj=recv_obj,
+ recv_obj_index=0,
+ )
+
+ for key in (
+ "input_top_logprobs",
+ "output_top_logprobs",
+ "input_token_ids_logprobs",
+ "output_token_ids_logprobs",
+ ):
+ self.assertEqual(meta_info[key], [])
+
def _make_batch_token_id_output(**overrides) -> BatchTokenIDOutput:
"""A two-request BatchTokenIDOutput with the required fields stubbed."""