From 2548920254586cf014679d5c0b7b53ae10d67d88 Mon Sep 17 00:00:00 2001 From: Richard Wang Date: Wed, 23 Sep 2026 21:18:54 +0000 Subject: [PATCH] [Feature] Add /v1/decisions for typed decisions on candidate scoring POST /v1/decisions answers typed questions about an input with the probability of every answer, without generating. Questions are choice (named options), score (ordered levels), and yes_no. Each question is rendered as one user message with the served chat template, gets one-token labels (A to Z, 0 to 9, yes and no) checked at the answer position, and all questions of a request are scored in one score_prompts call. Answers return the probabilities and label_mass, plus the chosen option for choice or the expected level for score. A request temperature scales the probabilities but not label_mass. The server owns this prompt wording and versions it. Responses carry prompt_format_version, a request can pin it, and return_prompt_token_ids returns the scored ids so any answer can be replayed through /v1/score. The route renders the way the chat route does. The reasoning toggle, named by the chat template or, when detection finds none, by the reasoning parser, is set off, server default chat template kwargs fill other keys, and request kwargs apply last. A request toggle set to anything but false gets a 400, as do templates that always reason or start every answer with a reasoning block, a rendered prompt that leaves a reasoning block open, and a model whose reasoning parser expects answers to start inside a block the prompt does not close. Setups the route cannot serve faithfully also get a 400: --enable-mis, --dllm-algorithm, built-in named chat templates, Python chat encoders, tokenizers whose chat text does not encode back to the same ids or that split an answer label, and LoRA adapters. So do malformed requests. Questions are encoded one at a time, and other requests run between them. Validating decisions also found two crashes and one wrong result, fixed here: - Keep token-ids logprobs tensors when only some requests in a batch ask for them, so move_logprobs_to_cpu no longer stops the scheduler when scoring is batched with plain generation. - Treat absent logprob arrays as empty in convert_logprob_style, so a request whose grammar fails to compile returns its 400 instead of stopping the server. - Give each token of a multi-token chat stream chunk its own top_logprobs row. Docs: a Decision models page under Specialized Models with a card on the Supported models overview, Semantic Decisions sections on the Qwen3.8-27B and Qwen3.5 model pages, and native API entries for /v1/decisions and the scoring fields it replays through. --- docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx | 31 + .../autoregressive/Qwen/Qwen3.8-27B.mdx | 31 + docs/docs.json | 3 +- docs/docs/basic_usage/native_api.mdx | 17 + docs/docs/supported-models.mdx | 9 + .../docs/supported-models/decision_models.mdx | 227 ++++++ python/sglang/srt/entrypoints/http_server.py | 13 + .../sglang/srt/entrypoints/openai/protocol.py | 124 +++ .../srt/entrypoints/openai/serving_chat.py | 15 +- .../entrypoints/openai/serving_decisions.py | 428 +++++++++++ python/sglang/srt/layers/logprob_processor.py | 10 +- .../sglang/srt/managers/tokenizer_manager.py | 36 +- .../entrypoints/openai/test_serving_chat.py | 25 + .../openai/test_serving_decisions.py | 717 ++++++++++++++++++ .../layers/test_logprob_chunk_stitching.py | 39 +- .../managers/test_flat_raw_top_logprobs.py | 35 + 16 files changed, 1729 insertions(+), 31 deletions(-) create mode 100644 docs/docs/supported-models/decision_models.mdx create mode 100644 python/sglang/srt/entrypoints/openai/serving_decisions.py create mode 100644 test/registered/unit/entrypoints/openai/test_serving_decisions.py diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx index 0ccb28dbd10b..78681d399d9b 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -777,6 +777,37 @@ Tool Call: None Arguments: } ``` +#### 4.3.3 Semantic Decisions + +Qwen3.5-35B-A3B can answer typed questions about an input with a probability for every option through `/v1/decisions`, without generating text. The server turns thinking off for these requests, so the answer is read after a closed think block, and the same server keeps serving chat traffic. No launch flag is needed. Until a release contains `/v1/decisions`, install a [nightly build](/docs/get-started/install#nightly-builds). With a server started by `sglang serve --model-path Qwen/Qwen3.5-35B-A3B --host 127.0.0.1 --port 30000`: + +```python Example +import requests + +response = requests.post( + "http://127.0.0.1:30000/v1/decisions", + json={ + "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.", + "questions": [ + { + "id": "team", + "type": "choice", + "question": "Which team should handle this ticket?", + "options": [{"name": "billing"}, {"name": "technical"}, {"name": "sales"}], + }, + {"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."}, + ], + }, + timeout=60, +) +response.raise_for_status() +answers = response.json()["answers"] +print(answers["team"]["choice"], answers["team"]["probabilities"]) +print(answers["urgent"]["probabilities"]["yes"]) +``` + +See [Decision models](/docs/supported-models/decision_models) for the request reference, how the probabilities are computed, and the limits. + ## 5. Benchmark ### 5.1 Accuracy Benchmark diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index 30cd5eac5e53..e99d6986f347 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -541,3 +541,34 @@ providers: ``` + +## 4. Semantic Decisions + +Qwen3.8-27B can answer typed questions about an input with a probability for every option through `/v1/decisions`, without generating text. The server turns thinking off for these requests, so the answer is read after a closed think block, and the same server keeps serving chat traffic. No launch flag is needed. Until a release contains `/v1/decisions`, install a [nightly build](/docs/get-started/install#nightly-builds). With a server started by `sglang serve --model-path Qwen/Qwen3.8-27B --host 127.0.0.1 --port 30000`: + +```python Example +import requests + +response = requests.post( + "http://127.0.0.1:30000/v1/decisions", + json={ + "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.", + "questions": [ + { + "id": "team", + "type": "choice", + "question": "Which team should handle this ticket?", + "options": [{"name": "billing"}, {"name": "technical"}, {"name": "sales"}], + }, + {"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."}, + ], + }, + timeout=60, +) +response.raise_for_status() +answers = response.json()["answers"] +print(answers["team"]["choice"], answers["team"]["probabilities"]) +print(answers["urgent"]["probabilities"]["yes"]) +``` + +See [Decision models](/docs/supported-models/decision_models) for the request reference, how the probabilities are computed, and the limits. diff --git a/docs/docs.json b/docs/docs.json index 0cf88a03b152..fc942d10650b 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -998,7 +998,8 @@ { "group": "Specialized Models", "pages": [ - "docs/supported-models/reward_models" + "docs/supported-models/reward_models", + "docs/supported-models/decision_models" ] }, { diff --git a/docs/docs/basic_usage/native_api.mdx b/docs/docs/basic_usage/native_api.mdx index eff2293166d9..1673efc5b5d3 100644 --- a/docs/docs/basic_usage/native_api.mdx +++ b/docs/docs/basic_usage/native_api.mdx @@ -15,6 +15,7 @@ Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its nati - `/encode`(embedding model) - `/v1/rerank`(cross encoder rerank model) - `/v1/score`(decoder-only scoring) +- `/v1/decisions`(typed decisions on a chat model) - `/classify`(reward model) - `/start_expert_distribution_record` - `/stop_expert_distribution_record` @@ -256,6 +257,8 @@ Parameters: - `item_first`: Whether items come first in concatenation order (default: False) - `model`: Model name +`query` and `items` also accept token ids, with `query` set to `[]` when each item is a complete prompt. `label_token_ids` can be one list per item. `temperature` divides the label logits before the softmax and needs `apply_softmax`, and `return_token_logprobs` adds `token_logprobs`, the full-vocabulary log-probabilities of the labels. + The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`. ```python Example @@ -293,6 +296,20 @@ for item, scores in zip(items, response_json["scores"]): terminate_process(score_process) ``` +## v1/decisions (typed decisions on a chat model) + +Answer typed questions about an input with per-option probabilities, without generating. The server renders each question with the model's chat template and thinking turned off, assigns one-token answer labels, and scores them through the same path as `v1/score`. + +Parameters: +- `input`: Text, object, or array the questions are about +- `questions`: List of questions, each with a unique `id` and a `type` of `choice` (with `options`), `score` (with `levels`), or `yes_no` +- `temperature`: Divides the label logits before the softmax (default: 1) +- `chat_template_kwargs`: Extra chat template arguments +- `prompt_format_version`: Optional pin of the server-owned prompt wording +- `return_prompt_token_ids`: Return the scored prompt and label token ids for replay through `v1/score` (default: False) + +The response contains `answers` keyed by question id, each with `type`, `probabilities`, and `label_mass`, plus `choice` for a choice question or `score` for a score question. A yes or no answer is read from `probabilities["yes"]`. See [Decision models](/docs/supported-models/decision_models) for examples, the full reference, and error cases. + ## Classify (reward model) SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations. diff --git a/docs/docs/supported-models.mdx b/docs/docs/supported-models.mdx index bcfcf89a75ca..33ae19614b5b 100644 --- a/docs/docs/supported-models.mdx +++ b/docs/docs/supported-models.mdx @@ -84,4 +84,13 @@ SGLang supports model families across text generation, retrieval, and reward wor > RLHF and reward scoring pipelines optimized for production latency. + + Typed choice, score, and yes or no answers with a probability for every option. + diff --git a/docs/docs/supported-models/decision_models.mdx b/docs/docs/supported-models/decision_models.mdx new file mode 100644 index 000000000000..122e5876d714 --- /dev/null +++ b/docs/docs/supported-models/decision_models.mdx @@ -0,0 +1,227 @@ +--- +title: Decision models +description: Answer typed choice, score, and yes or no questions with a probability for every option from a chat model, without generating text. +--- + +`/v1/decisions` turns a chat model into a decision model. You send an input and a list of typed questions, and each answer comes back with the probability of every option, read from the model's next-token scores at the answer position. No text is generated and no output is parsed. It needs no special checkpoint: any generation model served with a Jinja chat template can answer, as long as the checks below pass for it. + + + `/v1/decisions` is an SGLang extension under `/v1`, like `/v1/score` and `/v1/rerank`. It is not part of the OpenAI API, so call it over HTTP rather than through an OpenAI SDK method. Until a release contains it, install a [nightly build](/docs/get-started/install#nightly-builds), which is built from the main branch. + + +## Supported models + +| Model | Example HuggingFace identifier | Notes | +| --- | --- | --- | +| **Qwen3.8-27B** | `Qwen/Qwen3.8-27B` | Validated on one H200 in BF16, with and without NEXTN speculative decoding. See the [model page](/cookbook/autoregressive/Qwen/Qwen3.8-27B). | +| **Qwen3.5-35B-A3B** | `Qwen/Qwen3.5-35B-A3B` | Validated on one H200 in BF16. See the [model page](/cookbook/autoregressive/Qwen/Qwen3.5). | + +Other chat models are served when the answer labels are single tokens at the answer position and the answer does not start inside a reasoning block. The server checks these per request and returns a 400 that names the reason when one fails, see [Errors](#errors). It does not check how the chat template renders the question or where the template itself would put an answer, so inspect `prompt_token_ids` from `return_prompt_token_ids` before relying on a new model. + +## Launch command + +```bash Command +python -m sglang.launch_server \ + --model-path Qwen/Qwen3.8-27B \ + --host 127.0.0.1 --port 30000 +``` + +No extra flag is needed. The same server keeps serving `/v1/chat/completions` and `/generate`, and decisions can run alongside that traffic. A server launched with `--enable-mis` or `--dllm-algorithm`, or one that uses a built-in conversation template instead of the tokenizer's Jinja template, refuses decisions. + +## Example request + +```python Example +import requests + +URL = "http://127.0.0.1:30000" + +response = requests.post( + f"{URL}/v1/decisions", + json={ + "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales.", + "questions": [ + { + "id": "team", + "type": "choice", + "question": "Which team should handle this ticket?", + "options": [ + {"name": "billing", "description": "Payment or subscription issues"}, + {"name": "technical", "description": "Bugs or integration problems"}, + {"name": "sales", "description": "Pricing or account questions"}, + ], + }, + { + "id": "frustration", + "type": "score", + "question": "How frustrated is the customer?", + "levels": ["Calm", "Frustrated but civil", "Very angry"], + }, + { + "id": "urgent", + "type": "yes_no", + "question": "The customer needs an answer today.", + }, + ], + }, + timeout=60, +) +response.raise_for_status() +for question_id, answer in response.json()["answers"].items(): + print(question_id, answer) +``` + +Each answer contains: + +- `type`: the question type, `choice`, `score`, or `yes_no`. +- `probabilities`: one value per option name, level index, or `yes` and `no`, summing to 1. For a yes or no question, `probabilities["yes"]` is the answer. +- `choice` for a choice question (the most probable option) or `score` for a score question (the probability-weighted mean level index). +- `label_mass`: the full-vocabulary probability of the answer labels at the answer position. A low value means the model puts most of its probability outside the offered answers. + +Questions are answered independently and scored together in one batch. To route the ticket in this example, send it to the chosen team, and hand it to a person when the top probability falls below a threshold you validate on your own labeled tickets. + +## How answers are computed + +The server renders each question as one user message with the model's chat template and thinking turned off, and labels the answers `A` to `Z` for options, `0` to `9` for levels, and `yes` and `no`. It checks that each label is one distinct token at the answer position, then runs one prefill pass per question through the scoring path of `/v1/score` and reads the next-token log-probabilities of the labels over the full vocabulary. For a yes or no question with log-probabilities `lp_yes` and `lp_no` and request temperature `T`: + +- `probabilities["yes"] = exp(lp_yes / T) / (exp(lp_yes / T) + exp(lp_no / T))`, which equals a softmax of the two label logits divided by `T`, because the vocabulary normalizer cancels. +- `label_mass = exp(lp_yes) + exp(lp_no)`, which does not depend on `T`. + +Choice and score questions work the same way over their labels. None of these values is a calibrated probability that the decision is correct. Validate any threshold on labeled data from your workload. + +## Request and response reference + +Request fields: + +| Field | Type | Meaning | +| --- | --- | --- | +| `input` | string, object, or array | What the questions are about. Objects and arrays are rendered as compact JSON. Must not be blank. | +| `questions` | list of questions | At least one. Each question has a unique `id`, which keys its answer. | +| `temperature` | number above 0, default 1 | Divides the label logits before the softmax over the labels. It does not change `label_mass`. | +| `chat_template_kwargs` | object, default empty | Extra chat template arguments, applied after `--default-chat-template-kwargs`. The thinking toggle that the chat template or its reasoning parser names stays off in either case. | +| `prompt_format_version` | integer, optional | Pins the prompt wording. A server that serves another version returns 400. | +| `return_prompt_token_ids` | boolean, default false | Adds `prompt_token_ids` and `label_token_ids` to each answer. | +| `model` | string, optional | Echoed in the response, as `default` when omitted. It does not select a LoRA adapter, and the `base:adapter` form returns 400. | + +Question fields: + +| Type | Fields | Labels | +| --- | --- | --- | +| `choice` | `id`, `question`, and `options`: 2 to 26 objects with a `name` and an optional `description` | `A` to `Z` in list order | +| `score` | `id`, `question`, and `levels`: 2 to 10 level descriptions, lowest first | `0` to `9` | +| `yes_no` | `id`, `question`, and optional `yes` and `no` descriptions | `yes` and `no` | + +`question`, descriptions, and levels can also be JSON objects or arrays. Each question becomes one user message: the input, a blank line, the question line, one line per option, level, or described yes or no answer, and a closing instruction to answer with one label only. An option renders as `A: billing` or `A: billing - `, a level as `0: `, and a described yes or no answer as `yes: `. + +The response has `object` set to `decisions`, `model`, `prompt_format_version`, `answers` keyed by question id, and `usage` with `prompt_tokens` for all questions and `completion_tokens` 0. + +## Errors + +The request fails with HTTP 400 for: + +- an unknown question type or an unknown field anywhere in the request +- option or level counts outside the ranges above +- a blank input, question, question id, or level +- a repeated question id, or option names that are blank, contain control or line break characters, or repeat another name after trimming and case folding +- a label that is not one distinct token at the answer position for the served tokenizer +- a prompt that does not fit the model's context length +- a chat template that always thinks before answering or starts every answer with a reasoning block, `chat_template_kwargs` that set the thinking toggle to anything but `false`, a rendered prompt that leaves a reasoning block open, or a model whose reasoning parser expects answers to start inside a reasoning block when the rendered prompt does not close one +- a `prompt_format_version` other than the served one, or a `model` that names a LoRA adapter +- a server launched with `--enable-mis` or `--dllm-algorithm`, or one that uses a built-in conversation template, whether named with `--chat-template`, loaded from a JSON template file, or inferred from the model path +- a model whose chat route uses a built-in encoder instead of a chat template, a tokenizer that does not encode the rendered chat text back to the same ids, or a model that is not a generation model +- a chat template that raises an error for the question message, or a server launched with `--skip-tokenizer-init` + +Errors about one question name its id or its position in `questions`. + +## Pin and replay the prompt + +The server owns the prompt wording of `/v1/decisions` and versions it. Every response carries `prompt_format_version`, and a change to the wording ships as a new version. Send `prompt_format_version` to fail loudly instead of receiving answers from different wording after a server upgrade. + +To keep an answer reproducible independently of the server's wording, ask for the scored ids and replay them through `/v1/score`, which only scores the ids you send: + +```python Example +import requests + +URL = "http://127.0.0.1:30000" + +body = { + "input": "The integration keeps failing.", + "questions": [{"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."}], + "prompt_format_version": 1, + "return_prompt_token_ids": True, +} +answer = requests.post(f"{URL}/v1/decisions", json=body, timeout=60).json()["answers"]["urgent"] +replay = requests.post( + f"{URL}/v1/score", + json={ + "query": [], + "items": [answer["prompt_token_ids"]], + "label_token_ids": [answer["label_token_ids"]], + "apply_softmax": True, + "return_token_logprobs": True, + }, + timeout=60, +).json() +print(answer["probabilities"], replay["scores"][0]) +``` + +The replayed scores equal the answer's probabilities when both requests use the same `temperature` and run in the same cache state. If the decision set `temperature`, send the same value in the `/v1/score` body. A replay right after the decision reuses its cached prefix, so it can differ slightly, as described in [Reproducibility](#reproducibility). + +## Thinking models + +Qwen3.8-27B and Qwen3.5-35B-A3B think by default. `/v1/decisions` turns thinking off for every question, refuses a request that turns it back on, and refuses a rendered prompt that leaves a reasoning block open, so the answer is never read inside the reasoning. Chat requests to the same server keep the model's default, and launching with `--reasoning-parser qwen3` affects chat only. + +## Reproducibility + +These models mix full and linear attention. Answer probabilities can move by up to several hundredths (0.07 in our checks), and `label_mass` by up to about 0.14, between cold and prefix-cached requests and across batch compositions, while the chosen option stayed the same in our checks. On Qwen3.5-35B-A3B, `--disable-radix-cache` gave identical values across sequential repeats, at the cost of prefix reuse. + +## Build decisions without `/v1/decisions` + +When you need your own prompt or labels, build the prompt on the client and score it through `/v1/score`, which takes token ids and one label list per item. This needs `transformers` and `jinja2` on the client. Every label must be exactly one token at the answer position: + +```python Example +import requests +from transformers import AutoTokenizer + +MODEL = "Qwen/Qwen3.8-27B" +URL = "http://127.0.0.1:30000" +tokenizer = AutoTokenizer.from_pretrained(MODEL) + +options = ["billing", "technical", "sales"] +letters = ["A", "B", "C"] +prompt = ( + "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.\n\n" + "Question: Which team should handle this ticket?\n" + + "\n".join(f"{letter}: {name}" for letter, name in zip(letters, options)) + + "\nAnswer with the letter of one option only." +) +text = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], add_generation_prompt=True, enable_thinking=False, tokenize=False +) +input_ids = tokenizer.encode(text, add_special_tokens=False) +label_ids = [] +for letter in letters: + ids = tokenizer.encode(text + letter, add_special_tokens=False) + if ids[: len(input_ids)] != input_ids or len(ids) != len(input_ids) + 1: + raise ValueError(f"Label {letter} is not one token at the answer position") + label_ids.append(ids[-1]) + +scores = requests.post( + f"{URL}/v1/score", + json={"query": [], "items": [input_ids], "label_token_ids": [label_ids], "apply_softmax": True}, + timeout=60, +).json()["scores"][0] +print(dict(zip(options, scores))) +``` + +`enable_thinking=False` renders a closed think block, so the answer token follows it directly. Do not score a prompt that leaves the think block open, because the scored position then falls inside the reasoning. + +Structured output gives the same decision through generation. A one-token regex such as `(A|B|C)` with `temperature` 0 and logprobs returns probabilities renormalized over the labels, equal to the `/v1/score` values for one-character labels in the same cache state. A JSON schema with `enum` fields fills several fields in one response, but gives no clean per-option probability, because an enum value can span several tokens. See [Structured outputs](/docs/advanced_features/structured_outputs) for the request formats. + +## Limitations + +- Labels must be one token at the answer position. `/v1/decisions` assigns one-token labels for you, and a client-built prompt must do the same. +- For `yes_no`, `label_mass` counts only the lowercase `yes` and `no` tokens. The model also puts probability on `Yes` and `No`, so this value reads lower than for choice and score questions even on clear cases, while `probabilities["yes"]` is unaffected. +- The server tells from the chat template and its reasoning parser whether answers could start inside a reasoning block, using the parser's reasoning tags in the generation prompt and in the template's own rendering of a finished answer. Reasoning that shows neither is not detected, so check `prompt_token_ids` for such a model. Some templates of models that do not reason still contain reasoning tags without ever closing a block in the generation prompt, and those models are refused. +- A default temperature in `--preferred-sampling-params` can reach scoring requests and scale both the probabilities and `label_mass` when a decision shares a batch with generation or runs under speculative decoding. Leave it unset for decisions. +- The server tokenizes each question's full prompt once on the HTTP event loop, as `/v1/score` does for its items, and lets other requests run between questions. Tokenizers whose label check falls back to the full prompt encode it once more per label. +- A decision takes a single `input`, not a chat history. To decide about a conversation, pass the history as the input text or as a JSON array. diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index ec0cacf01a7c..5045e033d2ba 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -88,6 +88,7 @@ ChatCompletionRequest, ClassifyRequest, CompletionRequest, + DecisionRequest, DetokenizeRequest, EmbeddingRequest, ErrorResponse, @@ -100,6 +101,7 @@ ) from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion +from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding from sglang.srt.entrypoints.openai.serving_rerank import OpenAIServingRerank from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore @@ -323,6 +325,9 @@ async def lifespan(fast_api_app: FastAPI): fast_api_app.state.openai_serving_score = OpenAIServingScore( _global_state.tokenizer_manager ) + fast_api_app.state.openai_serving_decisions = OpenAIServingDecisions( + fast_api_app.state.openai_serving_chat + ) fast_api_app.state.openai_serving_rerank = OpenAIServingRerank( _global_state.tokenizer_manager, _global_state.template_manager ) @@ -1933,6 +1938,14 @@ async def v1_score_request(request: ScoringRequest, raw_request: Request): ) +@app.post("/v1/decisions", dependencies=[Depends(validate_json_request)]) +async def v1_decisions_request(request: DecisionRequest, raw_request: Request): + """Answer typed choice, score, and yes or no questions about an input by scoring single-token answer labels through the scoring API, without generation.""" + return await raw_request.app.state.openai_serving_decisions.handle_request( + request, raw_request + ) + + @app.post("/v1/responses", dependencies=[Depends(validate_json_request)]) async def v1_responses_request(request: ResponsesRequest, raw_request: Request): """Endpoint for the responses API with reasoning support.""" diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 243b40e6ddba..2b197c1d3554 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -17,6 +17,7 @@ import logging import time +import unicodedata import uuid from dataclasses import dataclass from typing import ( @@ -1420,6 +1421,128 @@ class ScoringResponse(BaseModel): object: str = "scoring" +def _nonblank_decision_text(value): + if not (value.strip() if isinstance(value, str) else value): + raise ValueError("must not be blank") + return value + + +# Objects and arrays are rendered into the prompt as compact JSON. +DecisionText = Union[str, Dict[str, Any], List[Any]] +RequiredDecisionText = Annotated[DecisionText, AfterValidator(_nonblank_decision_text)] + + +class DecisionOption(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str + description: Optional[DecisionText] = None + + +class DecisionChoiceQuestion(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: Annotated[str, AfterValidator(_nonblank_decision_text)] + type: Literal["choice"] + question: RequiredDecisionText + # Options are labeled A to Z in order, so at most 26. + options: List[DecisionOption] = Field(min_length=2, max_length=26) + + @model_validator(mode="after") + def _option_names_distinct(self): + seen = set() + for option in self.options: + key = option.name.strip().casefold() + if not key: + raise ValueError(f"question {self.id!r}: option names must be nonempty") + # Each option is rendered as one prompt line. + if any(unicodedata.category(c) in ("Cc", "Zl", "Zp") for c in option.name): + raise ValueError( + f"question {self.id!r}: option name {option.name!r} must not " + "contain control or line break characters" + ) + if key in seen: + raise ValueError( + f"question {self.id!r}: option name {option.name!r} repeats " + "another option" + ) + seen.add(key) + return self + + +class DecisionScoreQuestion(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: Annotated[str, AfterValidator(_nonblank_decision_text)] + type: Literal["score"] + question: RequiredDecisionText + # Levels are labeled 0 to 9 in order, so at most 10. + levels: List[RequiredDecisionText] = Field(min_length=2, max_length=10) + + +class DecisionYesNoQuestion(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: Annotated[str, AfterValidator(_nonblank_decision_text)] + type: Literal["yes_no"] + question: RequiredDecisionText + yes: Optional[DecisionText] = None + no: Optional[DecisionText] = None + + +DecisionQuestion = Annotated[ + Union[DecisionChoiceQuestion, DecisionScoreQuestion, DecisionYesNoQuestion], + Field(discriminator="type"), +] + + +class DecisionRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + input: RequiredDecisionText + questions: List[DecisionQuestion] = Field(min_length=1) + # Scales option probabilities only, not label_mass. + temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False) + # Applied over the server defaults, with the template reasoning toggle off. + chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict) + # Pins the server-owned prompt wording. A different served version is refused. + prompt_format_version: Optional[int] = None + return_prompt_token_ids: bool = False + model: str = DEFAULT_MODEL_NAME + + @field_validator("questions") + @classmethod + def _question_ids_distinct(cls, questions): + seen = set() + for question in questions: + if question.id in seen: + raise ValueError( + f"question id {question.id!r} repeats another question" + ) + seen.add(question.id) + return questions + + +class DecisionAnswer(BaseModel): + type: Literal["choice", "score", "yes_no"] + probabilities: Dict[str, float] + # Full-vocabulary probability of all answer labels at the answer position. + label_mass: float + choice: Optional[str] = None + score: Optional[float] = None + # The exact /v1/score inputs, with return_prompt_token_ids. + prompt_token_ids: Optional[List[int]] = None + label_token_ids: Optional[List[int]] = None + + +class DecisionResponse(BaseModel): + object: str = "decisions" + model: str + prompt_format_version: int + answers: Dict[str, DecisionAnswer] + usage: UsageInfo + + class V1RerankReqInput(BaseModel): query: RerankContent = Field( ..., @@ -1545,6 +1668,7 @@ class DetokenizeResponse(BaseModel): EmbeddingRequest, ClassifyRequest, ScoringRequest, + DecisionRequest, V1RerankReqInput, TokenizeRequest, DetokenizeRequest, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 92842acff2b7..38fd20470e3e 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -2445,7 +2445,7 @@ def _process_response_logprobs(self, ret_item: dict[str, Any]) -> ChoiceLogprobs output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"] output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None) token_logprobs = self._build_token_logprobs_from_raw( - output_token_logprobs, output_top_logprobs, use_token_index=True + output_token_logprobs, output_top_logprobs ) return ChoiceLogprobs(content=token_logprobs) @@ -2453,7 +2453,6 @@ def _build_token_logprobs_from_raw( self, output_token_logprobs: list[Any], output_top_logprobs: list[Any] | None, - use_token_index: bool = False, ) -> list[ChatCompletionTokenLogprob]: """Build OpenAI ChatCompletionTokenLogprob from the engine's raw ``(logprob, token_id, token_text)`` triples. @@ -2487,13 +2486,9 @@ def _build_token_logprobs_from_raw( top_logprobs: list[TopLogprob] = [] if output_top_logprobs: - # - Non-streaming (use_token_index=True): output_top_logprobs is - # the full per-position list; take the row for this token. - # - Streaming (use_token_index=False): rows are pre-sliced so the - # current chunk holds exactly one row at index 0. - top_row_idx = token_idx if use_token_index else 0 - if top_row_idx < len(output_top_logprobs): - top_row = output_top_logprobs[top_row_idx] + # Both callers pass rows aligned with output_token_logprobs. + if token_idx < len(output_top_logprobs): + top_row = output_top_logprobs[token_idx] if top_row is not None: for top_logprob, top_id, top_text in top_row: if is_byte_level: @@ -2686,7 +2681,7 @@ def _process_streaming_logprobs( n_prev_token:total_output_logprobs ] token_logprobs = self._build_token_logprobs_from_raw( - output_token_logprobs, output_top_logprobs, use_token_index=False + output_token_logprobs, output_top_logprobs ) return ChoiceLogprobs(content=token_logprobs) diff --git a/python/sglang/srt/entrypoints/openai/serving_decisions.py b/python/sglang/srt/entrypoints/openai/serving_decisions.py new file mode 100644 index 000000000000..bfd8c0be9249 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_decisions.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import math +import string +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple + +from fastapi import Request +from fastapi.responses import ORJSONResponse +from transformers import PreTrainedTokenizerBase + +from sglang.srt.entrypoints.openai.protocol import ( + DecisionAnswer, + DecisionChoiceQuestion, + DecisionQuestion, + DecisionRequest, + DecisionResponse, + DecisionScoreQuestion, + DecisionText, + UsageInfo, +) +from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase +from sglang.srt.entrypoints.openai.serving_chat import _CHAT_TEMPLATE_CLIENT_ERRORS +from sglang.srt.parser.reasoning_parser import ReasoningParser +from sglang.srt.runtime_context import get_exec + +if TYPE_CHECKING: + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + +logger = logging.getLogger(__name__) + +# Version of the server-owned prompt wording and answer labels. +# Any change to _render_question or _answer_labels needs a new version. +PROMPT_FORMAT_VERSION = 1 + +# Parser defaults that name the chat template kwarg toggling reasoning. +_PARSER_TOGGLE_MODES = ( + "thinking", + "enable_thinking", + "explicit_thinking", + "explicit_enable_thinking", +) +# Answer text of a finished reply, rendered only to see what precedes an answer. +_REPLY_SENTINEL = "DECISION_ANSWER" + + +class OpenAIServingDecisions(OpenAIServingBase): + """Handler for /v1/decisions requests, answered by candidate scoring without generation""" + + def __init__(self, chat_serving: OpenAIServingChat): + super().__init__(chat_serving.tokenizer_manager) + # Render the way the chat route does, and refuse where it renders differently. + self.template_manager = chat_serving.template_manager + self.default_chat_template_kwargs = chat_serving.default_chat_template_kwargs + self.chat_encoding_spec = chat_serving.chat_encoding_spec + self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy + tokenizer = self.tokenizer_manager.tokenizer + # Other tokenizers skip the shortcut in _encode_labels and check the full prompt. + self.added_tokens = ( + {i: token for token, i in tokenizer.get_added_vocab().items()} + if isinstance(tokenizer, PreTrainedTokenizerBase) + else {} + ) + # The configured reasoning parser, or the one the chat template suggests, + # tells where reasoning blocks start and end and whether answers open one. + parser = ( + chat_serving.reasoning_parser + or self.template_manager.suggested_reasoning_parser + ) + self.reasoning_markers = None + self.answers_open_reasoning = False + # The kwarg that turns reasoning on and off: the template's, else, when + # detection finds no config, the one the configured or suggested parser names. + config = self.template_manager.reasoning_config + self.reasoning_toggle = config.toggle_param if config is not None else None + if parser is not None: + try: + detector = ReasoningParser( + model_type=parser, tokenizer=tokenizer + ).detector + except ValueError as e: + logger.warning( + "No reasoning block check for /v1/decisions with parser '%s': %s", + parser, + e, + ) + else: + mode = detector.reasoning_default + if config is None and mode in _PARSER_TOGGLE_MODES: + self.reasoning_toggle = mode.removeprefix("explicit_") + if detector.think_start_token and detector.think_end_token: + self.reasoning_markers = ( + detector.think_start_token, + detector.think_end_token, + ) + self.answers_open_reasoning = detector.reasoning_default == "always" + + def _request_id_prefix(self) -> str: + return "decision-" + + def _validate_request(self, request: DecisionRequest) -> Optional[str]: + if not self.tokenizer_manager.is_generation: + return "/v1/decisions requires a generation model" + if self.tokenizer_manager.tokenizer is None: + return "/v1/decisions requires the server tokenizer" + if self.chat_encoding_spec is not None: + return ( + "/v1/decisions requires a chat template, but this model's chat " + f"route uses the {self.chat_encoding_spec!r} encoder" + ) + if self.prompt_text_is_lossy: + return ( + "/v1/decisions places answer labels on the rendered chat text, " + "which this tokenizer does not encode back to the same ids" + ) + if self.template_manager.chat_template_name is not None: + return ( + "/v1/decisions renders the tokenizer's Jinja chat template, but " + "this server uses the built-in chat template " + f"{self.template_manager.chat_template_name!r}" + ) + if get_exec().features.enable_mis: + return "/v1/decisions does not support --enable-mis" + if get_exec().dllm.dllm_algorithm is not None: + return ( + "/v1/decisions does not support diffusion language models " + "served with --dllm-algorithm" + ) + _, adapter = self._parse_model_parameter(request.model) + if adapter is not None: + return ( + f"model names the LoRA adapter {adapter!r}, which /v1/decisions " + "does not support" + ) + version = request.prompt_format_version + if version is not None and version != PROMPT_FORMAT_VERSION: + return ( + f"prompt_format_version {version} is not served, this server " + f"uses version {PROMPT_FORMAT_VERSION}" + ) + # The answer position must follow the reasoning block, not sit inside it. + config = self.template_manager.reasoning_config + if config is not None and config.always_on: + return ( + "/v1/decisions does not support chat templates that always " + "reason before answering" + ) + toggle = self.reasoning_toggle + kwargs = request.chat_template_kwargs + if toggle in kwargs and kwargs[toggle] is not False: + return ( + f"chat_template_kwargs sets {toggle!r} to {kwargs[toggle]!r}, " + "but decisions need it false or unset" + ) + return None + + def _chat_template_kwargs(self, request: DecisionRequest) -> Dict[str, Any]: + """Reasoning off, then the server defaults, then the request kwargs.""" + kwargs = {} + if self.reasoning_toggle is not None: + kwargs[self.reasoning_toggle] = False + for key, value in self.default_chat_template_kwargs.items(): + kwargs.setdefault(key, value) + kwargs.update(request.chat_template_kwargs) + return kwargs + + def _convert_to_internal_request( + self, + request: DecisionRequest, + raw_request: Request = None, + ) -> Tuple[Iterator[Tuple[List[int], List[int]]], DecisionRequest]: + # Lazy, so the handler can let other requests run between questions. + return self._encoded_questions(request), request + + def _encoded_questions( + self, request: DecisionRequest + ) -> Iterator[Tuple[List[int], List[int]]]: + """Prompt and label ids for each question, in request order.""" + text = _render_text(request.input) + chat_template_kwargs = self._chat_template_kwargs(request) + for question in request.questions: + try: + encoded = self._encode_question( + text=text, + question=question, + chat_template_kwargs=chat_template_kwargs, + ) + except ValueError as e: + raise ValueError(f"question {question.id!r}: {e}") from e + yield encoded + + def _encode_question( + self, + text: str, + question: DecisionQuestion, + chat_template_kwargs: Dict[str, Any], + ) -> Tuple[List[int], List[int]]: + tokenizer = self.tokenizer_manager.tokenizer + _, labels = _answer_labels(question) + content = _render_question(text=text, question=question, labels=labels) + try: + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": content}], + tokenize=False, + add_generation_prompt=True, + **chat_template_kwargs, + ) + except _CHAT_TEMPLATE_CLIENT_ERRORS as e: + raise ValueError(f"the chat template failed: {e}") from e + if self.reasoning_markers is not None: + # Look only after the message, whose last line is fixed text. + closing = content.rsplit("\n", 1)[-1] + cut = prompt.rfind(closing) + generation_prompt = prompt if cut < 0 else prompt[cut + len(closing) :] + start, end = self.reasoning_markers + opened = generation_prompt.rfind(start) + closed = generation_prompt.rfind(end) + if opened > closed: + raise ValueError( + "the chat template leaves a reasoning block open at the " + "answer position, so this model is not supported with these " + "chat_template_kwargs" + ) + if self.answers_open_reasoning and closed < 0: + raise ValueError( + "the reasoning parser for this model expects answers to start " + "with a reasoning block, and the chat template does not close " + "one. Send chat_template_kwargs that turn thinking off, if the " + "template supports it" + ) + # The template's own finished reply shows whether answers start with + # a reasoning block that the generation prompt leaves out. + reply = self._render_reply(closing, chat_template_kwargs) + begin = reply.rfind(closing) if reply is not None else -1 + answer = reply.find(_REPLY_SENTINEL, begin) if begin >= 0 else -1 + if answer >= 0 and reply[begin:answer].count(start) > ( + generation_prompt.count(start) + ): + raise ValueError( + "the chat template starts every answer with a reasoning " + "block, so this model is not supported" + ) + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) + # Refuse here because --allow-auto-truncate would cut off the answer position. + context_len = self.tokenizer_manager.context_len + if len(prompt_ids) + self.tokenizer_manager.num_reserved_tokens >= context_len: + raise ValueError( + f"the prompt has {len(prompt_ids)} tokens, which does not fit " + f"the context length of {context_len} tokens" + ) + label_ids = _encode_labels( + tokenizer=tokenizer, + prompt=prompt, + prompt_ids=prompt_ids, + labels=labels, + added_tokens=self.added_tokens, + ) + return prompt_ids, label_ids + + def _render_reply( + self, message: str, chat_template_kwargs: Dict[str, Any] + ) -> Optional[str]: + try: + return self.tokenizer_manager.tokenizer.apply_chat_template( + [ + {"role": "user", "content": message}, + {"role": "assistant", "content": _REPLY_SENTINEL}, + ], + tokenize=False, + **chat_template_kwargs, + ) + except _CHAT_TEMPLATE_CLIENT_ERRORS: + # The generation prompt checks above still apply. + return None + + async def _handle_non_streaming_request( + self, + adapted_request: Iterator[Tuple[List[int], List[int]]], + request: DecisionRequest, + raw_request: Request, + ) -> ORJSONResponse: + prompts, label_token_ids = [], [] + for prompt_ids, label_ids in adapted_request: + prompts.append(prompt_ids) + label_token_ids.append(label_ids) + # Each question renders and tokenizes the whole input on the event loop. + await asyncio.sleep(0) + result = await self.tokenizer_manager.score_prompts( + prompts=prompts, + label_token_ids=label_token_ids, + apply_softmax=True, + request=raw_request, + temperature=request.temperature, + return_token_logprobs=True, + ) + answers = {} + for i, question in enumerate(request.questions): + answer = _build_answer( + question=question, + probabilities=result.scores[i], + token_logprobs=result.token_logprobs[i], + ) + if request.return_prompt_token_ids: + answer.prompt_token_ids = prompts[i] + answer.label_token_ids = label_token_ids[i] + answers[question.id] = answer + response = DecisionResponse( + model=request.model, + prompt_format_version=PROMPT_FORMAT_VERSION, + answers=answers, + usage=UsageInfo( + prompt_tokens=result.prompt_tokens, + total_tokens=result.prompt_tokens, + ), + ) + return ORJSONResponse(content=response.model_dump(exclude_none=True)) + + +def _render_text(value: Optional[DecisionText]) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def _answer_labels(question: DecisionQuestion) -> Tuple[List[str], List[str]]: + """Answer names in the response and their single-token labels, in candidate order.""" + if isinstance(question, DecisionChoiceQuestion): + names = [option.name for option in question.options] + return names, list(string.ascii_uppercase[: len(names)]) + if isinstance(question, DecisionScoreQuestion): + levels = [str(level) for level in range(len(question.levels))] + return levels, levels + return ["yes", "no"], ["yes", "no"] + + +def _render_question(text: str, question: DecisionQuestion, labels: List[str]) -> str: + """Prompt wording of PROMPT_FORMAT_VERSION.""" + question_text = _render_text(question.question) + if isinstance(question, DecisionChoiceQuestion): + lines = [f"Question: {question_text}"] + for label, option in zip(labels, question.options): + detail = _render_text(option.description) + lines.append( + f"{label}: {option.name} - {detail}" + if detail + else f"{label}: {option.name}" + ) + lines.append("Answer with the letter of one option only.") + elif isinstance(question, DecisionScoreQuestion): + lines = [f"Question: {question_text}"] + lines += [ + f"{label}: {_render_text(level)}" + for label, level in zip(labels, question.levels) + ] + lines.append("Answer with the number of one level only.") + else: + lines = [f"Is the following true? {question_text}"] + for label, description in zip(labels, (question.yes, question.no)): + detail = _render_text(description) + if detail: + lines.append(f"{label}: {detail}") + lines.append("Answer with yes or no only.") + return "\n".join([text, "", *lines]) + + +def _encode_labels( + tokenizer: Any, + prompt: str, + prompt_ids: List[int], + labels: List[str], + added_tokens: Dict[int, str], +) -> List[int]: + """Check that each label adds exactly one distinct token after the prompt.""" + # Added tokens are split off before tokenization. + # The text after the last one tokenizes on its own, + # so the check does not grow with the input. + # When the prompt ends with an added token, each label starts a new segment, + # which is how the model continues after that token. + text, text_ids = prompt, prompt_ids + last = next( + (i for i in reversed(range(len(prompt_ids))) if prompt_ids[i] in added_tokens), + None, + ) + if last is not None: + token = added_tokens[prompt_ids[last]] + start = prompt.rfind(token) + suffix = prompt[start + len(token) :] + if start >= 0 and ( + tokenizer.encode(suffix, add_special_tokens=False) == prompt_ids[last + 1 :] + ): + text, text_ids = suffix, prompt_ids[last + 1 :] + label_ids = [] + for label in labels: + ids = tokenizer.encode(text + label, add_special_tokens=False) + if ( + len(ids) != len(text_ids) + 1 + or ids[:-1] != text_ids + or ids[-1] in label_ids + ): + raise ValueError( + f"the answer label {label!r} is not one distinct token after the " + "chat prompt for this tokenizer, so this model is not supported" + ) + label_ids.append(ids[-1]) + return label_ids + + +def _build_answer( + question: DecisionQuestion, + probabilities: List[float], + token_logprobs: List[float], +) -> DecisionAnswer: + names, _ = _answer_labels(question) + value = {} + if isinstance(question, DecisionChoiceQuestion): + value["choice"] = names[probabilities.index(max(probabilities))] + elif isinstance(question, DecisionScoreQuestion): + value["score"] = math.fsum(i * p for i, p in enumerate(probabilities)) + return DecisionAnswer( + type=question.type, + probabilities=dict(zip(names, probabilities)), + label_mass=math.fsum(math.exp(logprob) for logprob in token_logprobs), + **value, + ) diff --git a/python/sglang/srt/layers/logprob_processor.py b/python/sglang/srt/layers/logprob_processor.py index 7844ed4e427d..65f945024ea7 100644 --- a/python/sglang/srt/layers/logprob_processor.py +++ b/python/sglang/srt/layers/logprob_processor.py @@ -155,15 +155,17 @@ def get_token_ids_logprobs_raw( if stage == LogprobStage.DECODE: for i, token_ids in enumerate(token_ids_logprobs_list): if token_ids is None: - vals.append([]) - idxs.append([]) + # The CPU copy expects a tensor in every entry under no_copy_to_cpu. + # A new empty one does not keep the full logprobs alive like a view. + row = logprobs.new_empty((0,)) + token_ids = [] else: token_ids_tensor = torch.tensor( token_ids, dtype=torch.long, pin_memory=pin_memory ).to(logprobs.device, non_blocking=True) row = logprobs[i, token_ids_tensor] - vals.append(row if no_copy_to_cpu else row.tolist()) - idxs.append(token_ids) + vals.append(row if no_copy_to_cpu else row.tolist()) + idxs.append(token_ids) else: # prefill pt = 0 for i, (token_ids, pruned_len) in enumerate( diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index f6164c6d0172..3dc5b4647f46 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -2852,7 +2852,10 @@ def convert_logprob_style( ) if top_logprobs_num > 0: - if len(recv_obj.input_top_logprobs_val) > 0: + if ( + recv_obj.input_top_logprobs_val is not None + and len(recv_obj.input_top_logprobs_val) > 0 + ): state.input_top_logprobs_val.extend( recv_obj.input_top_logprobs_val[recv_obj_index] ) @@ -2868,27 +2871,32 @@ def convert_logprob_style( recv_obj.input_top_logprobs_idx_flat[recv_obj_index], recv_obj.input_top_logprobs_flat_null_prefix[recv_obj_index], ) - state.output_top_logprobs_val.extend( - recv_obj.output_top_logprobs_val[recv_obj_index] - ) - state.output_top_logprobs_idx.extend( - recv_obj.output_top_logprobs_idx[recv_obj_index] - ) + if recv_obj.output_top_logprobs_val is not None: + state.output_top_logprobs_val.extend( + recv_obj.output_top_logprobs_val[recv_obj_index] + ) + state.output_top_logprobs_idx.extend( + recv_obj.output_top_logprobs_idx[recv_obj_index] + ) if token_ids_logprob is not None: - if len(recv_obj.input_token_ids_logprobs_val) > 0: + if ( + recv_obj.input_token_ids_logprobs_val is not None + and len(recv_obj.input_token_ids_logprobs_val) > 0 + ): state.input_token_ids_logprobs_val.extend( recv_obj.input_token_ids_logprobs_val[recv_obj_index] ) state.input_token_ids_logprobs_idx.extend( recv_obj.input_token_ids_logprobs_idx[recv_obj_index] ) - state.output_token_ids_logprobs_val.extend( - recv_obj.output_token_ids_logprobs_val[recv_obj_index] - ) - state.output_token_ids_logprobs_idx.extend( - recv_obj.output_token_ids_logprobs_idx[recv_obj_index] - ) + if recv_obj.output_token_ids_logprobs_val is not None: + state.output_token_ids_logprobs_val.extend( + recv_obj.output_token_ids_logprobs_val[recv_obj_index] + ) + state.output_token_ids_logprobs_idx.extend( + recv_obj.output_token_ids_logprobs_idx[recv_obj_index] + ) self.add_logprob_to_meta_info( meta_info, diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 42115543cc56..17a4cc47f9ba 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -2956,6 +2956,31 @@ async def _collect_stream_content(self, content, choice_logprobs, req): chunks.append(chunk) return chunks + def test_streaming_top_logprobs_follow_each_token_in_chunk(self): + """Each token of a multi-token streaming chunk keeps its own alternatives.""" + content = { + "meta_info": { + "output_token_logprobs": [ + (-0.1, 1, "a"), + (-0.2, 2, "b"), + (-0.3, 3, "c"), + ], + "output_top_logprobs": [ + [(-0.1, 1, "a"), (-2.0, 9, "x")], + [(-0.2, 2, "b"), (-3.0, 8, "y")], + [(-0.3, 3, "c"), (-4.0, 7, "z")], + ], + }, + } + choice_logprobs = self.chat._process_streaming_logprobs(content, 1, 3) + tokens = [entry.token for entry in choice_logprobs.content] + alternatives = [ + [top.token for top in entry.top_logprobs] + for entry in choice_logprobs.content + ] + self.assertEqual(tokens, ["b", "c"]) + self.assertEqual(alternatives, [["b", "y"], ["c", "z"]]) + def test_streaming_logprobs_attached_with_reasoning_parser(self): """Logprobs must ride on the reasoning chunk when a reasoning parser is active.""" self.chat.reasoning_parser = "qwen3" diff --git a/test/registered/unit/entrypoints/openai/test_serving_decisions.py b/test/registered/unit/entrypoints/openai/test_serving_decisions.py new file mode 100644 index 000000000000..3b8ec38c556b --- /dev/null +++ b/test/registered/unit/entrypoints/openai/test_serving_decisions.py @@ -0,0 +1,717 @@ +"""Unit tests for /v1/decisions: request limits, prompt text, answer labels, and scoring.""" + +import asyncio +import json +import string +import unittest +from types import SimpleNamespace + +import torch +from pydantic import ValidationError +from transformers import AddedToken, AutoTokenizer + +from sglang.srt.entrypoints.openai import chat_encoding +from sglang.srt.entrypoints.openai.protocol import DecisionRequest +from sglang.srt.entrypoints.openai.serving_decisions import ( + PROMPT_FORMAT_VERSION, + OpenAIServingDecisions, + _encode_labels, + _render_question, +) +from sglang.srt.managers.tokenizer_manager_score_mixin import TokenizerManagerScoreMixin +from sglang.srt.parser.template_detection import ( + ReasoningToggleConfig, + detect_reasoning_parser, + detect_reasoning_pattern, +) +from sglang.srt.runtime_context import publish, restore_context, snapshot_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=12, suite="base-a-test-cpu") + +# Tokenizer files only, of a chat template that thinks by default. +TOKENIZER = "Qwen/Qwen3.5-35B-A3B" + + +def _question(kind, entries=None, question="Q"): + """Options as a name to description map, levels as a list, or yes and no text.""" + body = {"type": kind, "question": question} + if kind == "choice" and entries is not None: + body["options"] = [ + {"name": name} if text is None else {"name": name, "description": text} + for name, text in entries.items() + ] + elif kind == "score" and entries is not None: + body["levels"] = entries + elif entries is not None: + body.update(entries) + return body + + +def _request(input_, questions, **kwargs): + """Questions as an id to question map, in order.""" + return DecisionRequest( + input=input_, + questions=[{"id": qid, **question} for qid, question in questions.items()], + **kwargs, + ) + + +def _by_id(request, question_id): + return next(q for q in request.questions if q.id == question_id) + + +def _encoded(handler, request): + """Prompt and label ids for every question, as the handler scores them.""" + encoded, _ = handler._convert_to_internal_request(request) + prompts, label_ids = zip(*encoded) + return list(prompts), list(label_ids) + + +def _handler(manager, reasoning_config=None, reasoning_parser=None, lossy=False): + """Build the handler over the chat serving state the server builds at startup.""" + template = manager.tokenizer.chat_template + force_reasoning, detected = detect_reasoning_pattern(template) + if reasoning_config is None: + reasoning_config = detected + template_manager = SimpleNamespace( + chat_template_name=None, + reasoning_config=reasoning_config, + suggested_reasoning_parser=detect_reasoning_parser( + template, manager.tokenizer, detected, force_reasoning + ), + ) + chat_serving = SimpleNamespace( + tokenizer_manager=manager, + template_manager=template_manager, + default_chat_template_kwargs={}, + chat_encoding_spec=chat_encoding.resolve_chat_encoding_spec( + hf_config=manager.model_config.hf_config, + tokenizer=manager.tokenizer, + tool_call_parser=None, + ), + _prompt_text_round_trip_is_lossy=lossy, + reasoning_parser=reasoning_parser, + ) + return OpenAIServingDecisions(chat_serving) + + +class UnknownTokenizer: + """Encodes every character as the same unknown token.""" + + def encode(self, text, add_special_tokens=False): + return [0] * len(text) + + +class PlainTokenizer: + """A tokenizer outside transformers, without added-token metadata.""" + + def __init__(self, tokenizer): + self.chat_template = tokenizer.chat_template + self.encode = tokenizer.encode + self.apply_chat_template = tokenizer.apply_chat_template + self.vocab_size = len(tokenizer) + + def __len__(self): + return self.vocab_size + + +class ScoringManager(TokenizerManagerScoreMixin): + """Replace only model execution with fixed full-vocabulary logprobs.""" + + def __init__( + self, + tokenizer, + is_generation=True, + context_len=4096, + architecture=None, + **server_args, + ): + self.server_args = ServerArgs(model_path="dummy", **server_args) + publish(self.server_args, role="test") + self.tokenizer = tokenizer + self.model_config = SimpleNamespace( + hf_config=SimpleNamespace( + architectures=[architecture or "Qwen3_5MoeForConditionalGeneration"], + model_type="qwen3_5_moe", + ) + ) + self.is_generation = is_generation + self.context_len = context_len + self.num_reserved_tokens = 0 + self.request_logger = SimpleNamespace(log_requests=False) + generator = torch.Generator().manual_seed(0) + logits = torch.randn(len(tokenizer), generator=generator, dtype=torch.float64) + self.logprobs = torch.log_softmax(logits * 4, dim=0) + self.requests = [] + + def config_value(self, name): + return None + + async def generate_request(self, request, raw_request): + self.requests.append(request) + request.normalize_batch_and_arguments() + results = [] + for ids, labels in zip(request.input_ids, request.token_ids_logprob): + logprobs = [(self.logprobs[token].item(), token, None) for token in labels] + meta = {"prompt_tokens": len(ids), "output_token_ids_logprobs": [logprobs]} + results.append({"meta_info": meta}) + yield results + + +# Rendered lines of test_prompt_text by prompt format version. +PROMPT_FIXTURES = { + 1: { + "choice": [ + 'Question: {"question":"Which team?"}', + "A: billing - Payments", + "B: sales", + 'C: other - {"k":1}', + "Answer with the letter of one option only.", + ], + "score": [ + "Question: Mood?", + "0: Calm", + "1: Angry", + "Answer with the number of one level only.", + ], + "yes_no": [ + "Is the following true? Urgent", + "no: Can wait", + "Answer with yes or no only.", + ], + }, +} + + +class TestDecisions(unittest.IsolatedAsyncioTestCase): + @classmethod + def setUpClass(cls): + cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) + + def setUp(self): + self.addCleanup(restore_context, snapshot_context()) + + def test_request_limits_follow_the_label_alphabets(self): + options = {name: None for name in string.ascii_lowercase} + rejected = { + "unknown type": {"type": "rank", "question": "Q"}, + "one option": _question("choice", {"a": None}), + "27 options": _question("choice", {**options, "extra": None}), + "blank option name": _question("choice", {"a": None, " ": None}), + "repeated option name": _question("choice", {"a": None, " A": None}), + "option name with a line break": _question( + "choice", {"a\nB: b": None, "c": None} + ), + "one level": _question("score", ["low"]), + "11 levels": _question("score", [str(i) for i in range(11)]), + "blank level": _question("score", ["low", " "]), + "no question text": {"type": "yes_no"}, + "blank question text": _question("yes_no", question=" "), + "empty object question text": _question("yes_no", question={}), + "misspelled yes_no description": _question("yes_no", {"true": "Urgent"}), + } + for case, question in rejected.items(): + with self.subTest(case), self.assertRaises(ValidationError) as caught: + _request("s", {"q1": question}) + self.assertIn("questions.0", str(caught.exception)) + question = [{"id": "q1", **_question("yes_no")}] + for payload in ( + {"input": "s", "questions": []}, + {"input": "s", "questions": question, "temperature": 0}, + {"input": " ", "questions": question}, + {"input": [], "questions": question}, + {"input": "s", "questions": question, "top_p": 0.5}, + {"input": "s", "questions": [{"id": " ", **_question("yes_no")}]}, + {"input": "s", "questions": question * 2}, + ): + with self.subTest(payload), self.assertRaises(ValidationError): + DecisionRequest(**payload) + request = _request( + "s", + { + "choice": _question("choice", options), + "score": _question("score", [str(i) for i in range(10)]), + "yes_no": _question("yes_no"), + }, + ) + self.assertEqual(request.chat_template_kwargs, {}) + + def test_prompt_text(self): + request = _request( + {"ticket": "Refund please", "tags": ["billing"]}, + { + "choice": _question( + "choice", + {"billing": "Payments", "sales": None, "other": {"k": 1}}, + question={"question": "Which team?"}, + ), + "score": _question("score", ["Calm", "Angry"], question="Mood?"), + "yes_no": _question("yes_no", {"no": "Can wait"}, question="Urgent"), + }, + ) + text = '{"ticket":"Refund please","tags":["billing"]}' + # A wording change needs a new PROMPT_FORMAT_VERSION and its own fixture. + self.assertIn(PROMPT_FORMAT_VERSION, PROMPT_FIXTURES) + labels = { + "choice": ["A", "B", "C"], + "score": ["0", "1"], + "yes_no": ["yes", "no"], + } + for question_id, lines in PROMPT_FIXTURES[PROMPT_FORMAT_VERSION].items(): + rendered = _render_question( + text=text, + question=_by_id(request, question_id), + labels=labels[question_id], + ) + self.assertEqual(rendered, "\n".join([text, "", *lines])) + + def test_labels_are_vocabulary_tokens_after_the_non_thinking_prompt(self): + handler = _handler(ScoringManager(self.tokenizer)) + questions = { + "choice": _question("choice", {c: None for c in string.ascii_lowercase}), + "score": _question("score", [str(i) for i in range(10)]), + "yes_no": _question("yes_no"), + } + labels = [ + list(string.ascii_uppercase), + [str(i) for i in range(10)], + ["yes", "no"], + ] + label_ids = [self.tokenizer.convert_tokens_to_ids(t) for t in labels] + # Other kwargs and a server default that thinks keep reasoning off. + for kwargs, server_kwargs in ( + ({}, {}), + ({"chat_template_kwargs": {}}, {}), + ({"chat_template_kwargs": {"custom_flag": 1}}, {}), + ({}, {"enable_thinking": True}), + ): + with self.subTest(kwargs=kwargs, server_kwargs=server_kwargs): + handler.default_chat_template_kwargs = server_kwargs + request = _request("s", questions, **kwargs) + prompts, ids = _encoded(handler, request) + self.assertEqual(ids, label_ids) + for prompt in prompts: + self.assertTrue( + self.tokenizer.decode(prompt).endswith("\n\n") + ) + # The suffix check agrees with the whole-prompt check. + text = self.tokenizer.decode(prompt) + for question_labels, question_ids in zip(labels, label_ids): + self.assertEqual( + _encode_labels( + tokenizer=self.tokenizer, + prompt=text, + prompt_ids=prompt, + labels=question_labels, + added_tokens={}, + ), + question_ids, + ) + + def test_request_kwargs_override_the_server_defaults(self): + handler = _handler(ScoringManager(self.tokenizer)) + handler.default_chat_template_kwargs = {"note": "server"} + original_template = self.tokenizer.chat_template + self.addCleanup(setattr, self.tokenizer, "chat_template", original_template) + self.tokenizer.chat_template = "{{ messages[0]['content'] }} {{ note }}\n\n" + question = {"q": _question("choice", {"a": None, "b": None})} + for kwargs, note in (({}, "server"), ({"note": "request"}, "request")): + with self.subTest(kwargs): + request = _request("s", question, chat_template_kwargs=kwargs) + prompts, _ = _encoded(handler, request) + self.assertTrue( + self.tokenizer.decode(prompts[0]).endswith(f" {note}\n") + ) + + def test_label_check_skips_the_text_before_the_last_added_token(self): + encoded = [] + tokenizer = SimpleNamespace( + encode=lambda text, **kwargs: ( + encoded.append(text) or self.tokenizer.encode(text, **kwargs) + ) + ) + added_tokens = {i: t for t, i in self.tokenizer.get_added_vocab().items()} + prompt = self.tokenizer.apply_chat_template( + [{"role": "user", "content": "state " * 2000}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + prompt_ids = self.tokenizer.encode(prompt, add_special_tokens=False) + label_ids = _encode_labels( + tokenizer=tokenizer, + prompt=prompt, + prompt_ids=prompt_ids, + labels=["A", "B"], + added_tokens=added_tokens, + ) + self.assertEqual(label_ids, self.tokenizer.convert_tokens_to_ids(["A", "B"])) + self.assertLess(max(len(text) for text in encoded), 16) + + def test_label_check_falls_back_when_the_suffix_does_not_split(self): + # An added token that absorbs the following space tokenizes differently + # from the suffix on its own, so the whole prompt is checked. + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) + tokenizer.add_tokens([AddedToken("", rstrip=True)]) + prompt = "state " + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) + added_tokens = {i: t for t, i in tokenizer.get_added_vocab().items()} + self.assertEqual( + _encode_labels( + tokenizer=tokenizer, + prompt=prompt, + prompt_ids=prompt_ids, + labels=["A"], + added_tokens=added_tokens, + ), + tokenizer.encode(prompt + "A", add_special_tokens=False)[-1:], + ) + + def test_labels_must_be_distinct_tokens(self): + with self.assertRaisesRegex(ValueError, "label 'B' is not one distinct"): + _encode_labels( + tokenizer=UnknownTokenizer(), + prompt="p", + prompt_ids=[0], + labels=["A", "B"], + added_tokens={}, + ) + + async def test_questions_yield_to_other_requests(self): + handler = _handler(ScoringManager(self.tokenizer)) + request = _request("s", {q: _question("yes_no") for q in "abc"}) + events = [] + encode = handler._encode_question + + def recorded(**kwargs): + events.append("encode") + return encode(**kwargs) + + async def other_request(): + for _ in range(3): + events.append("other") + await asyncio.sleep(0) + + handler._encode_question = recorded + other = asyncio.create_task(other_request()) + response = await handler.handle_request(request, None) + await other + self.assertEqual(response.status_code, 200) + self.assertEqual(events, ["encode", "other"] * 3) + + async def test_all_questions_are_scored_in_one_call(self): + manager = ScoringManager(self.tokenizer) + request = _request( + "The integration keeps failing and I am losing sales.", + { + "department": _question( + "choice", {"billing": None, "technical": None, "sales": None} + ), + "frustration": _question("score", ["Calm", "Civil", "Angry"]), + "is_urgent": _question("yes_no"), + }, + temperature=2.0, + prompt_format_version=PROMPT_FORMAT_VERSION, + ) + response = await _handler(manager).handle_request(request, None) + body = json.loads(response.body) + self.assertEqual(len(manager.requests), 1) + self.assertEqual(body["object"], "decisions") + self.assertEqual(body["prompt_format_version"], PROMPT_FORMAT_VERSION) + prompt_tokens = sum(len(ids) for ids in manager.requests[0].input_ids) + self.assertEqual(body["usage"]["prompt_tokens"], prompt_tokens) + self.assertEqual(body["usage"]["total_tokens"], prompt_tokens) + self.assertEqual(body["usage"]["completion_tokens"], 0) + cases = { + "department": (["billing", "technical", "sales"], ["A", "B", "C"]), + "frustration": (["0", "1", "2"], ["0", "1", "2"]), + "is_urgent": (["yes", "no"], ["yes", "no"]), + } + self.assertEqual(list(body["answers"]), list(cases)) + for question_id, (names, tokens) in cases.items(): + answer = body["answers"][question_id] + logprobs = manager.logprobs[self.tokenizer.convert_tokens_to_ids(tokens)] + probabilities = torch.softmax(logprobs / 2.0, dim=0) + self.assertEqual(list(answer["probabilities"]), names) + torch.testing.assert_close( + torch.tensor( + list(answer["probabilities"].values()), dtype=torch.float64 + ), + probabilities, + ) + # Relative only, label mass under these logits is far below 1e-7. + torch.testing.assert_close( + torch.tensor(answer["label_mass"], dtype=torch.float64), + logprobs.exp().sum(), + rtol=1e-9, + atol=0, + ) + self.assertEqual(answer["type"], _by_id(request, question_id).type) + self.assertNotIn("prompt_token_ids", answer) + department = body["answers"]["department"] + self.assertEqual( + department["choice"], + max(department["probabilities"], key=department["probabilities"].get), + ) + self.assertNotIn("score", department) + frustration = body["answers"]["frustration"]["probabilities"] + self.assertAlmostEqual( + body["answers"]["frustration"]["score"], + sum(int(level) * p for level, p in frustration.items()), + ) + self.assertEqual( + set(body["answers"]["is_urgent"]), + {"type", "probabilities", "label_mass"}, + ) + + async def test_returned_ids_are_the_scored_ids(self): + manager = ScoringManager(self.tokenizer) + request = _request( + "s", + { + "team": _question("choice", {"a": None, "b": None}), + "urgent": _question("yes_no"), + }, + return_prompt_token_ids=True, + ) + response = await _handler(manager).handle_request(request, None) + answers = json.loads(response.body)["answers"] + scored = manager.requests[0] + self.assertEqual( + [answers[q]["prompt_token_ids"] for q in ("team", "urgent")], + scored.input_ids, + ) + self.assertEqual( + [answers[q]["label_token_ids"] for q in ("team", "urgent")], + scored.token_ids_logprob, + ) + + async def test_refusals_name_the_question_and_skip_scoring(self): + request = _request("s", {"first": _question("choice", {"a": None, "b": None})}) + prompt_len = len( + _encoded(_handler(ScoringManager(self.tokenizer)), request)[0][0] + ) + at_limit = ScoringManager(self.tokenizer, context_len=prompt_len + 3) + at_limit.num_reserved_tokens = 3 + # The label merges with the trailing space, or follows a split double space. + trailing_space = "{{ messages[0]['content'] }}\nAnswer: " + double_space = "{{ messages[0]['content'] }}\nAnswer: " + cases = { + "generation model": ( + ScoringManager(self.tokenizer, is_generation=False), + None, + ), + "context length": (at_limit, None), + "label 'A' is not one distinct token": ( + ScoringManager(self.tokenizer), + trailing_space, + ), + "label 'A' is not one distinct token after": ( + ScoringManager(self.tokenizer), + double_space, + ), + } + original_template = self.tokenizer.chat_template + self.addCleanup(setattr, self.tokenizer, "chat_template", original_template) + for message, (manager, template) in cases.items(): + with self.subTest(message): + self.tokenizer.chat_template = template or original_template + response = await _handler(manager).handle_request(request, None) + self.assertEqual(response.status_code, 400) + error = json.loads(response.body)["message"] + self.assertIn(message, error) + if message != "generation model": + self.assertIn("question 'first'", error) + self.assertEqual(manager.requests, []) + self.tokenizer.chat_template = original_template + below_limit = ScoringManager(self.tokenizer, context_len=prompt_len + 4) + below_limit.num_reserved_tokens = 3 + response = await _handler(below_limit).handle_request(request, None) + self.assertEqual(response.status_code, 200) + + async def test_refusals_keep_the_answer_outside_reasoning(self): + question = {"first": _question("yes_no")} + original_template = self.tokenizer.chat_template + self.addCleanup(setattr, self.tokenizer, "chat_template", original_template) + # Reasoning that the detected toggle does not control. + open_block = "{{ messages[0]['content'] }}\nassistant\n\n" + cases = { + "always reason": (None, ReasoningToggleConfig(special_case="always"), {}), + "sets 'enable_thinking' to True": (None, None, {"enable_thinking": True}), + "sets 'enable_thinking' to None": (None, None, {"enable_thinking": None}), + "sets 'enable_thinking' to 0": (None, None, {"enable_thinking": 0}), + "leaves a reasoning block open": (open_block, None, {}), + } + for message, (template, config, kwargs) in cases.items(): + with self.subTest(message): + manager = ScoringManager(self.tokenizer) + handler = _handler(manager, config) + self.tokenizer.chat_template = template or original_template + request = _request("s", question, chat_template_kwargs=kwargs) + response = await handler.handle_request(request, None) + self.tokenizer.chat_template = original_template + self.assertEqual(response.status_code, 400) + self.assertIn(message, json.loads(response.body)["message"]) + self.assertEqual(manager.requests, []) + # A parser whose answers start inside reasoning needs a closed block, and + # its advice comes first when the template's own replies also open one. + no_block = ( + "{% for m in messages %}{% if m['role'] == 'user' %}" + "{{ m['content'] }}\nassistant:\n\n{% else %}" + "{{ m['content'] }}{% endif %}{% endfor %}" + ) + closed_block = ( + "{{ messages[0]['content'] }}\nassistant:\n\n\n\n\n" + ) + for template, status in ((no_block, 400), (closed_block, 200)): + with self.subTest(template=template[-24:]): + manager = ScoringManager(self.tokenizer) + handler = _handler(manager, reasoning_parser="deepseek-r1") + self.tokenizer.chat_template = template + request = _request( + "s", {"first": _question("choice", {"a": None, "b": None})} + ) + response = await handler.handle_request(request, None) + self.tokenizer.chat_template = original_template + self.assertEqual(response.status_code, status) + if status == 400: + self.assertIn( + "expects answers to start with a reasoning block", + json.loads(response.body)["message"], + ) + # A toggle that only the parser names, in its plain or explicit form, is + # turned off and checked like a detected one. + choice = {"first": _question("choice", {"a": None, "b": None})} + for parser, toggle in ( + ("qwen3", "enable_thinking"), + ("deepseek-v3", "thinking"), + ): + with self.subTest(parser=parser): + self.tokenizer.chat_template = ( + f"{{% if {toggle} is defined and {toggle} %}}/think\n{{% endif %}}" + "{{ messages[0]['content'] }}\nassistant:\n\n" + ) + handler = _handler( + ScoringManager(self.tokenizer), reasoning_parser=parser + ) + handler.default_chat_template_kwargs = {toggle: True} + prompts, _ = _encoded(handler, _request("s", choice)) + thinking = _request("s", choice, chat_template_kwargs={toggle: True}) + response = await handler.handle_request(thinking, None) + self.tokenizer.chat_template = original_template + self.assertNotIn("/think", self.tokenizer.decode(prompts[0])) + self.assertEqual(response.status_code, 400) + self.assertIn( + f"sets '{toggle}' to True", json.loads(response.body)["message"] + ) + # A template that opens reasoning before every answer is refused. + for reply, status in (("", 400), ("", 200)): + with self.subTest(reply=reply): + self.tokenizer.chat_template = ( + "{% for m in messages %}{% if m['role'] == 'user' %}" + "{{ m['content'] }}\nassistant:\n\n{% else %}" + + reply + + "{{ m['content'] }}\n{% endif %}{% endfor %}" + ) + manager = ScoringManager(self.tokenizer) + handler = _handler(manager, reasoning_parser="qwen3") + response = await handler.handle_request(_request("s", choice), None) + self.tokenizer.chat_template = original_template + self.assertEqual(response.status_code, status) + if status == 400: + self.assertIn( + "starts every answer with a reasoning block", + json.loads(response.body)["message"], + ) + self.assertEqual(manager.requests, []) + # A reasoning tag inside the input does not count, with or without a + # reasoning block after the message. + request = _request( + " draft", {"first": _question("choice", {"a": None, "b": None})} + ) + no_block = "{{ messages[0]['content'] }}\nassistant:\n\n" + for template in (original_template, no_block): + with self.subTest(template=template[-20:]): + handler = _handler(ScoringManager(self.tokenizer)) + self.tokenizer.chat_template = template + response = await handler.handle_request(request, None) + self.tokenizer.chat_template = original_template + self.assertEqual(response.status_code, 200) + + async def test_refusals_for_unsupported_serving_setups(self): + def named_template(): + handler = _handler(ScoringManager(self.tokenizer)) + handler.template_manager.chat_template_name = "chatml" + return handler + + # Each handler is built in its own case, since ScoringManager publishes + # its server args globally. + cases = { + "model names the LoRA adapter 'adapter'": ( + lambda: _handler(ScoringManager(self.tokenizer)), + {"model": "base:adapter"}, + ), + "uses version 1": ( + lambda: _handler(ScoringManager(self.tokenizer)), + {"prompt_format_version": 2}, + ), + "'dsv4' encoder": ( + lambda: _handler( + ScoringManager(self.tokenizer, architecture="DeepseekV4ForCausalLM") + ), + {}, + ), + "built-in chat template 'chatml'": (named_template, {}), + "does not encode back to the same ids": ( + lambda: _handler(ScoringManager(self.tokenizer), lossy=True), + {}, + ), + "--enable-mis": ( + lambda: _handler(ScoringManager(self.tokenizer, enable_mis=True)), + {}, + ), + "--dllm-algorithm": ( + lambda: _handler( + ScoringManager(self.tokenizer, dllm_algorithm="LowConfidence") + ), + {}, + ), + } + for message, (build, kwargs) in cases.items(): + with self.subTest(message): + handler = build() + request = _request("s", {"q": _question("yes_no")}, **kwargs) + response = await handler.handle_request(request, None) + self.assertEqual(response.status_code, 400) + self.assertIn(message, json.loads(response.body)["message"]) + self.assertEqual(handler.tokenizer_manager.requests, []) + + async def test_tokenizers_without_added_tokens_use_the_full_prompt(self): + request = _request("s", {"q": _question("choice", {"a": None, "b": None})}) + plain = _handler(ScoringManager(PlainTokenizer(self.tokenizer))) + self.assertEqual(plain.added_tokens, {}) + _, plain_ids = _encoded(plain, request) + _, ids = _encoded(_handler(ScoringManager(self.tokenizer)), request) + self.assertEqual(plain_ids, ids) + + def test_label_after_a_final_added_token_starts_a_new_segment(self): + added_tokens = {i: t for t, i in self.tokenizer.get_added_vocab().items()} + prompt = "state" + prompt_ids = self.tokenizer.encode(prompt, add_special_tokens=False) + self.assertIn(prompt_ids[-1], added_tokens) + self.assertEqual( + _encode_labels( + tokenizer=self.tokenizer, + prompt=prompt, + prompt_ids=prompt_ids, + labels=["A"], + added_tokens=added_tokens, + ), + self.tokenizer.encode("A", add_special_tokens=False), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_logprob_chunk_stitching.py b/test/registered/unit/layers/test_logprob_chunk_stitching.py index 51e5cafd98cb..c4bdaa7b8da0 100644 --- a/test/registered/unit/layers/test_logprob_chunk_stitching.py +++ b/test/registered/unit/layers/test_logprob_chunk_stitching.py @@ -3,7 +3,8 @@ Regression for the cross-chunk stitching accounting: zero-logprob-row sequences (logprob opt-outs in mixed batches, mid-chunked-prefill segments) were skipped or double-emitted, drifting the per-request entry counts that -the scheduler asserts on. +the scheduler asserts on. Next-token token-ids logprobs of mixed batches +must also copy to the CPU like the eager path. """ import unittest @@ -13,7 +14,13 @@ import torch from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.logprob_processor import InputLogprobProcessor +from sglang.srt.layers.logprob_processor import ( + InputLogprobProcessor, + get_token_ids_logprobs, +) +from sglang.srt.managers.scheduler_components.batch_result_processor import ( + SchedulerBatchResultProcessor, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.logprob_test_utils import coverage_cases from sglang.test.test_utils import CustomTestCase @@ -158,6 +165,34 @@ def test_finalizing_input_logprobs_preserves_request_boundaries(self): output.finalize_input_logprobs() self.assertIs(output.input_token_ids_logprobs_val, rows) + def test_mixed_batch_token_ids_copy_to_cpu_like_the_eager_path(self): + # One request with token ids, one without, and one with an empty probe set. + token_ids = [[1, 3], None, []] + logprobs = torch.log_softmax(torch.randn(len(token_ids), 5), dim=-1) + vals, idxs = get_token_ids_logprobs(logprobs, token_ids, no_copy_to_cpu=True) + output = LogitsProcessorOutput( + next_token_logits=None, + next_token_token_ids_logprobs_val=vals, + next_token_token_ids_logprobs_idx=idxs, + ) + SchedulerBatchResultProcessor.move_logprobs_to_cpu( + None, + batch=SimpleNamespace(return_logprob=True), + logits_output=output, + ) + # The empty entry must not be a view that keeps the full logprobs alive. + self.assertIsNone(vals[1]._base) + expected = get_token_ids_logprobs(logprobs, token_ids, no_copy_to_cpu=False) + self.assertEqual( + ( + output.next_token_token_ids_logprobs_val, + output.next_token_token_ids_logprobs_idx, + ), + expected, + ) + self.assertEqual(expected[0][1:], [[], []]) + self.assertEqual(expected[1], [[1, 3], [], []]) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_flat_raw_top_logprobs.py b/test/registered/unit/managers/test_flat_raw_top_logprobs.py index f7e4e09e64a6..646ccb9d2823 100644 --- a/test/registered/unit/managers/test_flat_raw_top_logprobs.py +++ b/test/registered/unit/managers/test_flat_raw_top_logprobs.py @@ -569,6 +569,41 @@ def test_output_logprobs_without_input_logprobs(self): self.assertEqual(meta_info["output_token_logprobs"], [(-0.25, 42, None)]) self.assertEqual(meta_info["output_token_logprobs_length"], 1) + def test_top_and_token_ids_logprobs_without_logprob_lists(self): + """An output without logprob lists, as sent for a request aborted before + prefill, must not crash a request that asked for top or token id logprobs.""" + state = _make_state( + return_logprob=True, top_logprobs_num=2, token_ids_logprob=[42] + ) + recv_obj = SimpleNamespace( + input_token_logprobs_val=None, + output_token_logprobs_val=None, + input_top_logprobs_val=None, + input_top_logprobs_val_flat=None, + output_top_logprobs_val=None, + input_token_ids_logprobs_val=None, + output_token_ids_logprobs_val=None, + ) + meta_info = {} + + _TokenizerManagerStub().convert_logprob_style( + meta_info, + state, + top_logprobs_num=2, + token_ids_logprob=[42], + return_text_in_logprobs=False, + recv_obj=recv_obj, + recv_obj_index=0, + ) + + for key in ( + "input_top_logprobs", + "output_top_logprobs", + "input_token_ids_logprobs", + "output_token_ids_logprobs", + ): + self.assertEqual(meta_info[key], []) + def _make_batch_token_id_output(**overrides) -> BatchTokenIDOutput: """A two-request BatchTokenIDOutput with the required fields stubbed."""