[Feature] Add per-item candidate token scoring and calibration

examples/runtime/semantic_scoring/semif.py

@@ -0,0 +1,72 @@
+"""Score SemIf JSONL decisions using its original prompts and an SGLang server."""
+
+import argparse
+import json
+from pathlib import Path
+
+import requests
+from semif_phase1.direct import PROMPT_VERSION, encode_prompt
+from transformers import AutoTokenizer
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--base-url", default="http://127.0.0.1:30000")
+    parser.add_argument("--model", default="Qwen/Qwen3.5-4B")
+    parser.add_argument("--revision", required=True)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--batch-size", type=int, default=32)
+    parser.add_argument("--max-tokens", type=int, default=4096)
+    parser.add_argument("--temperature", type=float, default=1.0)
+    args = parser.parse_args()
+    if args.batch_size < 1:
+        parser.error("--batch-size must be positive")
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision)
+    rows = [
+        json.loads(line) for line in args.input.read_text().splitlines() if line.strip()
+    ]
+    with args.output.open("x") as output, requests.Session() as session:
+        for start in range(0, len(rows), args.batch_size):
+            batch = rows[start : start + args.batch_size]
+            encoded = [encode_prompt(tokenizer, row, args.max_tokens) for row in batch]
+            response = session.post(
+                args.base_url.rstrip("/") + "/v1/score",
+                json={
+                    "model": args.model,
+                    "query": [],
+                    "items": [ids for ids, _, _ in encoded],
+                    "label_token_ids": [slots for _, slots, _ in encoded],
+                    "apply_softmax": True,
+                    "temperature": args.temperature,
+                    "return_token_logprobs": True,
+                },
+                timeout=300,
+            )
+            response.raise_for_status()
+            result = response.json()
+            for row, (ids, _, prompt_hash), probabilities, logprobs in zip(
+                batch, encoded, result["scores"], result["token_logprobs"], strict=True
+            ):
+                output.write(
+                    json.dumps(
+                        {
+                            "id": row["id"],
+                            "option_ids": [option["id"] for option in row["options"]],
+                            "probabilities": probabilities,
+                            "token_logprobs": logprobs,
+                            "input_tokens": len(ids),
+                            "prompt_sha256": prompt_hash,
+                            "prompt_version": PROMPT_VERSION,
+                            "temperature": args.temperature,
+                            "readout": "sglang candidate token logprobs",
+                        },
+                        allow_nan=False,
+                    )
+                    + "\n"
+                )
+
+
+if __name__ == "__main__":
+    main()

python/sglang/srt/entrypoints/engine_score_mixin.py

@@ -30,13 +30,15 @@ def score(
         self,
         query: Optional[Union[str, List[int]]] = None,
         items: Optional[Union[str, List[str], List[List[int]]]] = None,
-        label_token_ids: Optional[List[int]] = None,
+        label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
         apply_softmax: bool = False,
         item_first: bool = False,
         embed_override_token_id: Optional[int] = None,
         query_embed_overrides: Optional[List[torch.Tensor]] = None,
         item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
         return_pooled_hidden_states: bool = False,
+        temperature: float = 1.0,
+        return_token_logprobs: bool = False,
     ) -> ScoreResult:
         """
         Score items against a query using the loaded model.
@@ -54,8 +56,8 @@ def score(
         Args:
             query: The query text or pre-tokenized token IDs.
             items: The item text(s) or pre-tokenized token IDs.
-            label_token_ids: Token IDs to score (required for CausalLM; ignored for
-                SequenceClassification).
+            label_token_ids: Shared token IDs or one list per item, preserving candidate
+                order (required for CausalLM; ignored for SequenceClassification).
             apply_softmax: Whether to normalize scores using softmax.
             item_first: If True, prepend items before query (single-item mode only).
             embed_override_token_id: Placeholder token ID used to locate override positions.
@@ -64,6 +66,10 @@ def score(
             return_pooled_hidden_states: Whether to include raw pooled transformer
                 hidden states (before the task head) in the result. Only supported
                 for non-generation models (SequenceClassification, RewardModel).
+            temperature: Positive calibration temperature, applied before candidate/class
+                softmax. Requires apply_softmax=True when different from 1.
+            return_token_logprobs: Include uncalibrated full-vocabulary logprobs for
+                each item's candidates. Only supported for CausalLM, not raw logits.
 
         Returns:
             ScoreResult with scores (one list per item), prompt token count, and
@@ -81,20 +87,24 @@ def score(
                 item_embed_overrides=item_embed_overrides,
                 request=None,
                 return_pooled_hidden_states=return_pooled_hidden_states,
+                temperature=temperature,
+                return_token_logprobs=return_token_logprobs,
             )
         )
 
     async def async_score(
         self,
         query: Optional[Union[str, List[int]]] = None,
         items: Optional[Union[str, List[str], List[List[int]]]] = None,
-        label_token_ids: Optional[List[int]] = None,
+        label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
         apply_softmax: bool = False,
         item_first: bool = False,
         embed_override_token_id: Optional[int] = None,
         query_embed_overrides: Optional[List[torch.Tensor]] = None,
         item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
         return_pooled_hidden_states: bool = False,
+        temperature: float = 1.0,
+        return_token_logprobs: bool = False,
     ) -> ScoreResult:
         """Asynchronous version of score(). See score() for full documentation."""
         return await self.tokenizer_manager.score_request(
@@ -108,4 +118,6 @@ async def async_score(
             item_embed_overrides=item_embed_overrides,
             request=None,
             return_pooled_hidden_states=return_pooled_hidden_states,
+            temperature=temperature,
+            return_token_logprobs=return_token_logprobs,
         )

python/sglang/srt/entrypoints/openai/protocol.py

@@ -1397,10 +1397,12 @@ class ScoringRequest(BaseModel):
     item_embed_overrides: Optional[List[Optional[List[List[float]]]]] = (
         None  # [num_items][num_item_embed_overrides][hidden_size]
     )
-    label_token_ids: Optional[List[int]] = (
-        None  # Token IDs to compute probabilities for
+    label_token_ids: Optional[Union[List[int], List[List[int]]]] = (
+        None  # shared candidates or one candidate list per item
     )
     apply_softmax: bool = False
+    temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
+    return_token_logprobs: bool = False
     item_first: bool = False
     return_pooled_hidden_states: bool = False
     model: str = DEFAULT_MODEL_NAME
@@ -1411,6 +1413,7 @@ class ScoringResponse(BaseModel):
         List[float]
     ]  # List of lists of probabilities, each in the order of label_token_ids
     pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
+    token_logprobs: Optional[List[List[float]]] = None
     model: str
     usage: Optional[UsageInfo] = None
     object: str = "scoring"

python/sglang/srt/entrypoints/openai/serving_score.py

@@ -78,6 +78,8 @@ async def _handle_non_streaming_request(
                 item_embed_overrides=item_embed_overrides,
                 request=raw_request,
                 return_pooled_hidden_states=request.return_pooled_hidden_states,
+                temperature=request.temperature,
+                return_token_logprobs=request.return_token_logprobs,
             )
 
             phs_as_lists = None
@@ -90,6 +92,7 @@ async def _handle_non_streaming_request(
             response = ScoringResponse(
                 scores=result.scores,
                 pooled_hidden_states=phs_as_lists,
+                token_logprobs=result.token_logprobs,
                 model=request.model,
                 usage=UsageInfo(
                     prompt_tokens=result.prompt_tokens,