From 28175af67ad6d78923814000b873fca1d4abbe00 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Mon, 20 Jul 2026 17:40:13 -0700 Subject: [PATCH 01/20] [Cookbook] Add poolside/Laguna-S-2.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 118B hybrid-SWA MoE (8B active, 1M context) — config-driven page with H200 / B300 / GB300 × BF16 / FP8 / NVFP4 / INT4 × low-latency (DFlash) / high-throughput. All cells marked unverified; benchmarks pending. Strips NEW from Laguna-XS-2.1 and Laguna-M.1 siblings. Co-Authored-By: Claude Sonnet 4.6 --- .../autoregressive/Poolside/Laguna-M.1.mdx | 1 - .../autoregressive/Poolside/Laguna-S-2.1.mdx | 232 ++++++++ .../autoregressive/Poolside/Laguna-XS-2.1.mdx | 1 - docs_new/cookbook/autoregressive/intro.mdx | 2 +- docs_new/docs.json | 1 + .../poolside/laguna-s21-benchmarks.jsx | 42 ++ .../snippets/configs/poolside/laguna-s21.jsx | 548 ++++++++++++++++++ 7 files changed, 824 insertions(+), 3 deletions(-) create mode 100644 docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx create mode 100644 docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx create mode 100644 docs_new/src/snippets/configs/poolside/laguna-s21.jsx diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx index ea643df4adcc..32976dcbe10a 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx @@ -1,7 +1,6 @@ --- title: Laguna-M.1 description: "Deploy poolside's Laguna-M.1 — a 225B-parameter Mixture-of-Experts model (23B active) for agentic coding — with SGLang on NVIDIA H200, B200, B300, GB200, and GB300, across BF16, FP8, and NVFP4." -tag: NEW --- ## Deployment diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx new file mode 100644 index 000000000000..ca20b871c3ab --- /dev/null +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -0,0 +1,232 @@ +--- +title: Laguna-S-2.1 +description: "Deploy poolside's Laguna-S-2.1 — a 118B hybrid-SWA Mixture-of-Experts model (8B active) for agentic coding — with SGLang on NVIDIA H200, B300, and GB300 in BF16, FP8, NVFP4, and INT4." +tag: NEW +--- + +## Deployment + + + + + +Laguna-S-2.1 uses the same `laguna` model architecture as [Laguna-XS-2.1](./Laguna-XS-2.1), which is fully supported in SGLang `main`. The model ships custom config code on the Hub, so `--trust-remote-code` is required (included in the launch commands). + + + + + +```bash Command +pip install -U uv +uv venv --python 3.12 && source .venv/bin/activate + +git clone https://github.com/sgl-project/sglang.git +cd sglang +uv pip install -e python +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:dev +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick your hardware + quantization + strategy to generate the launch command. The two serving strategies cover the common operating points: + +- **Low-latency** — DFlash speculative decoding with a matched draft model. Pick for chat and interactive agents. +- **High-throughput** — plain serving. Best for batch workloads, where speculation's draft + rejection overhead costs more than it saves. + +On the 8-GPU HGX platforms (H200 / B300) all quantizations run `--tp 8`. The 4-GPU GB300 node runs `--tp 4` throughout. NVFP4 is Blackwell-only (B300 / GB300 only). + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/poolside/laguna-s21.jsx"; +import { benchmarks } from "/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx"; + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations that have been signed off; the Playground lets you turn on additional knobs (TP degree, parsers) on top of whichever cell the Deploy panel is currently showing. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +[Laguna-S-2.1](https://huggingface.co/poolside/Laguna-S-2.1) is an open-weight **118B-parameter** hybrid sliding-window-attention MoE model (**~8B active per token**) from [poolside](https://poolside.ai), built for agentic coding and long-horizon software engineering. It sits between [Laguna XS 2.1](./Laguna-XS-2.1) (33B/3B active) and Laguna M.1 (222B/23B active) in the Laguna family. + +**Key Features:** + +- **Sparse MoE**: 48 layers, 256 routed experts, top-10 routing, plus 1 shared expert. +- **Hybrid attention**: 36 sliding-window layers (window 512) interleaved with 12 full-attention layers (1:3 global-to-SWA ratio); 8 KV heads, head dim 128; per-head sigmoid output gating with per-layer-type rotary scales. +- **Long context**: 1,048,576 tokens. +- **DFlash drafts**: matched draft models ship per quantization for low-latency serving. +- **Hybrid reasoning**: `…` toggled per request via `chat_template_kwargs={"enable_thinking": …}`. + +**Available quantizations:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PrecisionTarget modelDraft model
BF16[`poolside/Laguna-S-2.1`](https://huggingface.co/poolside/Laguna-S-2.1)[`poolside/Laguna-S-2.1-DFlash`](https://huggingface.co/poolside/Laguna-S-2.1-DFlash)
FP8[`poolside/Laguna-S-2.1-FP8`](https://huggingface.co/poolside/Laguna-S-2.1-FP8)[`poolside/Laguna-S-2.1-DFlash-FP8`](https://huggingface.co/poolside/Laguna-S-2.1-DFlash-FP8)
NVFP4[`poolside/Laguna-S-2.1-NVFP4`](https://huggingface.co/poolside/Laguna-S-2.1-NVFP4)[`poolside/Laguna-S-2.1-DFlash-NVFP4`](https://huggingface.co/poolside/Laguna-S-2.1-DFlash-NVFP4)
INT4[`poolside/Laguna-S-2.1-INT4`](https://huggingface.co/poolside/Laguna-S-2.1-INT4)[`poolside/Laguna-S-2.1-DFlash-INT4`](https://huggingface.co/poolside/Laguna-S-2.1-DFlash-INT4)
+ +The drafts are small BF16 models, each *calibrated against its quantized target* — always pair a target with its matched draft (mixing precisions degrades accept-length). + +**License:** [OpenMDW-1.1](https://openmdw.ai/) + +**Resources:** [Hugging Face](https://huggingface.co/poolside/Laguna-S-2.1) · [Technical report](https://poolside.ai/assets/laguna/laguna-m1-xs2-technical-report.pdf) · [API platform](https://platform.poolside.ai) + +## 2. Configuration Tips + +**Attention backend** + +Leave `--attention-backend` unset for High-throughput cells — auto-select is correct (`fa3` on Hopper, `trtllm_mha` on Blackwell). With DFlash active, auto-select instead falls back to `flashinfer`, which breaks this hybrid-SWA model at `tp ≥ 4` on Blackwell (reproduced on Laguna-XS-2.1, greedy GSM8K 76% → 28%), so the Low-latency commands pin the target backend explicitly. Leave `--speculative-draft-attention-backend` unset. Other attention backend choices have not been fully validated on Laguna; keep the default. + +**FP8 shared expert** + +The generated FP8 commands include `SGLANG_SHARED_EXPERT_TP1=1` as a precaution. The Laguna family's FP8 checkpoint block-quantizes the shared expert; this env var replicates it instead of TP-sharding it (consistent with the Laguna-XS-2.1 verified recipe). Remove it only after confirming shared-expert sharding works on this model. + +**DFlash memory** + +Low-latency cells carry `--mem-fraction-static 0.7`: at larger TP degrees the default fraction may OOM in the draft vocab all-gather. Dense cells use the default heuristic. + +**Chat template** + +On transformers ≥ 5.10 the standalone `chat_template.jinja` auto-loads — no flag needed (the server logs `Auto-detected template features: reasoning_parser=poolside_v1, ...`). On older transformers (≤ ~5.8) pass `--chat-template /chat_template.jinja` explicitly. + +**Thinking** + +Off by default; opt in per request with `extra_body={"chat_template_kwargs": {"enable_thinking": True}}`. The template gates on `enable_thinking` — the generic `thinking` key is ignored. + +**Served model id** + +The server registers the model under whatever you pass to `--model-path`; a client's `model` field must match it (`poolside/Laguna-S-2.1`, or the `-FP8` / `-NVFP4` / `-INT4` id). + +## 3. Advanced Usage + +### 3.1 DFlash Speculative Decoding + +DFlash is a block-wise speculative decoder: the draft proposes a block of tokens and the target verifies the whole block in one forward pass — output quality is the target's by construction. The speedup lever is **accept-length**, the number of draft tokens surviving verification per target step. + +Best for interactive / few-stream serving. Under batch-saturated load prefer High-throughput: once the GPU is compute-bound, draft + rejected-token overhead costs aggregate throughput. The generated commands always pair the draft calibrated for the selected target precision. + +### 3.2 Reasoning + +Launch with `--reasoning-parser poolside_v1` (baked into every generated command). Reasoning is opt-in via `enable_thinking=True`; the `` trace lands in `message.reasoning_content`, separate from the final answer in `message.content`. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="poolside/Laguna-S-2.1", + messages=[{"role": "user", "content": "What is 15% of 240? Explain briefly."}], + max_tokens=4096, + extra_body={"chat_template_kwargs": {"enable_thinking": True}}, +) + +message = response.choices[0].message +print("=============== Reasoning ===============") +print(message.reasoning_content) +print("=============== Answer ==================") +print(message.content) +``` + + + + +Give generous `max_tokens` when thinking is enabled — hard problems regularly reason +for thousands of tokens. Keep thinking off for short-form tasks. + + +### 3.3 Tool Calling + +Launch with `--tool-call-parser poolside_v1` (baked into every generated command). The parser converts Laguna's `` output into the standard OpenAI `tool_calls` structure. Tool calling works with reasoning off (the default). + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] + +response = client.chat.completions.create( + model="poolside/Laguna-S-2.1", + messages=[{"role": "user", "content": "What's the weather in Beijing?"}], + tools=tools, +) + +message = response.choices[0].message +if message.tool_calls: + for call in message.tool_calls: + print(f"Tool: {call.function.name}") + print(f"Args: {call.function.arguments}") +``` + + diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx index 5423eeda2ca7..7d0ff41fafe2 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx @@ -1,7 +1,6 @@ --- title: Laguna-XS-2.1 description: "Deploy poolside's Laguna-XS-2.1 — a 33B hybrid-SWA Mixture-of-Experts model (3B active) for agentic coding — with SGLang on NVIDIA H200, B300, and GB300 in BF16, FP8, NVFP4, and INT4." -tag: NEW --- ## Deployment diff --git a/docs_new/cookbook/autoregressive/intro.mdx b/docs_new/cookbook/autoregressive/intro.mdx index 5dac029219dc..3731e8e3f2ce 100644 --- a/docs_new/cookbook/autoregressive/intro.mdx +++ b/docs_new/cookbook/autoregressive/intro.mdx @@ -160,7 +160,7 @@ metatags: diff --git a/docs_new/docs.json b/docs_new/docs.json index 1c2d81bcf712..c01f7bb5e3aa 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -1172,6 +1172,7 @@ { "group": "Poolside", "pages": [ + "cookbook/autoregressive/Poolside/Laguna-S-2.1", "cookbook/autoregressive/Poolside/Laguna-XS-2.1", "cookbook/autoregressive/Poolside/Laguna-M.1", "cookbook/autoregressive/Poolside/Laguna-XS.2" diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx new file mode 100644 index 000000000000..e48de4c5373a --- /dev/null +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -0,0 +1,42 @@ +// Laguna-S-2.1 benchmarks — one entry per cell `match` (same 5 keys as laguna-s21.jsx cells). +// +// All entries below are PENDING stubs — no measurements yet. Fill in real numbers as cells +// are verified (set verified:true in laguna-s21.jsx and add accuracy + sglang_version here). +// The benchmark card renders "pending" for bare { match } stubs. +// +// Target eval set: +// GSM8K — sgl-eval run gsm8k, full 1319 questions, greedy/non-thinking. +// AIME25 — sgl-eval run aime25, 30 problems × 16 repeats, temperature 1.0, top-p 0.95, +// max-tokens 64000, 128 threads. Requires serving with enable_thinking=true +// (Laguna's template gates on enable_thinking, not the generic 'thinking' key; +// see Configuration Tips: Thinking in the cookbook page). + +export const benchmarks = [ + // ===== H200 (8-GPU HGX, --tp 8) — PENDING ===== + { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, + + // ===== B300 (8-GPU HGX, --tp 8) — PENDING ===== + { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, + + // ===== GB300 (4-GPU single node, --tp 4) — PENDING ===== + { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, +]; diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx new file mode 100644 index 000000000000..570e26588f09 --- /dev/null +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -0,0 +1,548 @@ +// Laguna-S-2.1 (poolside) — config-driven cookbook page. +// Consumed by the shared _deployment.jsx + _playground.jsx engines (no model code there). +// +// Architecture: 118B total / ~8B active per token (top-10 routing over 256 routed experts +// + 1 shared expert). 48 layers: 12 full-attention + 36 sliding-window (window 512). +// 1,048,576-token context window. Per-head sigmoid output gating on attention. +// +// Build: uses the same `laguna` model type as Laguna-XS-2.1. All support is in SGLang main. +// The model ships custom config code on the Hub, so --trust-remote-code is required. +// +// Attention backend (same rules as Laguna-XS-2.1 — hybrid-SWA is backend-sensitive): +// - Dense (High-Throughput): leave --attention-backend UNSET. Auto-select is correct: +// fa3 on Hopper (H200), trtllm_mha on Blackwell (B300/GB300). +// - DFlash (Low-Latency): MUST pin the target backend explicitly — with a speculative +// algorithm active, the resolver falls back to flashinfer, which breaks this +// hybrid-SWA model at tp≥4 on Blackwell (reproduced on Laguna-XS-2.1 at GB300, +// GSM8K 76%→28%). Pin fa3 on H200, trtllm_mha on B300/GB300. +// - NEVER use --attention-backend triton for Laguna (broken SWA handling). +// +// TP sizing (118B BF16 ≈ 236 GB; FP8 ≈ 118 GB; NVFP4/INT4 ≈ 59 GB): +// - H200 8-GPU (8×141 GB): --tp 8 for all quants (29.5 GB weights/GPU for BF16). +// Unlike Laguna-XS-2.1, the much larger moe_intermediate_size of this 118B model +// is expected to allow plain TP=8 for FP8 and INT4 without --ep-size (the XS-2.1 +// constraint was 512/8=64 < block_size=128; that does not hold at the larger expert +// width in S-2.1). FP8 cells include SGLANG_SHARED_EXPERT_TP1=1 as a precaution +// (unverified whether the shared expert needs replication here). +// - B300 8-GPU (8×288 GB): same --tp 8 reasoning. +// - GB300 4-GPU (4×288 GB): --tp 4 throughout (59 GB/GPU for BF16, ≥229 GB KV budget). +// +// NVFP4 is Blackwell-only → no h200×nvfp4 cells. +// +// DFlash memory: Low-Latency cells carry --mem-fraction-static 0.7 (consistent with +// Laguna-XS-2.1; the larger model makes draft-vocab all-gather OOM more likely). +// +// verified:false on all cells — commands follow the XS-2.1 verified shape + the TP +// scaling above but have not yet been run against Laguna-S-2.1. Replace with +// verified:true after confirming each cell with a full GSM8K pass. +// +// Draft/target precision ALWAYS matches: each quantized target pairs with the DFlash +// draft calibrated for it (poolside/Laguna-S-2.1-DFlash[-FP8/-NVFP4/-INT4]). + +export const config = { + modelName: "Laguna-S-2.1", + + supportedHardware: ["h200", "b300", "gb300"], + + variants: [ + { id: "default", label: "Default" }, + ], + + quantizations: [ + { id: "bf16", label: "BF16" }, + { id: "fp8", label: "FP8" }, + { id: "nvfp4", label: "NVFP4" }, + { id: "int4", label: "INT4" }, + ], + + // Two operating points: + // low-latency = DFlash speculative decoding (matched-precision draft) — interactive / + // few-stream serving. + // high-throughput = plain serving (no speculation) — batch-saturated workloads, where + // speculation's draft+rejection overhead costs more than it saves. + strategies: [ + { id: "low-latency", label: "Low-latency" }, + { id: "high-throughput", label: "High-throughput" }, + ], + + nodesOptions: [ + { id: "single", label: "Single Node" }, + ], + + modelNames: { + "default|bf16": "poolside/Laguna-S-2.1", + "default|fp8": "poolside/Laguna-S-2.1-FP8", + "default|nvfp4": "poolside/Laguna-S-2.1-NVFP4", + "default|int4": "poolside/Laguna-S-2.1-INT4", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, + // GSM8K is the required accuracy sanity on every verified cell, via sgl-eval. + accuracy: { + gsm8k_pct: +`# pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --num-threads 128`, + // Laguna's template gates on enable_thinking (not the generic 'thinking' key). + // To run AIME25 with thinking, serve with a copy of the model's chat template + // whose enable_thinking default is flipped to true, passed via --chat-template. + aime25_pct: +`# pip install git+https://github.com/sgl-project/sgl-eval +# Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking). +sgl-eval run aime25 \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --n-repeats 16 --max-tokens 64000 \\ + --temperature 1.0 --top-p 0.95 --thinking \\ + --num-threads 128`, + }, + numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 }, + }, + + // No variant-wide accuracy default; real numbers are per-cell in laguna-s21-benchmarks.jsx. + defaultAccuracy: { + default: { gsm8k_pct: null, aime25_pct: null }, + }, + + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ["aime25_pct", "AIME25", "%"], + ], + + dockerImages: { + h200: "lmsysorg/sglang:dev", + b300: "lmsysorg/sglang:dev", + gb300: "lmsysorg/sglang:dev", + }, + + github: { + cookbookModel: "poolside/Laguna-S-2.1", + }, + + playgroundFeatures: { + + // Hybrid-SWA GQA model (8 KV heads) — TP shards cleanly at 1/2/4/8. + // No DP-Attention / CP knobs: unvalidated on this model family. + attention: { + knobs: [ + { id: "tp", label: "TP", values: [null, 1, 2, 4, 8] }, + ], + }, + + // Reasoning + tool-call parsers (poolside_v1, same family as Laguna-XS-2.1). + // Baked into every Deploy cell. The chat template auto-detects both on + // transformers ≥ 5.10 (`Auto-detected template features: reasoning_parser=poolside_v1, + // tool_call_parser=poolside_v1`), so these are explicit-but-redundant there. + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser poolside_v1" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser poolside_v1" }, + ], + }, + }, + + // Cells: (h200 × {bf16,fp8,int4} + b300/gb300 × {bf16,fp8,nvfp4,int4}) × {low-latency, high-throughput}. + // All verified:false — not yet measured on Laguna-S-2.1. + // NVFP4 is Blackwell-only: no h200×nvfp4 cells. + cells: [ + + // ══════════════ NVIDIA Hopper H200 (8-GPU HGX, --tp 8) ══════════════ + // Dense auto-selects fa3 on Hopper (no flag needed). LL pins fa3 (DFlash-safe on + // Hopper; with a spec algorithm active, auto would fall back to flashinfer). + { + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend fa3", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // SGLANG_SHARED_EXPERT_TP1=1 included as precaution — the shared expert is likely + // also block-FP8-quantized (same architecture family as XS-2.1). Unverified. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: false, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend fa3", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend fa3", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ══════════════ NVIDIA Blackwell Ultra B300 (8-GPU HGX) ══════════════ + // Dense auto-selects trtllm_mha on Blackwell (no flag). LL MUST pin trtllm_mha — + // with DFlash active, auto falls back to flashinfer, which is broken for this + // hybrid-SWA model at tp≥4 (reproduced + bisected on GB300 with Laguna-XS-2.1). + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: false, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-NVFP4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node, --tp 4) ══════════════ + // BF16 118B at tp 4 = 59 GB/GPU (out of 288 GB): ample KV budget. + // Dense auto-selects trtllm_mha on Blackwell. LL pins trtllm_mha (same flashinfer + // concern as B300 above). + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-NVFP4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +}; From 8895bd25d62e823c910dc2009eb2dee98c4fafa3 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 00:55:48 -0700 Subject: [PATCH 02/20] [Cookbook] Laguna-S-2.1: add H200 benchmarks + fix commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mark all 6 H200 cells verified (sglang 0.5.15.post1, 2026-07-20) - Add --sampling-defaults openai to all cells (generation_config top_k=0 causes HTTP 400 on non-greedy requests without it) - Add --mem-fraction-static 0.80 to H200 BF16 high-throughput (default/ 0.88 OOM at cuda-graph capture / NCCL alloc at high concurrency) - Confirm SGLANG_SHARED_EXPERT_TP1=1 for FP8 only (INT4 shared expert stays bf16, no flag needed) - Fill H200 accuracy: GSM8K 93.18–95.00%; AIME25 FP8 67.29% / INT4 67.71% (BF16 null pending higher max_tokens re-run due to truncation) - Document FP8 DFlash draft rope_theta issue and BF16 reasoning-length caveat in Configuration Tips Co-Authored-By: Claude Sonnet 4.6 --- .../autoregressive/Poolside/Laguna-S-2.1.mdx | 20 ++- .../poolside/laguna-s21-benchmarks.jsx | 79 +++++++++--- .../snippets/configs/poolside/laguna-s21.jsx | 115 ++++++++++++++---- 3 files changed, 169 insertions(+), 45 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index ca20b871c3ab..e36275abfa1c 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -127,13 +127,29 @@ The drafts are small BF16 models, each *calibrated against its quantized target* Leave `--attention-backend` unset for High-throughput cells — auto-select is correct (`fa3` on Hopper, `trtllm_mha` on Blackwell). With DFlash active, auto-select instead falls back to `flashinfer`, which breaks this hybrid-SWA model at `tp ≥ 4` on Blackwell (reproduced on Laguna-XS-2.1, greedy GSM8K 76% → 28%), so the Low-latency commands pin the target backend explicitly. Leave `--speculative-draft-attention-backend` unset. Other attention backend choices have not been fully validated on Laguna; keep the default. +**Sampling defaults** + +`generation_config.json` ships with `top_k=0`. With the default `--sampling-defaults model`, sglang applies `top_k=0` at serving time and rejects any non-greedy sampling request (AIME25, chat sampling) with HTTP 400. All generated commands include `--sampling-defaults openai` to use OpenAI API defaults instead — greedy inference (GSM8K) is unaffected. + +**BF16 memory on H200** + +BF16 weights consume ~29.5 GB/GPU on H200 (about 2× FP8), leaving less headroom for CUDA-graph capture and NCCL allocations at high concurrency. The default `mem-fraction-static 0.893` OOMs at cuda-graph capture; `0.88` crashes under NCCL at `conc ≥ 1024`. The High-throughput BF16 command carries `--mem-fraction-static 0.80`. FP8 and INT4 use the default heuristic. On B300 and GB300 (288 GB/GPU) BF16 has ample headroom and no adjustment is needed. + **FP8 shared expert** -The generated FP8 commands include `SGLANG_SHARED_EXPERT_TP1=1` as a precaution. The Laguna family's FP8 checkpoint block-quantizes the shared expert; this env var replicates it instead of TP-sharding it (consistent with the Laguna-XS-2.1 verified recipe). Remove it only after confirming shared-expert sharding works on this model. +`SGLANG_SHARED_EXPERT_TP1=1` is required for FP8 cells: the FP8 checkpoint block-quantizes the shared expert (128×128 scales), which cannot TP-shard cleanly. This env var replicates the shared expert instead of sharding it. INT4 keeps the shared expert in BF16 (no flag needed); BF16 is unquantized. + +**FP8 DFlash draft** + +The HF checkpoint `poolside/Laguna-S-2.1-DFlash-FP8` (as of 2026-07-20) is missing `rope_theta` in `rope_parameters`. If the server crashes at draft-model load, patch the draft config locally by adding `"rope_theta"` to that block. The BF16 and INT4 draft models are unaffected. **DFlash memory** -Low-latency cells carry `--mem-fraction-static 0.7`: at larger TP degrees the default fraction may OOM in the draft vocab all-gather. Dense cells use the default heuristic. +Low-latency cells carry `--mem-fraction-static 0.7` (sufficient even for BF16 on H200). Dense cells use the default heuristic (except BF16 on H200 — see above). + +**BF16 reasoning length** + +BF16 reasons approximately 2× longer than FP8/INT4 on AIME25 (median 34.8 k vs 16.9 k tokens), consistently truncating at `max_tokens=64000`. FP8/INT4 truncate at ≈ 2%. For a valid BF16 AIME25 score, serve with `max_tokens ≥ 131072` (the model supports a 1 M context window). **Chat template** diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index e48de4c5373a..860f65e8a297 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -1,26 +1,69 @@ // Laguna-S-2.1 benchmarks — one entry per cell `match` (same 5 keys as laguna-s21.jsx cells). // -// All entries below are PENDING stubs — no measurements yet. Fill in real numbers as cells -// are verified (set verified:true in laguna-s21.jsx and add accuracy + sglang_version here). -// The benchmark card renders "pending" for bare { match } stubs. +// H200 cells: REAL measured values (sglang 0.5.15.post1, lmsysorg/sglang:latest, +// 8×H200 TP=8, 2026-07-20). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. +// AIME25 = sgl-eval run aime25, 30 problems × 16 repeats, temp 1.0, top_p 0.95, +// max_tokens 64000, 128 threads, thinking ON via enable_thinking (not the generic +// 'thinking' key). AIME25 measured on high-throughput cells only. // -// Target eval set: -// GSM8K — sgl-eval run gsm8k, full 1319 questions, greedy/non-thinking. -// AIME25 — sgl-eval run aime25, 30 problems × 16 repeats, temperature 1.0, top-p 0.95, -// max-tokens 64000, 128 threads. Requires serving with enable_thinking=true -// (Laguna's template gates on enable_thinking, not the generic 'thinking' key; -// see Configuration Tips: Thinking in the cookbook page). +// BF16 AIME25 invalidity: BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs +// 16.9k tokens); 33.8% of responses truncated at max_tokens=64000. aime25_pct is +// left null — re-run with max_tokens ≥ 128k for a valid score. +// +// B300/GB300 cells: PENDING (bare match stubs — benchmark card renders "pending"). export const benchmarks = [ - // ===== H200 (8-GPU HGX, --tp 8) — PENDING ===== - { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, + // ===== H200 (8-GPU HGX, tp 8) — ✅ REAL, full GSM8K ===== + { + // ✅ REAL — 8×H200, BF16 dense, tp8, fa3 (Hopper auto-select), mem-frac 0.80. + // GSM8K 93.18%. AIME25 pending re-run with higher max_tokens (truncation at 64k). + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 93.18, aime25_pct: null }, + }, + { + // ✅ REAL — 8×H200, BF16 + DFlash (matched bf16 draft), tp8, fa3, mem-frac 0.7. + // GSM8K 93.33%. AIME25 not measured on low-latency cells. + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, + }, + { + // ✅ REAL — 8×H200, FP8 dense, tp8, fa3, SGLANG_SHARED_EXPERT_TP1=1. + // GSM8K 94.24%, AIME25 67.29% pass@1 (avg-16, pass@16 86.7%, majority@16 76.7%). + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.24, aime25_pct: 67.29 }, + }, + { + // ✅ REAL — 8×H200, FP8 + DFlash (fp8-calibrated draft, patched rope_theta), tp8, fa3. + // GSM8K 94.47%. AIME25 not measured on low-latency cells. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.47, aime25_pct: null }, + }, + { + // ✅ REAL — 8×H200, INT4 dense, tp8, fa3. INT4 shared expert stays bf16 (no flag). + // GSM8K 95.00%, AIME25 67.71% pass@1 (avg-16, pass@16 86.7%, majority@16 80.0%). + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 95.00, aime25_pct: 67.71 }, + }, + { + // ✅ REAL — 8×H200, INT4 + DFlash (int4-calibrated draft), tp8, fa3. + // GSM8K 94.24%. AIME25 not measured on low-latency cells. + match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, + }, - // ===== B300 (8-GPU HGX, --tp 8) — PENDING ===== + // ===== B300 (8-GPU HGX, tp 8) — PENDING ===== { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, @@ -30,7 +73,7 @@ export const benchmarks = [ { match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, { match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, - // ===== GB300 (4-GPU single node, --tp 4) — PENDING ===== + // ===== GB300 (4-GPU single node, tp 4) — PENDING ===== { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index 570e26588f09..a6618c2b96d5 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -17,24 +17,45 @@ // GSM8K 76%→28%). Pin fa3 on H200, trtllm_mha on B300/GB300. // - NEVER use --attention-backend triton for Laguna (broken SWA handling). // +// --sampling-defaults openai (ALL cells): +// generation_config.json ships with top_k=0. With --sampling-defaults model (the +// default), sglang applies top_k=0 at serving time and rejects any non-greedy sampling +// request with HTTP 400. --sampling-defaults openai uses OpenAI API defaults instead, +// ignoring the model-level top_k=0. Greedy (GSM8K) is unaffected; non-greedy +// (AIME25, chat sampling) requires this flag. Verified 2026-07-20 on 8×H200. +// // TP sizing (118B BF16 ≈ 236 GB; FP8 ≈ 118 GB; NVFP4/INT4 ≈ 59 GB): // - H200 8-GPU (8×141 GB): --tp 8 for all quants (29.5 GB weights/GPU for BF16). // Unlike Laguna-XS-2.1, the much larger moe_intermediate_size of this 118B model // is expected to allow plain TP=8 for FP8 and INT4 without --ep-size (the XS-2.1 // constraint was 512/8=64 < block_size=128; that does not hold at the larger expert -// width in S-2.1). FP8 cells include SGLANG_SHARED_EXPERT_TP1=1 as a precaution -// (unverified whether the shared expert needs replication here). +// width in S-2.1). Confirmed on H200 for BF16, FP8, INT4 (2026-07-20). // - B300 8-GPU (8×288 GB): same --tp 8 reasoning. // - GB300 4-GPU (4×288 GB): --tp 4 throughout (59 GB/GPU for BF16, ≥229 GB KV budget). // +// BF16 memory on H200: +// BF16 (29.5 GB/GPU) consumes ~2× more weight memory than FP8 on H200, leaving less +// headroom for CUDA-graph capture and NCCL allocs. Default mem-fraction 0.893 → OOM +// at cuda-graph capture; 0.88 → NCCL crash at high concurrency. --mem-fraction-static +// 0.80 is required for BF16 high-throughput on H200. Low-latency already carries 0.7. +// B300/GB300 (288 GB/GPU) are unaffected and use the default heuristic. +// +// SGLANG_SHARED_EXPERT_TP1=1 (FP8 cells only): +// Confirmed required: the FP8 checkpoint block-quantizes the shared expert (128×128 +// scales, can't TP-shard cleanly). INT4 keeps the shared expert in BF16 (no flag +// needed). BF16 is unquantized. Verified 2026-07-20 on 8×H200. +// // NVFP4 is Blackwell-only → no h200×nvfp4 cells. // -// DFlash memory: Low-Latency cells carry --mem-fraction-static 0.7 (consistent with -// Laguna-XS-2.1; the larger model makes draft-vocab all-gather OOM more likely). +// DFlash memory: Low-Latency cells carry --mem-fraction-static 0.7. // -// verified:false on all cells — commands follow the XS-2.1 verified shape + the TP -// scaling above but have not yet been run against Laguna-S-2.1. Replace with -// verified:true after confirming each cell with a full GSM8K pass. +// FP8 DFlash draft rope issue: +// poolside/Laguna-S-2.1-DFlash-FP8 (as of 2026-07-20) is missing rope_theta in +// rope_parameters. If the server crashes at draft model load, patch the draft config +// locally: add "rope_theta" to the rope_parameters block. INT4 draft is unaffected. +// +// verified:true = command ran and passed full GSM8K on that cell. H200 cells verified +// 2026-07-20 (sglang 0.5.15.post1). B300/GB300 cells are unverified pending measurement. // // Draft/target precision ALWAYS matches: each quantized target pairs with the DFlash // draft calibrated for it (poolside/Laguna-S-2.1-DFlash[-FP8/-NVFP4/-INT4]). @@ -105,11 +126,14 @@ sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 128`, // Laguna's template gates on enable_thinking (not the generic 'thinking' key). - // To run AIME25 with thinking, serve with a copy of the model's chat template - // whose enable_thinking default is flipped to true, passed via --chat-template. + // Serve with a copy of the model's chat template whose enable_thinking default + // is flipped to true, passed via --chat-template. + // Note: BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs 16.9k tokens) + // and routinely truncates at 64k — use max_tokens ≥ 128k for a valid BF16 score. aime25_pct: `# pip install git+https://github.com/sgl-project/sgl-eval # Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking). +# For BF16: use --max-tokens 131072 (see Configuration Tips: BF16 reasoning length). sgl-eval run aime25 \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --n-repeats 16 --max-tokens 64000 \\ @@ -129,8 +153,10 @@ sgl-eval run aime25 \\ ["aime25_pct", "AIME25", "%"], ], + // H200: lmsysorg/sglang:latest (sglang 0.5.15.post1, verified 2026-07-20). + // B300/GB300: unverified, using dev (nightly) as default. dockerImages: { - h200: "lmsysorg/sglang:dev", + h200: "lmsysorg/sglang:latest", b300: "lmsysorg/sglang:dev", gb300: "lmsysorg/sglang:dev", }, @@ -162,16 +188,24 @@ sgl-eval run aime25 \\ }, // Cells: (h200 × {bf16,fp8,int4} + b300/gb300 × {bf16,fp8,nvfp4,int4}) × {low-latency, high-throughput}. - // All verified:false — not yet measured on Laguna-S-2.1. // NVFP4 is Blackwell-only: no h200×nvfp4 cells. + // H200 cells: verified:true (measured 2026-07-20, sglang 0.5.15.post1). + // B300/GB300 cells: verified:false — commands follow the H200 verified shape + TP scaling. cells: [ - // ══════════════ NVIDIA Hopper H200 (8-GPU HGX, --tp 8) ══════════════ + // ══════════════ NVIDIA Hopper H200 (8-GPU HGX, --tp 8) — VERIFIED ══════════════ // Dense auto-selects fa3 on Hopper (no flag needed). LL pins fa3 (DFlash-safe on // Hopper; with a spec algorithm active, auto would fall back to flashinfer). + // BF16 high-throughput requires --mem-fraction-static 0.80 on H200: default 0.893 + // OOMs at cuda-graph capture; 0.88 crashes at high concurrency under NCCL alloc. + // BF16 low-latency already carries 0.7 (sufficient). + // FP8/INT4 use default mem-fraction (their smaller weight footprint leaves enough + // headroom). { + // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 93.18%. + // --mem-fraction-static 0.80 required — default/0.88 OOMs (see header comment). match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -179,13 +213,16 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--mem-fraction-static 0.80", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, { + // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 93.33%. match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -198,15 +235,16 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, { - // SGLANG_SHARED_EXPERT_TP1=1 included as precaution — the shared expert is likely - // also block-FP8-quantized (same architecture family as XS-2.1). Unverified. + // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.24%, AIME25 67.29%. + // SGLANG_SHARED_EXPERT_TP1=1 confirmed required (FP8 block-quantizes shared expert). match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], flags: [ "--model-path {{MODEL_NAME}}", @@ -214,13 +252,16 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, { + // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.47%. + // Note: DFlash-FP8 draft may crash at load (rope_theta missing) — see header. match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], flags: [ "--model-path {{MODEL_NAME}}", @@ -233,13 +274,16 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, { + // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 95.00%, AIME25 67.71%. + // INT4 shared expert stays bf16 — no SGLANG_SHARED_EXPERT_TP1 needed. match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -247,13 +291,15 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, { + // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.24%. match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -266,15 +312,18 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, - // ══════════════ NVIDIA Blackwell Ultra B300 (8-GPU HGX) ══════════════ + // ══════════════ NVIDIA Blackwell Ultra B300 (8-GPU HGX) — UNVERIFIED ══════════════ // Dense auto-selects trtllm_mha on Blackwell (no flag). LL MUST pin trtllm_mha — // with DFlash active, auto falls back to flashinfer, which is broken for this // hybrid-SWA model at tp≥4 (reproduced + bisected on GB300 with Laguna-XS-2.1). + // B300 has 288 GB/GPU: BF16 29.5 GB/GPU leaves ≥258 GB for KV — no mem-fraction + // adjustment needed for dense cells (unlike H200). { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, @@ -285,6 +334,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -304,6 +354,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -318,6 +369,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -337,6 +389,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -351,6 +404,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -370,6 +424,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-NVFP4", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -384,6 +439,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -403,15 +459,16 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, - // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node, --tp 4) ══════════════ - // BF16 118B at tp 4 = 59 GB/GPU (out of 288 GB): ample KV budget. - // Dense auto-selects trtllm_mha on Blackwell. LL pins trtllm_mha (same flashinfer - // concern as B300 above). + // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node, --tp 4) — UNVERIFIED ══════════════ + // BF16 118B at tp 4 = 59 GB/GPU (out of 288 GB): ample KV budget, no mem-fraction + // adjustment needed for dense cells. Dense auto-selects trtllm_mha on Blackwell. + // LL pins trtllm_mha (same flashinfer concern as B300 above). { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, @@ -422,6 +479,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -441,6 +499,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -455,6 +514,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -474,6 +534,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -488,6 +549,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -507,6 +569,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-NVFP4", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -521,6 +584,7 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -540,6 +604,7 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", "--page-size 1", "--mem-fraction-static 0.7", + "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], From cefccda523ceb423ca85368d04c017f544f434c3 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 01:02:55 -0700 Subject: [PATCH 03/20] [Cookbook] Laguna-S-2.1: add GB300 benchmarks + NVFP4 broken note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mark GB300 BF16/FP8/INT4 cells verified (sglang dev build, 2026-07-20) - Add SGLANG_SHARED_EXPERT_TP1=1 to GB300 FP8 cells (confirmed required at TP=4 on S-2.1, unlike XS-2.1 where TP=4 works without it) - Fill GB300 accuracy: GSM8K 93.3–95.2% across BF16/FP8/INT4 - GB300 AIME25 left null (sgl-eval --thinking sends wrong key for Laguna) - NVFP4 on GB300: verified:false — degenerate output (repetition loop), both LL and HT; documented in config header and Configuration Tips - Fix DFlash draft rope_theta note to cover NVFP4 draft as well Co-Authored-By: Claude Sonnet 4.6 --- .../autoregressive/Poolside/Laguna-S-2.1.mdx | 10 +- .../poolside/laguna-s21-benchmarks.jsx | 81 +++++++++--- .../snippets/configs/poolside/laguna-s21.jsx | 125 +++++++++++------- 3 files changed, 147 insertions(+), 69 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index e36275abfa1c..49d58d3fa490 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -137,11 +137,15 @@ BF16 weights consume ~29.5 GB/GPU on H200 (about 2× FP8), leaving less headroom **FP8 shared expert** -`SGLANG_SHARED_EXPERT_TP1=1` is required for FP8 cells: the FP8 checkpoint block-quantizes the shared expert (128×128 scales), which cannot TP-shard cleanly. This env var replicates the shared expert instead of sharding it. INT4 keeps the shared expert in BF16 (no flag needed); BF16 is unquantized. +`SGLANG_SHARED_EXPERT_TP1=1` is required for FP8 cells on **all hardware** — confirmed on both H200 (TP=8) and GB300 (TP=4). The FP8 checkpoint block-quantizes the shared expert (128×128 scales), which cannot TP-shard cleanly at either TP degree on S-2.1. This env var replicates the shared expert instead of sharding it. INT4 keeps the shared expert in BF16 (no flag needed); BF16 is unquantized. Note: this differs from Laguna-XS-2.1 where TP=4 does not require the flag — the constraint is architecture-specific. -**FP8 DFlash draft** +**FP8 and NVFP4 DFlash drafts** -The HF checkpoint `poolside/Laguna-S-2.1-DFlash-FP8` (as of 2026-07-20) is missing `rope_theta` in `rope_parameters`. If the server crashes at draft-model load, patch the draft config locally by adding `"rope_theta"` to that block. The BF16 and INT4 draft models are unaffected. +The HF checkpoints `poolside/Laguna-S-2.1-DFlash-FP8` and `poolside/Laguna-S-2.1-DFlash-NVFP4` (as of 2026-07-20) are missing `rope_theta` in `rope_parameters`. If the server crashes at draft-model load with a `KeyError: 'rope_theta'`, patch the draft config locally: add `"rope_theta": 500000.0` to the `rope_parameters` block. The BF16 and INT4 draft models are unaffected. + +**NVFP4 on GB300** + +NVFP4 produces degenerate output on 4×GB300 (repetition loop, never emits EOS) — reproduced identically with and without DFlash spec-decode, ruling out a draft interaction. Root cause is likely the NVFP4 checkpoint or the sglang sm103 kernel path. The NVFP4 cells on GB300 remain unverified until this is resolved upstream. NVFP4 on B300 is unverified. **DFlash memory** diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index 860f65e8a297..c7e18e29f23f 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -3,20 +3,26 @@ // H200 cells: REAL measured values (sglang 0.5.15.post1, lmsysorg/sglang:latest, // 8×H200 TP=8, 2026-07-20). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. // AIME25 = sgl-eval run aime25, 30 problems × 16 repeats, temp 1.0, top_p 0.95, -// max_tokens 64000, 128 threads, thinking ON via enable_thinking (not the generic -// 'thinking' key). AIME25 measured on high-throughput cells only. +// max_tokens 64000, 128 threads, thinking ON via patched sgl-eval (enable_thinking). +// AIME25 measured on high-throughput cells only. +// BF16 AIME25 null: BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs 16.9k tokens), +// truncating at 33.8% at max_tokens=64000. Valid score requires max_tokens ≥ 131072. // -// BF16 AIME25 invalidity: BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs -// 16.9k tokens); 33.8% of responses truncated at max_tokens=64000. aime25_pct is -// left null — re-run with max_tokens ≥ 128k for a valid score. +// GB300 cells (BF16/FP8/INT4): REAL measured values (sglang dev/custom build, +// 4×GB300 TP=4, 2026-07-20). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. +// AIME25 not measured on GB300: the sgl-eval `--thinking` flag sends the generic +// `thinking` key which Laguna's template ignores (it requires `enable_thinking`); +// without the sgl-eval patch applied on H200, thinking was not activated — the +// fp8_ht result (13.54%) is therefore invalid and omitted. +// GB300 NVFP4: quality broken (degenerate loop) — no accuracy numbers. // -// B300/GB300 cells: PENDING (bare match stubs — benchmark card renders "pending"). +// B300 cells: PENDING (bare match stubs — benchmark card renders "pending"). export const benchmarks = [ // ===== H200 (8-GPU HGX, tp 8) — ✅ REAL, full GSM8K ===== { - // ✅ REAL — 8×H200, BF16 dense, tp8, fa3 (Hopper auto-select), mem-frac 0.80. - // GSM8K 93.18%. AIME25 pending re-run with higher max_tokens (truncation at 64k). + // ✅ REAL — 8×H200, BF16 dense, tp8, fa3, mem-frac 0.80. GSM8K 93.18%. + // AIME25 null: truncation at 64k (BF16 reasons ~2× longer; needs max_tokens ≥ 131072). match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", @@ -47,7 +53,7 @@ export const benchmarks = [ accuracy: { gsm8k_pct: 94.47, aime25_pct: null }, }, { - // ✅ REAL — 8×H200, INT4 dense, tp8, fa3. INT4 shared expert stays bf16 (no flag). + // ✅ REAL — 8×H200, INT4 dense, tp8, fa3. INT4 shared expert stays bf16. // GSM8K 95.00%, AIME25 67.71% pass@1 (avg-16, pass@16 86.7%, majority@16 80.0%). match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, @@ -73,13 +79,56 @@ export const benchmarks = [ { match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, { match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, - // ===== GB300 (4-GPU single node, tp 4) — PENDING ===== - { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, + // ===== GB300 (4-GPU single node, tp 4) ===== + { + // ✅ REAL — 4×GB300, BF16 dense, tp4, trtllm_mha (auto-select). + // GSM8K 93.33%. AIME25 not measured (enable_thinking patch not applied on GB300). + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "dev/custom build (2026-07-20)", + accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, + }, + { + // ✅ REAL — 4×GB300, BF16 + DFlash (matched bf16 draft), tp4, trtllm_mha. + // GSM8K 93.86%. AIME25 not measured. + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "dev/custom build (2026-07-20)", + accuracy: { gsm8k_pct: 93.86, aime25_pct: null }, + }, + { + // ✅ REAL — 4×GB300, FP8 dense, tp4, trtllm_mha, SGLANG_SHARED_EXPERT_TP1=1. + // GSM8K 94.77%. AIME25 not measured (enable_thinking patch not applied on GB300). + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "dev/custom build (2026-07-20)", + accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, + }, + { + // ✅ REAL — 4×GB300, FP8 + DFlash (fp8-calibrated draft, patched rope_theta), + // tp4, trtllm_mha, SGLANG_SHARED_EXPERT_TP1=1. GSM8K 94.54%. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "dev/custom build (2026-07-20)", + accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, + }, + // NVFP4: quality broken on GB300 (degenerate loop) — no accuracy numbers, pending stubs. { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" } }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, + { + // ✅ REAL — 4×GB300, INT4 dense, tp4, trtllm_mha. INT4 shared expert stays bf16. + // GSM8K 94.77%. AIME25 skipped per user decision. + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "dev/custom build (2026-07-20)", + accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, + }, + { + // ✅ REAL — 4×GB300, INT4 + DFlash (int4-calibrated draft), tp4, trtllm_mha. + // GSM8K 95.15%. + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "dev/custom build (2026-07-20)", + accuracy: { gsm8k_pct: 95.15, aime25_pct: null }, + }, ]; diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index a6618c2b96d5..41d38b2c6cc4 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -22,40 +22,53 @@ // default), sglang applies top_k=0 at serving time and rejects any non-greedy sampling // request with HTTP 400. --sampling-defaults openai uses OpenAI API defaults instead, // ignoring the model-level top_k=0. Greedy (GSM8K) is unaffected; non-greedy -// (AIME25, chat sampling) requires this flag. Verified 2026-07-20 on 8×H200. +// (AIME25, chat sampling) requires this flag. Verified 2026-07-20 on 8×H200 and +// 4×GB300. // // TP sizing (118B BF16 ≈ 236 GB; FP8 ≈ 118 GB; NVFP4/INT4 ≈ 59 GB): // - H200 8-GPU (8×141 GB): --tp 8 for all quants (29.5 GB weights/GPU for BF16). // Unlike Laguna-XS-2.1, the much larger moe_intermediate_size of this 118B model -// is expected to allow plain TP=8 for FP8 and INT4 without --ep-size (the XS-2.1 -// constraint was 512/8=64 < block_size=128; that does not hold at the larger expert -// width in S-2.1). Confirmed on H200 for BF16, FP8, INT4 (2026-07-20). +// allows plain TP=8 for FP8 and INT4 without --ep-size. Confirmed on H200 (2026-07-20). // - B300 8-GPU (8×288 GB): same --tp 8 reasoning. // - GB300 4-GPU (4×288 GB): --tp 4 throughout (59 GB/GPU for BF16, ≥229 GB KV budget). +// Confirmed on GB300 (2026-07-20). // // BF16 memory on H200: -// BF16 (29.5 GB/GPU) consumes ~2× more weight memory than FP8 on H200, leaving less -// headroom for CUDA-graph capture and NCCL allocs. Default mem-fraction 0.893 → OOM -// at cuda-graph capture; 0.88 → NCCL crash at high concurrency. --mem-fraction-static -// 0.80 is required for BF16 high-throughput on H200. Low-latency already carries 0.7. -// B300/GB300 (288 GB/GPU) are unaffected and use the default heuristic. +// BF16 (29.5 GB/GPU) leaves less headroom on H200 for CUDA-graph capture and NCCL +// allocs than FP8/INT4. Default mem-fraction 0.893 → OOM at cuda-graph capture; 0.88 +// → NCCL crash at high concurrency. --mem-fraction-static 0.80 is required for BF16 +// high-throughput on H200. Low-latency already carries 0.7. +// GB300 (288 GB/GPU) and B300 (288 GB/GPU) are unaffected — default heuristic suffices. // -// SGLANG_SHARED_EXPERT_TP1=1 (FP8 cells only): -// Confirmed required: the FP8 checkpoint block-quantizes the shared expert (128×128 -// scales, can't TP-shard cleanly). INT4 keeps the shared expert in BF16 (no flag -// needed). BF16 is unquantized. Verified 2026-07-20 on 8×H200. +// SGLANG_SHARED_EXPERT_TP1=1 (FP8 cells, ALL hardware): +// Confirmed required on H200 (TP=8) and GB300 (TP=4): the FP8 checkpoint +// block-quantizes the shared expert (128×128 scales), which cannot TP-shard cleanly at +// either TP degree on S-2.1. INT4 keeps the shared expert in BF16 (no flag needed); +// BF16 is unquantized. Note: Laguna-XS-2.1 at TP=4 does NOT need this flag — the +// difference is architecture-specific (XS-2.1 shared expert shards cleanly at TP=4). +// +// NVFP4 on GB300 — BROKEN (2026-07-20): +// Both NVFP4 cells produce degenerate output (repetition loop, never emits EOS) on +// 4×GB300 — verified identical behaviour with and without spec-decode. Perf numbers +// exist (mechanically valid) but quality is unusable. Root cause: likely a checkpoint +// or sglang sm103 kernel issue; needs poolside/sglang follow-up. NVFP4 cells on GB300 +// remain verified:false until resolved. NVFP4 on B300 is unverified. // // NVFP4 is Blackwell-only → no h200×nvfp4 cells. // // DFlash memory: Low-Latency cells carry --mem-fraction-static 0.7. // -// FP8 DFlash draft rope issue: -// poolside/Laguna-S-2.1-DFlash-FP8 (as of 2026-07-20) is missing rope_theta in -// rope_parameters. If the server crashes at draft model load, patch the draft config -// locally: add "rope_theta" to the rope_parameters block. INT4 draft is unaffected. +// FP8 / NVFP4 DFlash draft rope issue: +// poolside/Laguna-S-2.1-DFlash-FP8 and poolside/Laguna-S-2.1-DFlash-NVFP4 (as of +// 2026-07-20) are missing rope_theta in rope_parameters. If the server crashes at +// draft model load, patch the draft config locally by adding "rope_theta": 500000.0 +// to the rope_parameters block. The BF16 and INT4 draft models are unaffected. // -// verified:true = command ran and passed full GSM8K on that cell. H200 cells verified -// 2026-07-20 (sglang 0.5.15.post1). B300/GB300 cells are unverified pending measurement. +// verified:true = command ran and passed full GSM8K on that cell. +// H200 cells: verified 2026-07-20, sglang 0.5.15.post1. +// GB300 cells (BF16/FP8/INT4): verified 2026-07-20, sglang dev/custom build. +// B300 cells: unverified pending measurement. +// GB300 NVFP4 cells: verified:false — quality broken, see note above. // // Draft/target precision ALWAYS matches: each quantized target pairs with the DFlash // draft calibrated for it (poolside/Laguna-S-2.1-DFlash[-FP8/-NVFP4/-INT4]). @@ -128,8 +141,7 @@ sgl-eval run gsm8k \\ // Laguna's template gates on enable_thinking (not the generic 'thinking' key). // Serve with a copy of the model's chat template whose enable_thinking default // is flipped to true, passed via --chat-template. - // Note: BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs 16.9k tokens) - // and routinely truncates at 64k — use max_tokens ≥ 128k for a valid BF16 score. + // For BF16: use --max-tokens 131072 (see Configuration Tips: BF16 reasoning length). aime25_pct: `# pip install git+https://github.com/sgl-project/sgl-eval # Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking). @@ -189,21 +201,18 @@ sgl-eval run aime25 \\ // Cells: (h200 × {bf16,fp8,int4} + b300/gb300 × {bf16,fp8,nvfp4,int4}) × {low-latency, high-throughput}. // NVFP4 is Blackwell-only: no h200×nvfp4 cells. - // H200 cells: verified:true (measured 2026-07-20, sglang 0.5.15.post1). - // B300/GB300 cells: verified:false — commands follow the H200 verified shape + TP scaling. + // H200 cells: verified (2026-07-20, sglang 0.5.15.post1). + // GB300 BF16/FP8/INT4 cells: verified (2026-07-20, sglang dev/custom build). + // GB300 NVFP4 cells: verified:false — output quality broken (see header). + // B300 cells: verified:false — pending measurement. cells: [ // ══════════════ NVIDIA Hopper H200 (8-GPU HGX, --tp 8) — VERIFIED ══════════════ - // Dense auto-selects fa3 on Hopper (no flag needed). LL pins fa3 (DFlash-safe on - // Hopper; with a spec algorithm active, auto would fall back to flashinfer). - // BF16 high-throughput requires --mem-fraction-static 0.80 on H200: default 0.893 - // OOMs at cuda-graph capture; 0.88 crashes at high concurrency under NCCL alloc. - // BF16 low-latency already carries 0.7 (sufficient). - // FP8/INT4 use default mem-fraction (their smaller weight footprint leaves enough - // headroom). + // Dense auto-selects fa3 on Hopper (no flag needed). LL pins fa3. + // BF16 high-throughput requires --mem-fraction-static 0.80 on H200 (see header). + // BF16 low-latency already carries 0.7; FP8/INT4 use default. { // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 93.18%. - // --mem-fraction-static 0.80 required — default/0.88 OOMs (see header comment). match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, env: [], @@ -242,7 +251,7 @@ sgl-eval run aime25 \\ }, { // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.24%, AIME25 67.29%. - // SGLANG_SHARED_EXPERT_TP1=1 confirmed required (FP8 block-quantizes shared expert). + // SGLANG_SHARED_EXPERT_TP1=1 confirmed required (block-FP8 shared expert). match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], @@ -259,7 +268,7 @@ sgl-eval run aime25 \\ }, { // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.47%. - // Note: DFlash-FP8 draft may crash at load (rope_theta missing) — see header. + // DFlash-FP8 draft: see rope_theta note in header. match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], @@ -319,11 +328,9 @@ sgl-eval run aime25 \\ }, // ══════════════ NVIDIA Blackwell Ultra B300 (8-GPU HGX) — UNVERIFIED ══════════════ - // Dense auto-selects trtllm_mha on Blackwell (no flag). LL MUST pin trtllm_mha — - // with DFlash active, auto falls back to flashinfer, which is broken for this - // hybrid-SWA model at tp≥4 (reproduced + bisected on GB300 with Laguna-XS-2.1). - // B300 has 288 GB/GPU: BF16 29.5 GB/GPU leaves ≥258 GB for KV — no mem-fraction - // adjustment needed for dense cells (unlike H200). + // Dense auto-selects trtllm_mha on Blackwell. LL MUST pin trtllm_mha (flashinfer + // breaks hybrid-SWA at tp≥4 with DFlash active — reproduced on Laguna-XS-2.1). + // B300 (288 GB/GPU): BF16 29.5 GB/GPU leaves ≥258 GB for KV — no mem-fraction needed. { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, @@ -465,13 +472,19 @@ sgl-eval run aime25 \\ ], }, - // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node, --tp 4) — UNVERIFIED ══════════════ - // BF16 118B at tp 4 = 59 GB/GPU (out of 288 GB): ample KV budget, no mem-fraction - // adjustment needed for dense cells. Dense auto-selects trtllm_mha on Blackwell. - // LL pins trtllm_mha (same flashinfer concern as B300 above). + // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node, --tp 4) ══════════════ + // BF16/FP8/INT4 verified 2026-07-20 (sglang dev/custom build). + // NVFP4: verified:false — degenerate output (repetition loop, never terminates); + // see header comment. Perf is mechanically valid but quality is broken. + // Dense auto-selects trtllm_mha on Blackwell. LL pins trtllm_mha. + // BF16 59 GB/GPU (out of 288 GB): ample KV budget, default mem-fraction suffices. + // FP8 cells require SGLANG_SHARED_EXPERT_TP1=1 — confirmed on GB300 at TP=4 + // (S-2.1's FP8 shared expert cannot shard 4-way; this differs from XS-2.1 + // where TP=4 works without the flag due to different expert widths). { + // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 93.33%. match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -485,8 +498,9 @@ sgl-eval run aime25 \\ ], }, { + // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 93.86%. match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -505,9 +519,11 @@ sgl-eval run aime25 \\ ], }, { + // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 94.77%. + // SGLANG_SHARED_EXPERT_TP1=1 confirmed required at TP=4 on S-2.1 (see header). match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - verified: false, - env: [], + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", @@ -520,9 +536,11 @@ sgl-eval run aime25 \\ ], }, { + // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 94.54%. + // DFlash-FP8 draft: see rope_theta note in header. match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - verified: false, - env: [], + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], flags: [ "--model-path {{MODEL_NAME}}", "--trust-remote-code", @@ -540,6 +558,8 @@ sgl-eval run aime25 \\ ], }, { + // verified:false — NVFP4 output quality broken on GB300 (degenerate loop). + // See header comment. Perf kernel runs but model quality is unusable. match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: false, env: [], @@ -555,6 +575,8 @@ sgl-eval run aime25 \\ ], }, { + // verified:false — NVFP4 output quality broken on GB300 (degenerate loop). + // DFlash-NVFP4 draft also has rope_theta issue (see header). match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: false, env: [], @@ -575,8 +597,10 @@ sgl-eval run aime25 \\ ], }, { + // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 94.77%. + // INT4 shared expert stays bf16 — no SGLANG_SHARED_EXPERT_TP1 needed. match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -590,8 +614,9 @@ sgl-eval run aime25 \\ ], }, { + // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 95.15%. match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", From 367d60e3dfacc645362e8581469404ae4b83cb6b Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:06:29 +0000 Subject: [PATCH 04/20] Add B300 GSM8K benchmark values for Laguna-S-2.1 Measured on 8xB300 TP=8, sglang 0.5.15.post1 (v0.5.15.post1-cu130 image), sgl-eval gsm8k full 1319 questions, greedy. Co-Authored-By: Claude Fable 5 --- .../poolside/laguna-s21-benchmarks.jsx | 75 ++++++++++++++++--- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index c7e18e29f23f..f488fa5ce7df 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -16,7 +16,9 @@ // fp8_ht result (13.54%) is therefore invalid and omitted. // GB300 NVFP4: quality broken (degenerate loop) — no accuracy numbers. // -// B300 cells: PENDING (bare match stubs — benchmark card renders "pending"). +// B300 cells: REAL measured values (sglang 0.5.15.post1, lmsysorg/sglang:v0.5.15.post1-cu130, +// 8×B300 TP=8, 2026-07-21). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. +// AIME25 not measured on B300. export const benchmarks = [ // ===== H200 (8-GPU HGX, tp 8) — ✅ REAL, full GSM8K ===== @@ -69,15 +71,68 @@ export const benchmarks = [ accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, }, - // ===== B300 (8-GPU HGX, tp 8) — PENDING ===== - { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" } }, + // ===== B300 (8-GPU HGX, tp 8) — ✅ REAL, full GSM8K ===== + { + // ✅ REAL — 8×B300, BF16 dense, tp8, trtllm_mha (auto-select). GSM8K 93.71%. + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 93.71, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, BF16 + DFlash (matched bf16 draft), tp8, trtllm_mha, mem-frac 0.7. + // GSM8K 93.63%. + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 93.63, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, FP8 dense, tp8 ep8, trtllm_mha, SGLANG_SHARED_EXPERT_TP1=1. + // GSM8K 94.39%. + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.39, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, FP8 + DFlash (fp8-calibrated draft, patched rope_theta), tp8 ep8, + // trtllm_mha, mem-frac 0.7. GSM8K 94.69%. + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, NVFP4 dense, tp8, trtllm_mha (auto-select). GSM8K 94.54%. + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, NVFP4 + DFlash (nvfp4-calibrated draft, patched rope_theta), tp8, + // trtllm_mha, mem-frac 0.7. GSM8K 95.30%. + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 95.30, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, INT4 dense, tp8 ep8, trtllm_mha. GSM8K 94.62%. + match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.62, aime25_pct: null }, + }, + { + // ✅ REAL — 8×B300, INT4 + DFlash (int4-calibrated draft), tp8 ep8, trtllm_mha, + // mem-frac 0.7. GSM8K 94.69%. + match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15.post1", + accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, + }, // ===== GB300 (4-GPU single node, tp 4) ===== { From afe8bd7585794a52322fcad5ca668e1df7634d00 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 01:10:48 -0700 Subject: [PATCH 05/20] [Cookbook] Laguna-S-2.1: strip verbose comments and remove GB300 NVFP4 benchmark stubs Remove all header paragraphs containing benchmark run conclusions and TP-sizing rationale from config and benchmarks files. Drop per-cell GSM8K inline notes (numbers live in the benchmarks data). Remove GB300 NVFP4 stub entries from benchmarks; config cells remain verified:false. Trim MDX BF16 memory and NVFP4 tips to drop internal debugging details. Co-Authored-By: Claude Sonnet 4.6 --- .../autoregressive/Poolside/Laguna-S-2.1.mdx | 4 +- .../poolside/laguna-s21-benchmarks.jsx | 76 ++------- .../snippets/configs/poolside/laguna-s21.jsx | 160 +++--------------- 3 files changed, 41 insertions(+), 199 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index 49d58d3fa490..25a8c34d9df8 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -133,7 +133,7 @@ Leave `--attention-backend` unset for High-throughput cells — auto-select is c **BF16 memory on H200** -BF16 weights consume ~29.5 GB/GPU on H200 (about 2× FP8), leaving less headroom for CUDA-graph capture and NCCL allocations at high concurrency. The default `mem-fraction-static 0.893` OOMs at cuda-graph capture; `0.88` crashes under NCCL at `conc ≥ 1024`. The High-throughput BF16 command carries `--mem-fraction-static 0.80`. FP8 and INT4 use the default heuristic. On B300 and GB300 (288 GB/GPU) BF16 has ample headroom and no adjustment is needed. +BF16 on H200 leaves less headroom for CUDA-graph capture and NCCL allocations than FP8/INT4. The High-throughput BF16 command carries `--mem-fraction-static 0.80`. FP8, INT4, and all B300/GB300 cells use the default heuristic. **FP8 shared expert** @@ -145,7 +145,7 @@ The HF checkpoints `poolside/Laguna-S-2.1-DFlash-FP8` and `poolside/Laguna-S-2.1 **NVFP4 on GB300** -NVFP4 produces degenerate output on 4×GB300 (repetition loop, never emits EOS) — reproduced identically with and without DFlash spec-decode, ruling out a draft interaction. Root cause is likely the NVFP4 checkpoint or the sglang sm103 kernel path. The NVFP4 cells on GB300 remain unverified until this is resolved upstream. NVFP4 on B300 is unverified. +NVFP4 is not yet verified on GB300 or B300; pending resolution upstream. **DFlash memory** diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index f488fa5ce7df..d7ab81ffb193 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -1,186 +1,132 @@ -// Laguna-S-2.1 benchmarks — one entry per cell `match` (same 5 keys as laguna-s21.jsx cells). -// -// H200 cells: REAL measured values (sglang 0.5.15.post1, lmsysorg/sglang:latest, -// 8×H200 TP=8, 2026-07-20). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. -// AIME25 = sgl-eval run aime25, 30 problems × 16 repeats, temp 1.0, top_p 0.95, -// max_tokens 64000, 128 threads, thinking ON via patched sgl-eval (enable_thinking). -// AIME25 measured on high-throughput cells only. -// BF16 AIME25 null: BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs 16.9k tokens), -// truncating at 33.8% at max_tokens=64000. Valid score requires max_tokens ≥ 131072. -// -// GB300 cells (BF16/FP8/INT4): REAL measured values (sglang dev/custom build, -// 4×GB300 TP=4, 2026-07-20). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. -// AIME25 not measured on GB300: the sgl-eval `--thinking` flag sends the generic -// `thinking` key which Laguna's template ignores (it requires `enable_thinking`); -// without the sgl-eval patch applied on H200, thinking was not activated — the -// fp8_ht result (13.54%) is therefore invalid and omitted. -// GB300 NVFP4: quality broken (degenerate loop) — no accuracy numbers. -// -// B300 cells: REAL measured values (sglang 0.5.15.post1, lmsysorg/sglang:v0.5.15.post1-cu130, -// 8×B300 TP=8, 2026-07-21). GSM8K = sgl-eval run gsm8k, full 1319 questions, greedy. -// AIME25 not measured on B300. - export const benchmarks = [ - // ===== H200 (8-GPU HGX, tp 8) — ✅ REAL, full GSM8K ===== + + // ── H200 (8×H200, tp 8, sglang 0.5.15.post1) ── + { - // ✅ REAL — 8×H200, BF16 dense, tp8, fa3, mem-frac 0.80. GSM8K 93.18%. - // AIME25 null: truncation at 64k (BF16 reasons ~2× longer; needs max_tokens ≥ 131072). match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", + // BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs 16.9k tokens); + // truncation at max_tokens=64000 invalidates the result. Needs max_tokens ≥ 131072. accuracy: { gsm8k_pct: 93.18, aime25_pct: null }, }, { - // ✅ REAL — 8×H200, BF16 + DFlash (matched bf16 draft), tp8, fa3, mem-frac 0.7. - // GSM8K 93.33%. AIME25 not measured on low-latency cells. match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, }, { - // ✅ REAL — 8×H200, FP8 dense, tp8, fa3, SGLANG_SHARED_EXPERT_TP1=1. - // GSM8K 94.24%, AIME25 67.29% pass@1 (avg-16, pass@16 86.7%, majority@16 76.7%). match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.24, aime25_pct: 67.29 }, }, { - // ✅ REAL — 8×H200, FP8 + DFlash (fp8-calibrated draft, patched rope_theta), tp8, fa3. - // GSM8K 94.47%. AIME25 not measured on low-latency cells. match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.47, aime25_pct: null }, }, { - // ✅ REAL — 8×H200, INT4 dense, tp8, fa3. INT4 shared expert stays bf16. - // GSM8K 95.00%, AIME25 67.71% pass@1 (avg-16, pass@16 86.7%, majority@16 80.0%). match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 95.00, aime25_pct: 67.71 }, }, { - // ✅ REAL — 8×H200, INT4 + DFlash (int4-calibrated draft), tp8, fa3. - // GSM8K 94.24%. AIME25 not measured on low-latency cells. match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, }, - // ===== B300 (8-GPU HGX, tp 8) — ✅ REAL, full GSM8K ===== + // ── B300 (8×B300, tp 8, sglang 0.5.15.post1) ── + { - // ✅ REAL — 8×B300, BF16 dense, tp8, trtllm_mha (auto-select). GSM8K 93.71%. match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 93.71, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, BF16 + DFlash (matched bf16 draft), tp8, trtllm_mha, mem-frac 0.7. - // GSM8K 93.63%. match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 93.63, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, FP8 dense, tp8 ep8, trtllm_mha, SGLANG_SHARED_EXPERT_TP1=1. - // GSM8K 94.39%. match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.39, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, FP8 + DFlash (fp8-calibrated draft, patched rope_theta), tp8 ep8, - // trtllm_mha, mem-frac 0.7. GSM8K 94.69%. match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, NVFP4 dense, tp8, trtllm_mha (auto-select). GSM8K 94.54%. match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, NVFP4 + DFlash (nvfp4-calibrated draft, patched rope_theta), tp8, - // trtllm_mha, mem-frac 0.7. GSM8K 95.30%. match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 95.30, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, INT4 dense, tp8 ep8, trtllm_mha. GSM8K 94.62%. match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.62, aime25_pct: null }, }, { - // ✅ REAL — 8×B300, INT4 + DFlash (int4-calibrated draft), tp8 ep8, trtllm_mha, - // mem-frac 0.7. GSM8K 94.69%. match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, }, - // ===== GB300 (4-GPU single node, tp 4) ===== + // ── GB300 (4×GB300, tp 4, sglang dev build 2026-07-20) ── + // AIME25 null: sgl-eval --thinking sends the generic `thinking` key; Laguna's template + // requires `enable_thinking`. Scores would reflect thinking-off behavior — omitted. + { - // ✅ REAL — 4×GB300, BF16 dense, tp4, trtllm_mha (auto-select). - // GSM8K 93.33%. AIME25 not measured (enable_thinking patch not applied on GB300). match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "dev/custom build (2026-07-20)", accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, }, { - // ✅ REAL — 4×GB300, BF16 + DFlash (matched bf16 draft), tp4, trtllm_mha. - // GSM8K 93.86%. AIME25 not measured. match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "dev/custom build (2026-07-20)", accuracy: { gsm8k_pct: 93.86, aime25_pct: null }, }, { - // ✅ REAL — 4×GB300, FP8 dense, tp4, trtllm_mha, SGLANG_SHARED_EXPERT_TP1=1. - // GSM8K 94.77%. AIME25 not measured (enable_thinking patch not applied on GB300). match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "dev/custom build (2026-07-20)", accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, }, { - // ✅ REAL — 4×GB300, FP8 + DFlash (fp8-calibrated draft, patched rope_theta), - // tp4, trtllm_mha, SGLANG_SHARED_EXPERT_TP1=1. GSM8K 94.54%. match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "dev/custom build (2026-07-20)", accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, }, - // NVFP4: quality broken on GB300 (degenerate loop) — no accuracy numbers, pending stubs. - { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, { - // ✅ REAL — 4×GB300, INT4 dense, tp4, trtllm_mha. INT4 shared expert stays bf16. - // GSM8K 94.77%. AIME25 skipped per user decision. match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "dev/custom build (2026-07-20)", accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, }, { - // ✅ REAL — 4×GB300, INT4 + DFlash (int4-calibrated draft), tp4, trtllm_mha. - // GSM8K 95.15%. match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "dev/custom build (2026-07-20)", diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index 41d38b2c6cc4..2cc125ca2b3d 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -1,77 +1,29 @@ -// Laguna-S-2.1 (poolside) — config-driven cookbook page. -// Consumed by the shared _deployment.jsx + _playground.jsx engines (no model code there). +// Laguna-S-2.1 (poolside) — 118B MoE (8B active), 1M context, laguna arch (SGLang main). +// --trust-remote-code required (custom config code on Hub). // -// Architecture: 118B total / ~8B active per token (top-10 routing over 256 routed experts -// + 1 shared expert). 48 layers: 12 full-attention + 36 sliding-window (window 512). -// 1,048,576-token context window. Per-head sigmoid output gating on attention. +// Attention backend: leave unset for High-Throughput (auto-selects fa3/trtllm_mha). +// With DFlash active, auto falls back to flashinfer which breaks hybrid-SWA at tp≥4 +// on Blackwell — Low-Latency cells pin the target backend explicitly. +// Never use --attention-backend triton on Laguna (broken SWA handling). // -// Build: uses the same `laguna` model type as Laguna-XS-2.1. All support is in SGLang main. -// The model ships custom config code on the Hub, so --trust-remote-code is required. +// --sampling-defaults openai (all cells): generation_config.json ships top_k=0; +// default --sampling-defaults model rejects non-greedy requests with HTTP 400. // -// Attention backend (same rules as Laguna-XS-2.1 — hybrid-SWA is backend-sensitive): -// - Dense (High-Throughput): leave --attention-backend UNSET. Auto-select is correct: -// fa3 on Hopper (H200), trtllm_mha on Blackwell (B300/GB300). -// - DFlash (Low-Latency): MUST pin the target backend explicitly — with a speculative -// algorithm active, the resolver falls back to flashinfer, which breaks this -// hybrid-SWA model at tp≥4 on Blackwell (reproduced on Laguna-XS-2.1 at GB300, -// GSM8K 76%→28%). Pin fa3 on H200, trtllm_mha on B300/GB300. -// - NEVER use --attention-backend triton for Laguna (broken SWA handling). +// BF16 on H200 HT: --mem-fraction-static 0.80 required — BF16 leaves less headroom +// for CUDA-graph capture and NCCL allocs than FP8/INT4 on 141 GB/GPU. +// B300/GB300 (288 GB/GPU) unaffected. // -// --sampling-defaults openai (ALL cells): -// generation_config.json ships with top_k=0. With --sampling-defaults model (the -// default), sglang applies top_k=0 at serving time and rejects any non-greedy sampling -// request with HTTP 400. --sampling-defaults openai uses OpenAI API defaults instead, -// ignoring the model-level top_k=0. Greedy (GSM8K) is unaffected; non-greedy -// (AIME25, chat sampling) requires this flag. Verified 2026-07-20 on 8×H200 and -// 4×GB300. +// SGLANG_SHARED_EXPERT_TP1=1 (FP8 cells, all hardware): FP8 block-quantizes the shared +// expert; required at both TP=4 (GB300) and TP=8 (H200/B300). INT4 shared expert stays +// bf16 — no flag needed. Differs from Laguna-XS-2.1 where TP=4 works without this flag. // -// TP sizing (118B BF16 ≈ 236 GB; FP8 ≈ 118 GB; NVFP4/INT4 ≈ 59 GB): -// - H200 8-GPU (8×141 GB): --tp 8 for all quants (29.5 GB weights/GPU for BF16). -// Unlike Laguna-XS-2.1, the much larger moe_intermediate_size of this 118B model -// allows plain TP=8 for FP8 and INT4 without --ep-size. Confirmed on H200 (2026-07-20). -// - B300 8-GPU (8×288 GB): same --tp 8 reasoning. -// - GB300 4-GPU (4×288 GB): --tp 4 throughout (59 GB/GPU for BF16, ≥229 GB KV budget). -// Confirmed on GB300 (2026-07-20). -// -// BF16 memory on H200: -// BF16 (29.5 GB/GPU) leaves less headroom on H200 for CUDA-graph capture and NCCL -// allocs than FP8/INT4. Default mem-fraction 0.893 → OOM at cuda-graph capture; 0.88 -// → NCCL crash at high concurrency. --mem-fraction-static 0.80 is required for BF16 -// high-throughput on H200. Low-latency already carries 0.7. -// GB300 (288 GB/GPU) and B300 (288 GB/GPU) are unaffected — default heuristic suffices. -// -// SGLANG_SHARED_EXPERT_TP1=1 (FP8 cells, ALL hardware): -// Confirmed required on H200 (TP=8) and GB300 (TP=4): the FP8 checkpoint -// block-quantizes the shared expert (128×128 scales), which cannot TP-shard cleanly at -// either TP degree on S-2.1. INT4 keeps the shared expert in BF16 (no flag needed); -// BF16 is unquantized. Note: Laguna-XS-2.1 at TP=4 does NOT need this flag — the -// difference is architecture-specific (XS-2.1 shared expert shards cleanly at TP=4). -// -// NVFP4 on GB300 — BROKEN (2026-07-20): -// Both NVFP4 cells produce degenerate output (repetition loop, never emits EOS) on -// 4×GB300 — verified identical behaviour with and without spec-decode. Perf numbers -// exist (mechanically valid) but quality is unusable. Root cause: likely a checkpoint -// or sglang sm103 kernel issue; needs poolside/sglang follow-up. NVFP4 cells on GB300 -// remain verified:false until resolved. NVFP4 on B300 is unverified. +// FP8/NVFP4 DFlash drafts: missing rope_theta in rope_parameters as of 2026-07-20. +// If draft load crashes with KeyError: 'rope_theta', add "rope_theta": 500000.0 to +// the draft's rope_parameters block. BF16/INT4 drafts unaffected. // +// NVFP4 on GB300: unverified — degenerate output observed; pending upstream fix. // NVFP4 is Blackwell-only → no h200×nvfp4 cells. -// -// DFlash memory: Low-Latency cells carry --mem-fraction-static 0.7. -// -// FP8 / NVFP4 DFlash draft rope issue: -// poolside/Laguna-S-2.1-DFlash-FP8 and poolside/Laguna-S-2.1-DFlash-NVFP4 (as of -// 2026-07-20) are missing rope_theta in rope_parameters. If the server crashes at -// draft model load, patch the draft config locally by adding "rope_theta": 500000.0 -// to the rope_parameters block. The BF16 and INT4 draft models are unaffected. -// -// verified:true = command ran and passed full GSM8K on that cell. -// H200 cells: verified 2026-07-20, sglang 0.5.15.post1. -// GB300 cells (BF16/FP8/INT4): verified 2026-07-20, sglang dev/custom build. -// B300 cells: unverified pending measurement. -// GB300 NVFP4 cells: verified:false — quality broken, see note above. -// -// Draft/target precision ALWAYS matches: each quantized target pairs with the DFlash -// draft calibrated for it (poolside/Laguna-S-2.1-DFlash[-FP8/-NVFP4/-INT4]). +// DFlash cells carry --mem-fraction-static 0.7. export const config = { modelName: "Laguna-S-2.1", @@ -89,11 +41,6 @@ export const config = { { id: "int4", label: "INT4" }, ], - // Two operating points: - // low-latency = DFlash speculative decoding (matched-precision draft) — interactive / - // few-stream serving. - // high-throughput = plain serving (no speculation) — batch-saturated workloads, where - // speculation's draft+rejection overhead costs more than it saves. strategies: [ { id: "low-latency", label: "Low-latency" }, { id: "high-throughput", label: "High-throughput" }, @@ -131,17 +78,15 @@ export const config = { --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} \\ --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, - // GSM8K is the required accuracy sanity on every verified cell, via sgl-eval. accuracy: { gsm8k_pct: `# pip install git+https://github.com/sgl-project/sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 128`, - // Laguna's template gates on enable_thinking (not the generic 'thinking' key). + // Laguna's template gates on enable_thinking, not the generic 'thinking' key. // Serve with a copy of the model's chat template whose enable_thinking default - // is flipped to true, passed via --chat-template. - // For BF16: use --max-tokens 131072 (see Configuration Tips: BF16 reasoning length). + // is flipped to true. For BF16: use --max-tokens 131072 (see Configuration Tips). aime25_pct: `# pip install git+https://github.com/sgl-project/sgl-eval # Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking). @@ -155,7 +100,6 @@ sgl-eval run aime25 \\ numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 }, }, - // No variant-wide accuracy default; real numbers are per-cell in laguna-s21-benchmarks.jsx. defaultAccuracy: { default: { gsm8k_pct: null, aime25_pct: null }, }, @@ -165,8 +109,6 @@ sgl-eval run aime25 \\ ["aime25_pct", "AIME25", "%"], ], - // H200: lmsysorg/sglang:latest (sglang 0.5.15.post1, verified 2026-07-20). - // B300/GB300: unverified, using dev (nightly) as default. dockerImages: { h200: "lmsysorg/sglang:latest", b300: "lmsysorg/sglang:dev", @@ -178,19 +120,11 @@ sgl-eval run aime25 \\ }, playgroundFeatures: { - - // Hybrid-SWA GQA model (8 KV heads) — TP shards cleanly at 1/2/4/8. - // No DP-Attention / CP knobs: unvalidated on this model family. attention: { knobs: [ { id: "tp", label: "TP", values: [null, 1, 2, 4, 8] }, ], }, - - // Reasoning + tool-call parsers (poolside_v1, same family as Laguna-XS-2.1). - // Baked into every Deploy cell. The chat template auto-detects both on - // transformers ≥ 5.10 (`Auto-detected template features: reasoning_parser=poolside_v1, - // tool_call_parser=poolside_v1`), so these are explicit-but-redundant there. parsers: { items: [ { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser poolside_v1" }, @@ -199,20 +133,11 @@ sgl-eval run aime25 \\ }, }, - // Cells: (h200 × {bf16,fp8,int4} + b300/gb300 × {bf16,fp8,nvfp4,int4}) × {low-latency, high-throughput}. - // NVFP4 is Blackwell-only: no h200×nvfp4 cells. - // H200 cells: verified (2026-07-20, sglang 0.5.15.post1). - // GB300 BF16/FP8/INT4 cells: verified (2026-07-20, sglang dev/custom build). - // GB300 NVFP4 cells: verified:false — output quality broken (see header). - // B300 cells: verified:false — pending measurement. cells: [ - // ══════════════ NVIDIA Hopper H200 (8-GPU HGX, --tp 8) — VERIFIED ══════════════ - // Dense auto-selects fa3 on Hopper (no flag needed). LL pins fa3. - // BF16 high-throughput requires --mem-fraction-static 0.80 on H200 (see header). - // BF16 low-latency already carries 0.7; FP8/INT4 use default. + // ══════════════ H200 (8-GPU HGX, tp 8) ══════════════ + { - // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 93.18%. match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, env: [], @@ -229,7 +154,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 93.33%. match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, env: [], @@ -250,8 +174,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.24%, AIME25 67.29%. - // SGLANG_SHARED_EXPERT_TP1=1 confirmed required (block-FP8 shared expert). match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], @@ -267,8 +189,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.47%. - // DFlash-FP8 draft: see rope_theta note in header. match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], @@ -289,8 +209,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 95.00%, AIME25 67.71%. - // INT4 shared expert stays bf16 — no SGLANG_SHARED_EXPERT_TP1 needed. match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, env: [], @@ -306,7 +224,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 8×H200 tp8 (2026-07-20): GSM8K 94.24%. match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, env: [], @@ -327,10 +244,8 @@ sgl-eval run aime25 \\ ], }, - // ══════════════ NVIDIA Blackwell Ultra B300 (8-GPU HGX) — UNVERIFIED ══════════════ - // Dense auto-selects trtllm_mha on Blackwell. LL MUST pin trtllm_mha (flashinfer - // breaks hybrid-SWA at tp≥4 with DFlash active — reproduced on Laguna-XS-2.1). - // B300 (288 GB/GPU): BF16 29.5 GB/GPU leaves ≥258 GB for KV — no mem-fraction needed. + // ══════════════ B300 (8-GPU HGX, tp 8) — unverified ══════════════ + { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, @@ -472,17 +387,10 @@ sgl-eval run aime25 \\ ], }, - // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node, --tp 4) ══════════════ - // BF16/FP8/INT4 verified 2026-07-20 (sglang dev/custom build). - // NVFP4: verified:false — degenerate output (repetition loop, never terminates); - // see header comment. Perf is mechanically valid but quality is broken. - // Dense auto-selects trtllm_mha on Blackwell. LL pins trtllm_mha. - // BF16 59 GB/GPU (out of 288 GB): ample KV budget, default mem-fraction suffices. - // FP8 cells require SGLANG_SHARED_EXPERT_TP1=1 — confirmed on GB300 at TP=4 - // (S-2.1's FP8 shared expert cannot shard 4-way; this differs from XS-2.1 - // where TP=4 works without the flag due to different expert widths). + // ══════════════ GB300 (4-GPU single node, tp 4) ══════════════ + // NVFP4 cells: unverified — degenerate output observed on GB300. + { - // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 93.33%. match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, env: [], @@ -498,7 +406,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 93.86%. match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, env: [], @@ -519,8 +426,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 94.77%. - // SGLANG_SHARED_EXPERT_TP1=1 confirmed required at TP=4 on S-2.1 (see header). match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], @@ -536,8 +441,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 94.54%. - // DFlash-FP8 draft: see rope_theta note in header. match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], @@ -558,8 +461,6 @@ sgl-eval run aime25 \\ ], }, { - // verified:false — NVFP4 output quality broken on GB300 (degenerate loop). - // See header comment. Perf kernel runs but model quality is unusable. match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: false, env: [], @@ -575,8 +476,6 @@ sgl-eval run aime25 \\ ], }, { - // verified:false — NVFP4 output quality broken on GB300 (degenerate loop). - // DFlash-NVFP4 draft also has rope_theta issue (see header). match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: false, env: [], @@ -597,8 +496,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 94.77%. - // INT4 shared expert stays bf16 — no SGLANG_SHARED_EXPERT_TP1 needed. match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, env: [], @@ -614,7 +511,6 @@ sgl-eval run aime25 \\ ], }, { - // VERIFIED 4×GB300 tp4 (2026-07-20): GSM8K 95.15%. match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, env: [], From 93dd74840378316f400b62fcc745e0fa232bb007 Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:13:49 -0700 Subject: [PATCH 06/20] Clean up comments in laguna-s21.jsx Removed outdated comments regarding sampling defaults and FP8/NVFP4 DFlash drafts. --- docs_new/src/snippets/configs/poolside/laguna-s21.jsx | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index 2cc125ca2b3d..10e3a153caef 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -6,9 +6,6 @@ // on Blackwell — Low-Latency cells pin the target backend explicitly. // Never use --attention-backend triton on Laguna (broken SWA handling). // -// --sampling-defaults openai (all cells): generation_config.json ships top_k=0; -// default --sampling-defaults model rejects non-greedy requests with HTTP 400. -// // BF16 on H200 HT: --mem-fraction-static 0.80 required — BF16 leaves less headroom // for CUDA-graph capture and NCCL allocs than FP8/INT4 on 141 GB/GPU. // B300/GB300 (288 GB/GPU) unaffected. @@ -17,11 +14,7 @@ // expert; required at both TP=4 (GB300) and TP=8 (H200/B300). INT4 shared expert stays // bf16 — no flag needed. Differs from Laguna-XS-2.1 where TP=4 works without this flag. // -// FP8/NVFP4 DFlash drafts: missing rope_theta in rope_parameters as of 2026-07-20. -// If draft load crashes with KeyError: 'rope_theta', add "rope_theta": 500000.0 to -// the draft's rope_parameters block. BF16/INT4 drafts unaffected. -// -// NVFP4 on GB300: unverified — degenerate output observed; pending upstream fix. +// NVFP4 on GB300: unverified. // NVFP4 is Blackwell-only → no h200×nvfp4 cells. // DFlash cells carry --mem-fraction-static 0.7. From f9352b4ca0e533bba6f7a2de79f72c5fc62d51a1 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 01:22:42 -0700 Subject: [PATCH 07/20] [Cookbook] Laguna-S-2.1: mark B300 BF16/FP8/INT4 cells as verified B300 GSM8K benchmarks were added in a prior commit; update the corresponding config cells from verified:false to verified:true. NVFP4 cells remain unverified. Co-Authored-By: Claude Sonnet 4.6 --- .../src/snippets/configs/poolside/laguna-s21.jsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index 10e3a153caef..ceefa64c8530 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -237,11 +237,11 @@ sgl-eval run aime25 \\ ], }, - // ══════════════ B300 (8-GPU HGX, tp 8) — unverified ══════════════ + // ══════════════ B300 (8-GPU HGX, tp 8) ══════════════ { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -256,7 +256,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -276,7 +276,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], flags: [ "--model-path {{MODEL_NAME}}", @@ -291,7 +291,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: ["SGLANG_SHARED_EXPERT_TP1=1"], flags: [ "--model-path {{MODEL_NAME}}", @@ -346,7 +346,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -361,7 +361,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", From 63e975ef1f894bc82b3ff2e0ae47ecc2111b6e1b Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 01:25:10 -0700 Subject: [PATCH 08/20] [Cookbook] Laguna-S-2.1: set default selection to B300 FP8 low-latency Add config.defaultSelection support to _deployment.jsx (non-breaking; other configs without the field continue to use cells[0]). Set the default for Laguna-S-2.1 to B300 FP8 low-latency. Co-Authored-By: Claude Sonnet 4.6 --- docs_new/src/snippets/_deployment.jsx | 6 +++--- docs_new/src/snippets/configs/poolside/laguna-s21.jsx | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index 32ad7606ca36..77cab400c4f9 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -835,11 +835,11 @@ export const Deployment = ({ config, benchmarks }) => { }; const initialSelectionFromCells = () => { - const first = config.cells[0]; + const first = config.defaultSelection ?? config.cells[0]?.match; if (!first) return Object.fromEntries(DIMENSIONS.map((d) => [d, ""])); return { - hw: first.match.hw, variant: first.match.variant, quant: first.match.quant, - strategy: first.match.strategy, nodes: first.match.nodes, + hw: first.hw, variant: first.variant, quant: first.quant, + strategy: first.strategy, nodes: first.nodes, }; }; diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index ceefa64c8530..e0241f1a2a96 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -126,6 +126,8 @@ sgl-eval run aime25 \\ }, }, + defaultSelection: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + cells: [ // ══════════════ H200 (8-GPU HGX, tp 8) ══════════════ From e6f84bf9d122d1bc5f5f1fbc3918c1c97844c012 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 01:30:54 -0700 Subject: [PATCH 09/20] [Cookbook] Laguna-S-2.1: set default to B300 FP8 low-latency via cells[0] Move B300 FP8 low-latency cell to cells[0] so the deployment panel opens on it by default. Revert _deployment.jsx to its original form. Co-Authored-By: Claude Sonnet 4.6 --- docs_new/src/snippets/_deployment.jsx | 6 +-- .../snippets/configs/poolside/laguna-s21.jsx | 45 ++++++++++--------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index 77cab400c4f9..32ad7606ca36 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -835,11 +835,11 @@ export const Deployment = ({ config, benchmarks }) => { }; const initialSelectionFromCells = () => { - const first = config.defaultSelection ?? config.cells[0]?.match; + const first = config.cells[0]; if (!first) return Object.fromEntries(DIMENSIONS.map((d) => [d, ""])); return { - hw: first.hw, variant: first.variant, quant: first.quant, - strategy: first.strategy, nodes: first.nodes, + hw: first.match.hw, variant: first.match.variant, quant: first.match.quant, + strategy: first.match.strategy, nodes: first.match.nodes, }; }; diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index e0241f1a2a96..a0460a1aeae0 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -126,10 +126,31 @@ sgl-eval run aime25 \\ }, }, - defaultSelection: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - cells: [ + // ══════════════ B300 FP8 low-latency — default (cells[0]) ══════════════ + + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--sampling-defaults openai", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ══════════════ H200 (8-GPU HGX, tp 8) ══════════════ { @@ -291,26 +312,6 @@ sgl-eval run aime25 \\ "--port {{PORT}}", ], }, - { - match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, - verified: true, - env: ["SGLANG_SHARED_EXPERT_TP1=1"], - flags: [ - "--model-path {{MODEL_NAME}}", - "--trust-remote-code", - "--reasoning-parser poolside_v1", - "--tool-call-parser poolside_v1", - "--tp 8", - "--attention-backend trtllm_mha", - "--speculative-algorithm DFLASH", - "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", - "--page-size 1", - "--mem-fraction-static 0.7", - "--sampling-defaults openai", - "--host {{HOST_IP}}", - "--port {{PORT}}", - ], - }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: false, From f99b9a4955433a3afea09ad44ad7fb2d189d66df Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:45:48 +0000 Subject: [PATCH 10/20] Add GB300 NVFP4 benchmark values for Laguna-S-2.1 Measured on 4xGB300 TP=4, stock sglang 0.5.15, sgl-eval gsm8k full 1319 questions, greedy. Supersedes the earlier dev-build degenerate-quality observation, which does not reproduce on 0.5.15 with identical weights. Co-Authored-By: Claude Fable 5 --- .../configs/poolside/laguna-s21-benchmarks.jsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index d7ab81ffb193..2ee6f12c6dbb 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -120,6 +120,18 @@ export const benchmarks = [ sglang_version: "dev/custom build (2026-07-20)", accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "0.5.15", + accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "0.5.15", + accuracy: { gsm8k_pct: 94.16, aime25_pct: null }, + }, { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, From 8fb7e9957447dd8f9460b13be4d14151974c4cae Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 01:50:58 -0700 Subject: [PATCH 11/20] [Cookbook] Laguna-S-2.1: drop --sampling-defaults openai from all cells Co-Authored-By: Claude Sonnet 4.6 --- .../autoregressive/Poolside/Laguna-S-2.1.mdx | 4 ---- .../snippets/configs/poolside/laguna-s21.jsx | 22 ------------------- 2 files changed, 26 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index 25a8c34d9df8..def273575f9f 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -127,10 +127,6 @@ The drafts are small BF16 models, each *calibrated against its quantized target* Leave `--attention-backend` unset for High-throughput cells — auto-select is correct (`fa3` on Hopper, `trtllm_mha` on Blackwell). With DFlash active, auto-select instead falls back to `flashinfer`, which breaks this hybrid-SWA model at `tp ≥ 4` on Blackwell (reproduced on Laguna-XS-2.1, greedy GSM8K 76% → 28%), so the Low-latency commands pin the target backend explicitly. Leave `--speculative-draft-attention-backend` unset. Other attention backend choices have not been fully validated on Laguna; keep the default. -**Sampling defaults** - -`generation_config.json` ships with `top_k=0`. With the default `--sampling-defaults model`, sglang applies `top_k=0` at serving time and rejects any non-greedy sampling request (AIME25, chat sampling) with HTTP 400. All generated commands include `--sampling-defaults openai` to use OpenAI API defaults instead — greedy inference (GSM8K) is unaffected. - **BF16 memory on H200** BF16 on H200 leaves less headroom for CUDA-graph capture and NCCL allocations than FP8/INT4. The High-throughput BF16 command carries `--mem-fraction-static 0.80`. FP8, INT4, and all B300/GB300 cells use the default heuristic. diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index a0460a1aeae0..78dc4828bcdc 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -145,7 +145,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -164,7 +163,6 @@ sgl-eval run aime25 \\ "--tool-call-parser poolside_v1", "--tp 8", "--mem-fraction-static 0.80", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -184,7 +182,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -199,7 +196,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -219,7 +215,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -234,7 +229,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -254,7 +248,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -272,7 +265,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -292,7 +284,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -307,7 +298,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -322,7 +312,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -342,7 +331,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-NVFP4", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -357,7 +345,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 8", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -377,7 +364,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -396,7 +382,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -416,7 +401,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -431,7 +415,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -451,7 +434,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-FP8", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -466,7 +448,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -486,7 +467,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-NVFP4", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -501,7 +481,6 @@ sgl-eval run aime25 \\ "--reasoning-parser poolside_v1", "--tool-call-parser poolside_v1", "--tp 4", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -521,7 +500,6 @@ sgl-eval run aime25 \\ "--speculative-draft-model-path poolside/Laguna-S-2.1-DFlash-INT4", "--page-size 1", "--mem-fraction-static 0.7", - "--sampling-defaults openai", "--host {{HOST_IP}}", "--port {{PORT}}", ], From 0235aaa0bbb939aa6523fb43666d0f96f0f66306 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 02:31:30 -0700 Subject: [PATCH 12/20] [Cookbook] Laguna-S-2.1: mark NVFP4 cells verified on B300 and GB300 Benchmarks file already has verified:true + real GSM8K numbers for all four NVFP4 cells. Sync config and remove the stale "not yet verified" prose from the MDX Configuration Tips. Co-Authored-By: Claude Sonnet 4.6 --- .../cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx | 4 ---- docs_new/src/snippets/configs/poolside/laguna-s21.jsx | 10 ++++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index def273575f9f..d2c3483ffbb1 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -139,10 +139,6 @@ BF16 on H200 leaves less headroom for CUDA-graph capture and NCCL allocations th The HF checkpoints `poolside/Laguna-S-2.1-DFlash-FP8` and `poolside/Laguna-S-2.1-DFlash-NVFP4` (as of 2026-07-20) are missing `rope_theta` in `rope_parameters`. If the server crashes at draft-model load with a `KeyError: 'rope_theta'`, patch the draft config locally: add `"rope_theta": 500000.0` to the `rope_parameters` block. The BF16 and INT4 draft models are unaffected. -**NVFP4 on GB300** - -NVFP4 is not yet verified on GB300 or B300; pending resolution upstream. - **DFlash memory** Low-latency cells carry `--mem-fraction-static 0.7` (sufficient even for BF16 on H200). Dense cells use the default heuristic (except BF16 on H200 — see above). diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx index 78dc4828bcdc..cdf9ac360cc1 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21.jsx @@ -14,7 +14,6 @@ // expert; required at both TP=4 (GB300) and TP=8 (H200/B300). INT4 shared expert stays // bf16 — no flag needed. Differs from Laguna-XS-2.1 where TP=4 works without this flag. // -// NVFP4 on GB300: unverified. // NVFP4 is Blackwell-only → no h200×nvfp4 cells. // DFlash cells carry --mem-fraction-static 0.7. @@ -304,7 +303,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -318,7 +317,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -370,7 +369,6 @@ sgl-eval run aime25 \\ }, // ══════════════ GB300 (4-GPU single node, tp 4) ══════════════ - // NVFP4 cells: unverified — degenerate output observed on GB300. { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, @@ -440,7 +438,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -454,7 +452,7 @@ sgl-eval run aime25 \\ }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, - verified: false, + verified: true, env: [], flags: [ "--model-path {{MODEL_NAME}}", From abf21563a1c81af36c2d0328be3c34b0749b287d Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 02:37:00 -0700 Subject: [PATCH 13/20] [Cookbook] Laguna-S-2.1: update Docker pull command to use latest tag --- docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index d2c3483ffbb1..ac15a00ab6aa 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -32,7 +32,7 @@ Then run the **Python** output of the command panel below in that environment. ```bash Command -docker pull lmsysorg/sglang:dev +docker pull lmsysorg/sglang:latest ``` For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. From d0df9063917d71a1ec4317e6132ed6051d86ff24 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 02:41:16 -0700 Subject: [PATCH 14/20] [Cookbook] Laguna-S-2.1: update sglang_version to 0.5.15 and set aime25_pct to null in benchmarks --- .../configs/poolside/laguna-s21-benchmarks.jsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index 2ee6f12c6dbb..1c41b2cb6bf7 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -20,7 +20,7 @@ export const benchmarks = [ match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 94.24, aime25_pct: 67.29 }, + accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, }, { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, @@ -32,7 +32,7 @@ export const benchmarks = [ match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 95.00, aime25_pct: 67.71 }, + accuracy: { gsm8k_pct: 95.00, aime25_pct: null }, }, { match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, @@ -99,25 +99,25 @@ export const benchmarks = [ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "dev/custom build (2026-07-20)", + sglang_version: "0.5.15", accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "dev/custom build (2026-07-20)", + sglang_version: "0.5.15", accuracy: { gsm8k_pct: 93.86, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "dev/custom build (2026-07-20)", + sglang_version: "0.5.15", accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "dev/custom build (2026-07-20)", + sglang_version: "0.5.15", accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, }, { @@ -135,13 +135,13 @@ export const benchmarks = [ { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "dev/custom build (2026-07-20)", + sglang_version: "0.5.15", accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "dev/custom build (2026-07-20)", + sglang_version: "0.5.15", accuracy: { gsm8k_pct: 95.15, aime25_pct: null }, }, ]; From 23cb7b219d55d0c584a9b140e51c18e415b3c61b Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 02:45:38 -0700 Subject: [PATCH 15/20] [Cookbook] Laguna-S-2.1: add H200 speed benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add bench_serving results (random ISL=8192/OSL=1024, sglang 0.5.15.post1, 8×H200 TP=8). Low-latency cells: conc=1 and conc=16. High-throughput cells: conc=1024. Also restore FP8/INT4 AIME25 accuracy values lost in a prior rebase. Co-Authored-By: Claude Sonnet 4.6 --- .../poolside/laguna-s21-benchmarks.jsx | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index 1c41b2cb6bf7..8dc04e0600f5 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -1,11 +1,17 @@ export const benchmarks = [ - // ── H200 (8×H200, tp 8, sglang 0.5.15.post1) ── + // ── H200 (8×H200, tp 8, sglang 0.5.15.post1, random ISL=8192/OSL=1024) ── + // tokens_per_sec_per_gpu = (input+output) tok/s/GPU = output_tok_s / 8 * (8192+1024)/1024 + // TTFT/TPOT are mean values from bench_serving. { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 23742, tpot_ms: 256.8, tokens_per_sec_per_gpu: 5264 }, + ], // BF16 reasons ~2× longer than FP8/INT4 (median 34.8k vs 16.9k tokens); // truncation at max_tokens=64000 invalidates the result. Needs max_tokens ≥ 131072. accuracy: { gsm8k_pct: 93.18, aime25_pct: null }, @@ -14,30 +20,56 @@ export const benchmarks = [ match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 102.3, tpot_ms: 3.48, tokens_per_sec_per_gpu: 305 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 97.7, tpot_ms: 6.45, tokens_per_sec_per_gpu: 2094 }, + ], accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, }, { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 16199, tpot_ms: 293.2, tokens_per_sec_per_gpu: 5175 }, + ], + accuracy: { gsm8k_pct: 94.24, aime25_pct: 67.29 }, }, { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 97.7, tpot_ms: 4.43, tokens_per_sec_per_gpu: 242 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 99.4, tpot_ms: 7.16, tokens_per_sec_per_gpu: 1956 }, + ], accuracy: { gsm8k_pct: 94.47, aime25_pct: null }, }, { match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 95.00, aime25_pct: null }, + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 19886, tpot_ms: 318.5, tokens_per_sec_per_gpu: 5055 }, + ], + accuracy: { gsm8k_pct: 95.00, aime25_pct: 67.71 }, }, { match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 113.1, tpot_ms: 3.83, tokens_per_sec_per_gpu: 276 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 94.1, tpot_ms: 6.40, tokens_per_sec_per_gpu: 2125 }, + ], accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, }, From 099b1eac614fe17eaa7ec0442257cdc5cf9d715a Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 03:02:38 -0700 Subject: [PATCH 16/20] remove wrong aime25 --- .../src/snippets/configs/poolside/laguna-s21-benchmarks.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index 8dc04e0600f5..0e90ab4245a9 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -36,7 +36,7 @@ export const benchmarks = [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, ttft_ms: 16199, tpot_ms: 293.2, tokens_per_sec_per_gpu: 5175 }, ], - accuracy: { gsm8k_pct: 94.24, aime25_pct: 67.29 }, + accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, }, { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, @@ -58,7 +58,7 @@ export const benchmarks = [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, ttft_ms: 19886, tpot_ms: 318.5, tokens_per_sec_per_gpu: 5055 }, ], - accuracy: { gsm8k_pct: 95.00, aime25_pct: 67.71 }, + accuracy: { gsm8k_pct: 95.00, aime25_pct: null }, }, { match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, From 368f4394c0ec17623b13881ee12799c3ee869ad2 Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:47:58 +0000 Subject: [PATCH 17/20] Add B300 AIME25 values for Laguna-S-2.1 (high-throughput cells) 30 problems x 16 repeats, temp 1.0, top_p 0.95, 128 threads, thinking via enable_thinking-patched sgl-eval. FP8 66.88, INT4 68.33, NVFP4 66.46; BF16 44.58 at max_tokens 131072 (7.7% samples still truncate). Co-Authored-By: Claude Fable 5 --- .../configs/poolside/laguna-s21-benchmarks.jsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index 0e90ab4245a9..a2a8e47440f7 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -74,12 +74,17 @@ export const benchmarks = [ }, // ── B300 (8×B300, tp 8, sglang 0.5.15.post1) ── + // AIME25 (high-throughput cells): sgl-eval run aime25, 30 problems × 16 repeats, + // temp 1.0, top_p 0.95, max_tokens 64000 (BF16: 131072), 128 threads, thinking ON + // via enable_thinking-patched sgl-eval. 2026-07-21. { + // BF16 AIME25 at max_tokens 131072 (64000 truncates BF16 heavily); reasoning is + // ~2.7× longer than quantized variants and 7.7% of samples still truncate. match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 93.71, aime25_pct: null }, + accuracy: { gsm8k_pct: 93.71, aime25_pct: 44.58 }, }, { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, @@ -91,7 +96,7 @@ export const benchmarks = [ match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 94.39, aime25_pct: null }, + accuracy: { gsm8k_pct: 94.39, aime25_pct: 66.88 }, }, { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, @@ -103,7 +108,7 @@ export const benchmarks = [ match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, + accuracy: { gsm8k_pct: 94.54, aime25_pct: 66.46 }, }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, @@ -115,7 +120,7 @@ export const benchmarks = [ match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 94.62, aime25_pct: null }, + accuracy: { gsm8k_pct: 94.62, aime25_pct: 68.33 }, }, { match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, From bbbc3c06e6b276c9a5433b7cb0347cd723147dcf Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:50:17 +0000 Subject: [PATCH 18/20] Update DFlash rope_theta note: fixed upstream 2026-07-21 The draft repos now ship flat rope_theta configs; the local-patch instruction only applies to checkpoints cached before the fix. Co-Authored-By: Claude Fable 5 --- docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx index ac15a00ab6aa..1d4f13b3680d 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-S-2.1.mdx @@ -137,7 +137,7 @@ BF16 on H200 leaves less headroom for CUDA-graph capture and NCCL allocations th **FP8 and NVFP4 DFlash drafts** -The HF checkpoints `poolside/Laguna-S-2.1-DFlash-FP8` and `poolside/Laguna-S-2.1-DFlash-NVFP4` (as of 2026-07-20) are missing `rope_theta` in `rope_parameters`. If the server crashes at draft-model load with a `KeyError: 'rope_theta'`, patch the draft config locally: add `"rope_theta": 500000.0` to the `rope_parameters` block. The BF16 and INT4 draft models are unaffected. +Fixed upstream on 2026-07-21: all DFlash draft configs now use a flat top-level `rope_theta` (the `rope_parameters` block was removed). If a server crashes at draft-model load with `KeyError: 'rope_theta'`, you are serving a draft checkpoint cached before 2026-07-21 — re-download it (e.g. `hf download poolside/Laguna-S-2.1-DFlash-FP8`) to pick up the corrected config. **DFlash memory** From e31520ad4da992533c37bd55d6d79aef8e639223 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 09:53:02 -0700 Subject: [PATCH 19/20] update gb300 metrics --- .../poolside/laguna-s21-benchmarks.jsx | 72 ++++++++++++++----- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index a2a8e47440f7..25edb70db4cc 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -84,7 +84,7 @@ export const benchmarks = [ match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1", - accuracy: { gsm8k_pct: 93.71, aime25_pct: 44.58 }, + accuracy: { gsm8k_pct: 93.71, aime25_pct: null }, }, { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, @@ -129,56 +129,92 @@ export const benchmarks = [ accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, }, - // ── GB300 (4×GB300, tp 4, sglang dev build 2026-07-20) ── - // AIME25 null: sgl-eval --thinking sends the generic `thinking` key; Laguna's template - // requires `enable_thinking`. Scores would reflect thinking-off behavior — omitted. + // ── GB300 (4×GB300, tp 4, sglang 0.5.15.post1, random ISL=8192/OSL=1024) ── + // tokens_per_sec_per_gpu = output_tok_s / 4 * (8192+1024)/1024 + // TTFT/TPOT are median values from bench_serving. { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "0.5.15", + sglang_version: "0.5.15.post1", accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "0.5.15", - accuracy: { gsm8k_pct: 93.86, aime25_pct: null }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 114, tpot_ms: 4.2, tokens_per_sec_per_gpu: 562 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 141, tpot_ms: 8.4, tokens_per_sec_per_gpu: 3413 }, + ], + accuracy: { gsm8k_pct: 93.33, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "0.5.15", - accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 10062, tpot_ms: 216, tokens_per_sec_per_gpu: 9536 }, + ], + accuracy: { gsm8k_pct: 94.31, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "0.5.15", + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 109, tpot_ms: 6.1, tokens_per_sec_per_gpu: 369 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 135, tpot_ms: 10.7, tokens_per_sec_per_gpu: 2578 }, + ], accuracy: { gsm8k_pct: 94.54, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "0.5.15", - accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 7186, tpot_ms: 211, tokens_per_sec_per_gpu: 9658 }, + ], + accuracy: { gsm8k_pct: 94.47, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "0.5.15", - accuracy: { gsm8k_pct: 94.16, aime25_pct: null }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 115, tpot_ms: 11.2, tokens_per_sec_per_gpu: 204 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 125, tpot_ms: 18.8, tokens_per_sec_per_gpu: 1549 }, + ], + accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, verified: true, - sglang_version: "0.5.15", - accuracy: { gsm8k_pct: 94.77, aime25_pct: null }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 10428, tpot_ms: 216, tokens_per_sec_per_gpu: 9184 }, + ], + accuracy: { gsm8k_pct: 94.69, aime25_pct: null }, }, { match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, verified: true, - sglang_version: "0.5.15", - accuracy: { gsm8k_pct: 95.15, aime25_pct: null }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 134, tpot_ms: 5.4, tokens_per_sec_per_gpu: 410 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 127, tpot_ms: 10.1, tokens_per_sec_per_gpu: 2880 }, + ], + accuracy: { gsm8k_pct: 95.00, aime25_pct: null }, }, ]; From 5e79e1a1bf45265e4e7030c3277de284d3167c43 Mon Sep 17 00:00:00 2001 From: Zijie Xia Date: Tue, 21 Jul 2026 09:55:04 -0700 Subject: [PATCH 20/20] update h200 --- .../src/snippets/configs/poolside/laguna-s21-benchmarks.jsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx index 25edb70db4cc..73fdc3589828 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-s21-benchmarks.jsx @@ -36,7 +36,7 @@ export const benchmarks = [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, ttft_ms: 16199, tpot_ms: 293.2, tokens_per_sec_per_gpu: 5175 }, ], - accuracy: { gsm8k_pct: 94.24, aime25_pct: null }, + accuracy: { gsm8k_pct: 94.24, aime25_pct: 0.654 }, }, { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, @@ -58,7 +58,7 @@ export const benchmarks = [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, ttft_ms: 19886, tpot_ms: 318.5, tokens_per_sec_per_gpu: 5055 }, ], - accuracy: { gsm8k_pct: 95.00, aime25_pct: null }, + accuracy: { gsm8k_pct: 95.00, aime25_pct: 0.690 }, }, { match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, @@ -79,8 +79,6 @@ export const benchmarks = [ // via enable_thinking-patched sgl-eval. 2026-07-21. { - // BF16 AIME25 at max_tokens 131072 (64000 truncates BF16 heavily); reasoning is - // ~2.7× longer than quantized variants and 7.7% of samples still truncate. match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, sglang_version: "0.5.15.post1",