From cc07275279dffe59b29dc81b2c2f99a2ce0d89a2 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Thu, 23 Apr 2026 05:32:09 +0000 Subject: [PATCH 1/2] docs: add Hunyuan 3 Preview cookbook - Section 1: MoE architecture (~276B / ~20B active), hybrid thinking (reasoning_effort high/medium/low/none), 256K context, MTP - Section 2: Docker image table (lmsysorg/sglang:hy3-preview{,-cu130}) - Section 3: Interactive Hunyuan3PreviewDeployment jsx generator (NVIDIA A100/H100/H200/B200/B300/GB300, FP8/BF16, Blackwell auto-injects --attention-backend trtllm_mha, MTP toggle prepends SGLANG_ENABLE_SPEC_V2=1 and --speculative-algorithm EAGLE flags) - Section 4: Real invocation outputs (simple completion, thinking high-effort, instant mode, non-stream tool call, streaming tool call) - Section 5: GSM8K (95.0%), MMLU (82.5%), tool-call accuracy via MiniMax-Provider-Verifier, low/high-concurrency bench_serving results on 4x H200 --- .../Tencent/Hunyuan3-Preview.mdx | 616 ++++++++++++++++++ docs_new/docs.json | 6 + .../hunyuan3-preview-deployment.jsx | 206 ++++++ 3 files changed, 828 insertions(+) create mode 100644 docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx create mode 100644 docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx diff --git a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx new file mode 100644 index 000000000000..f4801b50df28 --- /dev/null +++ b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx @@ -0,0 +1,616 @@ +--- +title: Hunyuan 3 Preview +metatags: + description: "Deploy Tencent Hunyuan 3 Preview (~276B / ~20B active MoE) on NVIDIA GPUs with SGLang — hybrid thinking, native tool calling, 256K context, and built-in MTP speculative decoding." +tag: NEW +--- + +## 1. Model Introduction + +Hunyuan 3 Preview (Hy3-preview) is Tencent's preview of its third-generation flagship MoE language model, featuring hybrid thinking, native tool calling, long-context reasoning, and Multi-Token Prediction (MTP) for low-latency serving. + +**Key Features:** + +- **MoE Architecture**: 192 routed experts + 1 shared expert, 8 experts activated per token. ~276B total parameters with ~20B active, delivering dense-model quality at MoE inference cost. +- **Hybrid Thinking**: Reasoning modes (`high`, `medium`, `low`, `none`) controllable via OpenAI-standard `reasoning_effort`, allowing the same weights to trade off latency and depth of reasoning. +- **Native Tool Calling**: Trained on structured `` / `` / `` grammar. Pairs with SGLang's `hunyuan` tool-call parser for streaming OpenAI-compatible function-calling output. +- **Long Context**: 256K token context window (262,144 positions) for repository-scale code and document reasoning. +- **Multi-Token Prediction (MTP)**: Ships with a built-in MTP draft module enabling speculative decoding out of the box. + +**Available Models:** + +- [tencent/Hy3-preview](https://huggingface.co/tencent/Hy3-preview) — BF16 instruct +- [tencent/Hy3-preview-FP8](https://huggingface.co/tencent/Hy3-preview-FP8) — FP8 instruct +- [tencent/Hy3-preview-Base](https://huggingface.co/tencent/Hy3-preview-Base) — BF16 base + +**Recommended Generation Parameters:** + + + + + + + + + + + + + + + + + + + + + + +
ParameterValue
`temperature`0.7
`top_p`0.9
`reasoning_effort``high` / `medium` / `low` (thinking) or `none` (instant)
+ +**License:** TODO — verify on HuggingFace model card. + +## 2. SGLang Installation + +SGLang offers multiple installation methods. You can choose the most suitable installation method based on your hardware platform and requirements. + +Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions. + +**Docker Images by Hardware Platform:** + + + + + + + + + + + + + + + + + + +
Hardware PlatformDocker Image
NVIDIA A100 / H100 / H200 / B200`lmsysorg/sglang:hy3-preview`
NVIDIA B300 / GB300`lmsysorg/sglang:hy3-preview-cu130`
+ +The `hy3-preview` tag bundles the HYV3 model code, the `hunyuan` tool-call / reasoning parsers, and the MTP draft-module runtime. + +## 3. Model Deployment + +This section provides deployment configurations optimized for different hardware platforms and use cases. + +### 3.1 Basic Configuration + +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, quantization, and feature capabilities. + +import { Hunyuan3PreviewDeployment } from '/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx' + + + +### 3.2 Configuration Tips + +**Key Parameters:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ParameterDescriptionRecommended Value
`--tool-call-parser`Tool call parser for function-calling support`hunyuan`
`--reasoning-parser`Reasoning parser for hybrid thinking modes`hunyuan`
`--trust-remote-code`Required for Hunyuan model loadingAlways enabled
`--mem-fraction-static`Static memory fraction (KV + activations)`0.9`
`--tp`Tensor parallelism size`2` / `4` / `8` depending on hardware
`--attention-backend`Attention backend (Blackwell only)`trtllm_mha`
`--speculative-algorithm`Speculative decoding via the bundled MTP draft`EAGLE` + `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` (set env `SGLANG_ENABLE_SPEC_V2=1`)
+ +**Hardware Requirements: NVIDIA FP8 (`Hy3-preview-FP8`, ~276GB weights)** + +- **A100 (80GB) / H100 (80GB)**: TP=8 (minimum to fit weights with KV-cache headroom). +- **H200 (141GB) / B200 (180GB)**: TP=4 (balanced single-node default). +- **B300 (275GB)**: TP=2 (weights fit with headroom); TP=4 recommended for higher throughput. +- **GB300 (275GB, 4-GPU single-node)**: TP=4 (practical ceiling on a typical GB300 node). + +**Hardware Requirements: NVIDIA BF16 (`Hy3-preview`, ~552GB weights)** + +- **H200 (141GB) / B200 (180GB)**: TP=8 (minimum for BF16 to fit single-node). +- **B300 (275GB) / GB300**: TP=4. +- **A100 / H100 (80GB)**: not supported single-node — BF16 requires multi-node TP=16+ on 80GB-class GPUs. + +**Blackwell (B200 / B300 / GB300):** Auto-selected attention backend can mis-route for HYV3 on Blackwell. Always pass `--attention-backend trtllm_mha` explicitly on Blackwell hardware (the config generator above enforces this). + +**Multi-Token Prediction (MTP):** The `Hy3-preview` / `Hy3-preview-FP8` releases bundle an MTP draft module. SGLang runs it via its EAGLE speculative-decoding path — the draft module auto-loads from the same `--model-path`. Enable with the `SGLANG_ENABLE_SPEC_V2=1` env var and the standard MTP flags: + +```bash Command +SGLANG_ENABLE_SPEC_V2=1 sglang serve \ + --model-path tencent/Hy3-preview-FP8 \ + --tp 4 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --reasoning-parser hunyuan \ + --tool-call-parser hunyuan \ + --trust-remote-code \ + --mem-fraction-static 0.85 +``` + +Toggle the "Speculative Decoding (MTP)" option in the generator above to add these flags automatically. Tune `num-steps` / `num-draft-tokens` based on acceptance rate in your workload. + +## 4. Model Invocation + +### 4.1 Basic Usage + +For basic API usage and request examples, please refer to: + +- [SGLang Basic Usage Guide](../../../docs/basic_usage/send_request) + +**Deployment Command (H200, FP8, default):** + +```bash Command +sglang serve \ + --model-path tencent/Hy3-preview-FP8 \ + --tp 4 \ + --reasoning-parser hunyuan \ + --tool-call-parser hunyuan \ + --trust-remote-code \ + --mem-fraction-static 0.9 +``` + +**Testing Deployment:** + +After startup, you can test the SGLang OpenAI-compatible API with the following command: + +```bash Command +curl http://localhost:30000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tencent/Hy3-preview-FP8", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Who won the world series in 2020?"} + ] + }' +``` + +**Simple Completion Example:** + +```python Example +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:30000/v1", + api_key="EMPTY" +) + +response = client.chat.completions.create( + model="tencent/Hy3-preview-FP8", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Who won the world series in 2020?"} + ], + max_tokens=1024 +) + +print("Reasoning:", response.choices[0].message.reasoning_content) +print("Content: ", response.choices[0].message.content) +``` + +**Output Example:** + +```text Output +Reasoning: None +Content: The Los Angeles Dodgers won the 2020 World Series. They defeated the Tampa Bay Rays in six games (4-2). This was the Dodgers' first World Series championship since 1988. The series was notable for being played in a neutral-site bubble at Globe Life Field in Arlington, Texas, due to the COVID-19 pandemic. +``` + +When `reasoning_effort` is not set, the server defaults to instant mode (no thinking, `reasoning_content=None`). To opt into thinking, pass `reasoning_effort="high" / "medium" / "low"` on the request — see the Hybrid Thinking section below. + +### 4.2 Advanced Usage + +#### 4.2.1 Reasoning Parser (Hybrid Thinking) + +Hy3-preview is a hybrid-thinking model. Control the thinking budget via the OpenAI-standard `reasoning_effort`: + +- `high` / `medium` / `low` — increasing amounts of chain-of-thought in `reasoning_content` +- `none` — skip thinking entirely (instant responses, content-only) + +Enable the reasoning parser during deployment so that the thinking section (`...`) is separated into `reasoning_content`: + +```bash Command +sglang serve \ + --model-path tencent/Hy3-preview-FP8 \ + --tp 4 \ + --reasoning-parser hunyuan \ + --trust-remote-code \ + --mem-fraction-static 0.9 +``` + +**Thinking Mode — High Effort:** + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="tencent/Hy3-preview-FP8", + messages=[{"role": "user", "content": "Solve step by step: What is 15% of 240?"}], + reasoning_effort="high", + max_tokens=2048, +) + +msg = response.choices[0].message +print("=============== Thinking =================") +print(msg.reasoning_content) +print("=============== Content =================") +print(msg.content) +``` + +**Output Example:** + +```text Output +=============== Thinking ================= +We need to solve: "What is 15% of 240?" Step by step. So we need to compute 15% of 240. The process: 15% means 15 per hundred, i.e., 15/100 = 0.15. Multiply 0.15 by 240. Or we can do: 10% of 240 = 24, 5% is half of 10% = 12, so sum = 36. Or do multiplication: 15/100 * 240 = (15*240)/100 = (3600)/100 = 36. So answer is 36. + +We need to produce step-by-step explanation. The instruction: "Solve step by step: What is 15% of 240?" So we should provide a clear solution with steps. The final answer: 36. Also maybe include units? No units. + +We'll output the solution in a clear manner. +=============== Content ================= +To find 15% of 240, follow these steps: + +1. **Understand that percent means "per hundred."** + So, 15% = 15/100 or 0.15. + +2. **Multiply the number (240) by the percentage in decimal form.** + 0.15 × 240. + + Alternatively, you can use fractions: + (15/100) × 240. + +3. **Perform the multiplication.** + 0.15 × 240 = 36. + Or: + (15 × 240) / 100 = 3600 / 100 = 36. + +4. **Check using an alternative method:** + - 10% of 240 = 24. + - 5% of 240 = half of 10% = 12. + - 15% = 10% + 5% = 24 + 12 = 36. + +Thus, **15% of 240 is 36**. +``` + +**Instant Mode — No Thinking:** + +```python Example +response = client.chat.completions.create( + model="tencent/Hy3-preview-FP8", + messages=[{"role": "user", "content": "Give me a one-line summary of relativity."}], + reasoning_effort="none", + max_tokens=256, +) + +print("Content:", response.choices[0].message.content) +``` + +**Output Example:** + +```text Output +Content: Relativity is Einstein's theory that space, time, mass, and gravity are interconnected and relative, not fixed, fundamentally changing our understanding of the universe. +``` + +#### 4.2.2 Tool Calling + +Hy3-preview supports streaming OpenAI-compatible tool calls. Enable both parsers together — the reasoning parser strips thinking tokens before the tool-call parser runs: + +```bash Command +sglang serve \ + --model-path tencent/Hy3-preview-FP8 \ + --tp 4 \ + --reasoning-parser hunyuan \ + --tool-call-parser hunyuan \ + --trust-remote-code \ + --mem-fraction-static 0.9 +``` + +**Non-Streaming Example:** + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["city"], + }, + }, + } +] + +response = client.chat.completions.create( + model="tencent/Hy3-preview-FP8", + messages=[{"role": "user", "content": "What's the weather in Beijing? Use fahrenheit."}], + tools=tools, +) + +msg = response.choices[0].message +print("Reasoning:", msg.reasoning_content) +print("Content: ", msg.content) +for tc in msg.tool_calls or []: + print(f"Tool Call: {tc.function.name}") + print(f" Arguments: {tc.function.arguments}") +``` + +**Output Example:** + +```text Output +Reasoning: None +Content: I'll get the current weather for Beijing in Fahrenheit for you. +Tool Call: get_weather + Arguments: {"city": "Beijing", "unit": "fahrenheit"} +``` + +**Streaming Example (incremental argument deltas):** + +Hy3-preview's `hunyuan` tool-call parser emits tool names first, then argument JSON in incremental fragments — matching the OpenAI streaming contract: + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +stream = client.chat.completions.create( + model="tencent/Hy3-preview-FP8", + messages=[{"role": "user", "content": "What's the weather in Beijing? Use fahrenheit."}], + tools=tools, + stream=True, +) + +tool_buffer = {} +for chunk in stream: + delta = chunk.choices[0].delta + if delta.content: + print(delta.content, end="", flush=True) + for tc in delta.tool_calls or []: + buf = tool_buffer.setdefault(tc.index, {"name": "", "args": ""}) + if tc.function and tc.function.name: + buf["name"] += tc.function.name + if tc.function and tc.function.arguments: + buf["args"] += tc.function.arguments + +for idx, buf in tool_buffer.items(): + print(f"\nTool[{idx}] {buf['name']}({buf['args']})") +``` + +**Output Example:** + +```text Output +I'll check the current weather in Beijing for you using Fahrenheit. +Tool[0] get_weather({"city": "Beijing", "unit": "fahrenheit"}) +``` + +## 5. Benchmark + +### 5.1 Accuracy Benchmark + +**Test Environment:** + +- Hardware: 4× NVIDIA H200 (141GB) +- Docker Image: `lmsysorg/sglang:hy3-preview` +- Model: `tencent/Hy3-preview-FP8` +- Tensor Parallelism: 4 +- SGLang version: latest `main` + +#### 5.1.1 GSM8K + +- Benchmark Method: 5-shot CoT on 200 questions, evaluated via SGLang native backend +- Benchmark Command: + +```bash Command +python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --parallel 64 +``` + +- Test Results: + +```text Output +Accuracy: 0.950 +Invalid: 0.000 +Latency: 14.423 s +Output throughput: 1596.305 token/s +``` + +#### 5.1.2 MMLU + +- Benchmark Method: 5-shot, all 57 subjects +- Benchmark Command: + +```bash Command +python3 benchmark/mmlu/bench_sglang.py --nsub 60 --parallel 64 +``` + +- Test Results: + +```text Output +Average accuracy: 0.825 +Total latency: 79.652 s +``` + +#### 5.1.3 Tool-Call Accuracy (MiniMax-Provider-Verifier) + +- Benchmark Tool: [MiniMax-Provider-Verifier](https://github.com/MiniMax-AI/MiniMax-Provider-Verifier) +- Metric: function-call schema validity, argument match, and end-to-end response correctness +- Test Results: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MetricScore
Query-Success-Rate100.00%
ToolCalls-Match-Rate98.02%
ToolCalls-Schema-Accuracy96.43%
Response-Success-Rate100.00%
Language-Following-Success-Rate100.00%
+ +### 5.2 Speed Benchmark + +#### 5.2.1 Low Concurrency + +- Benchmark Command: + +```bash Command +python3 -m sglang.bench_serving \ + --backend sglang \ + --model tencent/Hy3-preview-FP8 \ + --dataset-name random \ + --random-input-len 1000 \ + --random-output-len 1000 \ + --num-prompts 10 \ + --max-concurrency 1 +``` + +- Test Results: + +```text Output +============ Serving Benchmark Result ============ +Backend: sglang +Traffic request rate: inf +Max request concurrency: 1 +Successful requests: 10 +Benchmark duration (s): 33.24 +Total input tokens: 6101 +Total generated tokens: 4220 +Request throughput (req/s): 0.30 +Input token throughput (tok/s): 183.52 +Output token throughput (tok/s): 126.94 +Total token throughput (tok/s): 310.46 +Concurrency: 1.00 +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): 3322.95 +Median E2E Latency (ms): 2665.29 +---------------Time to First Token---------------- +Mean TTFT (ms): 56.65 +Median TTFT (ms): 64.47 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 7.74 +Median TPOT (ms): 7.76 +---------------Inter-Token Latency---------------- +Mean ITL (ms): 7.76 +Median ITL (ms): 7.77 +================================================== +``` + +#### 5.2.2 High Concurrency + +- Benchmark Command: + +```bash Command +python3 -m sglang.bench_serving \ + --backend sglang \ + --model tencent/Hy3-preview-FP8 \ + --dataset-name random \ + --random-input-len 1000 \ + --random-output-len 1000 \ + --num-prompts 500 \ + --max-concurrency 100 +``` + +- Test Results: + +```text Output +============ Serving Benchmark Result ============ +Backend: sglang +Traffic request rate: inf +Max request concurrency: 100 +Successful requests: 500 +Benchmark duration (s): 102.58 +Total input tokens: 249831 +Total generated tokens: 252662 +Request throughput (req/s): 4.87 +Input token throughput (tok/s): 2435.54 +Output token throughput (tok/s): 2463.14 +Total token throughput (tok/s): 4898.68 +Concurrency: 90.35 +----------------End-to-End Latency---------------- +Mean E2E Latency (ms): 18535.76 +Median E2E Latency (ms): 18421.99 +---------------Time to First Token---------------- +Mean TTFT (ms): 273.79 +Median TTFT (ms): 101.47 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 36.62 +Median TPOT (ms): 37.58 +---------------Inter-Token Latency---------------- +Mean ITL (ms): 36.21 +Median ITL (ms): 30.02 +================================================== +``` diff --git a/docs_new/docs.json b/docs_new/docs.json index 6395e77452fc..f16c2dc6b046 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -1067,6 +1067,12 @@ "pages": [ "cookbook/autoregressive/FlashLabs/Chroma1.0" ] + }, + { + "group": "Tencent", + "pages": [ + "cookbook/autoregressive/Tencent/Hunyuan3-Preview" + ] } ] }, diff --git a/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx b/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx new file mode 100644 index 000000000000..ae5664c32098 --- /dev/null +++ b/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx @@ -0,0 +1,206 @@ +export const Hunyuan3PreviewDeployment = () => { + // Hunyuan 3 Preview (~276B total / ~20B active MoE) — NVIDIA-only single-node TP layout. + // FP8 (~276GB): A100 tp=8, H100 tp=8, H200 tp=4, B200 tp=4, B300 tp=2, GB300 tp=4 + // BF16 (~552GB): H200 tp=8, B200 tp=8, B300 tp=4, GB300 tp=4 (80GB-class GPUs skipped) + const options = { + hardware: { + name: 'hardware', + title: 'Hardware Platform', + items: [ + { id: 'h200', label: 'H200', default: true }, + { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, + { id: 'gb300', label: 'GB300', default: false }, + { id: 'h100', label: 'H100', default: false }, + { id: 'a100', label: 'A100', default: false } + ] + }, + quantization: { + name: 'quantization', + title: 'Quantization', + getDynamicItems: (values) => { + const hw = values.hardware; + const bf16Unsupported = hw === 'a100' || hw === 'h100'; + return [ + { id: 'fp8', label: 'FP8', subtitle: 'Recommended', default: true }, + { id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: false, + disabled: bf16Unsupported, + disabledReason: bf16Unsupported ? 'BF16 (~552GB) does not fit single-node on 80GB GPUs' : '' } + ]; + } + }, + reasoning: { + name: 'reasoning', + title: 'Reasoning Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: false }, + { id: 'enabled', label: 'Enabled', default: true } + ] + }, + toolcall: { + name: 'toolcall', + title: 'Tool Call Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: false }, + { id: 'enabled', label: 'Enabled', default: true } + ] + }, + speculative: { + name: 'speculative', + title: 'Speculative Decoding (MTP)', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', subtitle: 'Low Latency', default: false } + ] + } + }; + + const modelConfigs = { + a100: { fp8: { tp: 8, mem: 0.9 } }, + h100: { fp8: { tp: 8, mem: 0.9 } }, + h200: { fp8: { tp: 4, mem: 0.9 }, bf16: { tp: 8, mem: 0.9 } }, + b200: { fp8: { tp: 4, mem: 0.9 }, bf16: { tp: 8, mem: 0.9 } }, + b300: { fp8: { tp: 2, mem: 0.9 }, bf16: { tp: 4, mem: 0.9 } }, + gb300: { fp8: { tp: 4, mem: 0.9 }, bf16: { tp: 4, mem: 0.9 } } + }; + + const resolveItems = (option, values) => { + if (typeof option.getDynamicItems === 'function') return option.getDynamicItems(values); + return option.items; + }; + + const getInitialState = () => { + const initialState = {}; + for (const [key, option] of Object.entries(options)) { + const items = resolveItems(option, initialState); + const def = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0]; + initialState[key] = def.id; + } + return initialState; + }; + + const [values, setValues] = useState(getInitialState); + const [isDark, setIsDark] = useState(false); + + useEffect(() => { + const checkDarkMode = () => { + const html = document.documentElement; + const isDarkMode = html.classList.contains('dark') || + html.getAttribute('data-theme') === 'dark' || + html.style.colorScheme === 'dark'; + setIsDark(isDarkMode); + }; + checkDarkMode(); + const observer = new MutationObserver(checkDarkMode); + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] }); + return () => observer.disconnect(); + }, []); + + useEffect(() => { + setValues(prev => { + const next = { ...prev }; + for (const [key, option] of Object.entries(options)) { + if (typeof option.getDynamicItems !== 'function') continue; + const items = option.getDynamicItems(next); + const current = items.find(i => i.id === next[key]); + if (!current || current.disabled) { + const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled); + if (fallback) next[key] = fallback.id; + } + } + return next; + }); + }, [values.hardware]); + + const handleRadioChange = (optionName, value) => { + setValues(prev => ({ ...prev, [optionName]: value })); + }; + + const generateCommand = () => { + const { hardware, quantization } = values; + const isBlackwell = hardware === 'b200' || hardware === 'b300' || hardware === 'gb300'; + const hwConfig = modelConfigs[hardware] && modelConfigs[hardware][quantization]; + if (!hwConfig) return '# Configuration not available for the selected hardware and quantization.'; + + const suffix = quantization === 'fp8' ? '-FP8' : ''; + const modelName = `tencent/Hy3-preview${suffix}`; + const tpValue = hwConfig.tp; + const memFraction = hwConfig.mem; + const enableSpec = values.speculative === 'enabled'; + + let cmd = ''; + if (enableSpec) cmd += 'SGLANG_ENABLE_SPEC_V2=1 '; + cmd += 'sglang serve \\\n'; + cmd += ` --model-path ${modelName}`; + cmd += ` \\\n --tp ${tpValue}`; + + if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser hunyuan'; + if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser hunyuan'; + if (enableSpec) { + cmd += ' \\\n --speculative-algorithm EAGLE'; + cmd += ' \\\n --speculative-num-steps 3'; + cmd += ' \\\n --speculative-eagle-topk 1'; + cmd += ' \\\n --speculative-num-draft-tokens 4'; + } + + cmd += ' \\\n --trust-remote-code'; + cmd += ` \\\n --mem-fraction-static ${memFraction}`; + + if (isBlackwell) cmd += ' \\\n --attention-backend trtllm_mha'; + + return cmd; + }; + + const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' }; + const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' }; + const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' }; + const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 }; + const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' }; + const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' }; + const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 }; + const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 }; + const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` }; + + return ( +
+ {Object.entries(options).map(([key, option]) => { + if (typeof option.condition === 'function' && !option.condition(values)) return null; + const items = resolveItems(option, values); + return ( +
+
{option.title}
+
+ {items.map(item => { + const isChecked = values[option.name] === item.id; + const isDisabled = !!item.disabled; + return ( + + ); + })} +
+
+ ); + })} +
+
Run this Command:
+ 
{generateCommand()}
+
+
+ ); +}; From f90df7f3063b432327bebbd3a28bb0b9c071393f Mon Sep 17 00:00:00 2001 From: Xinyuan Tong Date: Thu, 23 Apr 2026 05:41:43 +0000 Subject: [PATCH 2/2] docs(hunyuan): drop FP8 variant (BF16-only release) - Remove Hy3-preview-FP8 from Available Models list - Remove FP8 Hardware Requirements block (Section 3.2) - JSX generator: drop A100/H100 hardware (BF16 won't fit single-node on 80GB GPUs) and drop the FP8 quantization option - All deploy commands switch from '--tp 4' to '--tp 8' (H200 BF16 default) - Docker table: narrow to H200/B200 and B300/GB300 - Section 5 benchmarks: replace FP8-Testing numbers with TODO placeholders for BF16 re-measure --- .../Tencent/Hunyuan3-Preview.mdx | 149 ++++-------------- .../hunyuan3-preview-deployment.jsx | 62 ++------ 2 files changed, 45 insertions(+), 166 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx index f4801b50df28..84749039bc87 100644 --- a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx +++ b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx @@ -1,7 +1,7 @@ --- title: Hunyuan 3 Preview metatags: - description: "Deploy Tencent Hunyuan 3 Preview (~276B / ~20B active MoE) on NVIDIA GPUs with SGLang — hybrid thinking, native tool calling, 256K context, and built-in MTP speculative decoding." + description: "Deploy Tencent Hunyuan 3 Preview BF16 (~276B / ~20B active MoE) on NVIDIA GPUs with SGLang — hybrid thinking, native tool calling, 256K context, and built-in MTP speculative decoding." tag: NEW --- @@ -20,7 +20,6 @@ Hunyuan 3 Preview (Hy3-preview) is Tencent's preview of its third-generation fla **Available Models:** - [tencent/Hy3-preview](https://huggingface.co/tencent/Hy3-preview) — BF16 instruct -- [tencent/Hy3-preview-FP8](https://huggingface.co/tencent/Hy3-preview-FP8) — FP8 instruct - [tencent/Hy3-preview-Base](https://huggingface.co/tencent/Hy3-preview-Base) — BF16 base **Recommended Generation Parameters:** @@ -67,7 +66,7 @@ Please refer to the [official SGLang installation guide](../../../docs/get-start - NVIDIA A100 / H100 / H200 / B200 + NVIDIA H200 / B200 `lmsysorg/sglang:hy3-preview` @@ -142,13 +141,6 @@ import { Hunyuan3PreviewDeployment } from '/src/snippets/autoregressive/hunyuan3 -**Hardware Requirements: NVIDIA FP8 (`Hy3-preview-FP8`, ~276GB weights)** - -- **A100 (80GB) / H100 (80GB)**: TP=8 (minimum to fit weights with KV-cache headroom). -- **H200 (141GB) / B200 (180GB)**: TP=4 (balanced single-node default). -- **B300 (275GB)**: TP=2 (weights fit with headroom); TP=4 recommended for higher throughput. -- **GB300 (275GB, 4-GPU single-node)**: TP=4 (practical ceiling on a typical GB300 node). - **Hardware Requirements: NVIDIA BF16 (`Hy3-preview`, ~552GB weights)** - **H200 (141GB) / B200 (180GB)**: TP=8 (minimum for BF16 to fit single-node). @@ -157,12 +149,12 @@ import { Hunyuan3PreviewDeployment } from '/src/snippets/autoregressive/hunyuan3 **Blackwell (B200 / B300 / GB300):** Auto-selected attention backend can mis-route for HYV3 on Blackwell. Always pass `--attention-backend trtllm_mha` explicitly on Blackwell hardware (the config generator above enforces this). -**Multi-Token Prediction (MTP):** The `Hy3-preview` / `Hy3-preview-FP8` releases bundle an MTP draft module. SGLang runs it via its EAGLE speculative-decoding path — the draft module auto-loads from the same `--model-path`. Enable with the `SGLANG_ENABLE_SPEC_V2=1` env var and the standard MTP flags: +**Multi-Token Prediction (MTP):** The `Hy3-preview` release bundles an MTP draft module. SGLang runs it via its EAGLE speculative-decoding path — the draft module auto-loads from the same `--model-path`. Enable with the `SGLANG_ENABLE_SPEC_V2=1` env var and the standard MTP flags: ```bash Command SGLANG_ENABLE_SPEC_V2=1 sglang serve \ - --model-path tencent/Hy3-preview-FP8 \ - --tp 4 \ + --model-path tencent/Hy3-preview \ + --tp 8 \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ @@ -183,12 +175,12 @@ For basic API usage and request examples, please refer to: - [SGLang Basic Usage Guide](../../../docs/basic_usage/send_request) -**Deployment Command (H200, FP8, default):** +**Deployment Command (H200 × 8, BF16 default):** ```bash Command sglang serve \ - --model-path tencent/Hy3-preview-FP8 \ - --tp 4 \ + --model-path tencent/Hy3-preview \ + --tp 8 \ --reasoning-parser hunyuan \ --tool-call-parser hunyuan \ --trust-remote-code \ @@ -203,7 +195,7 @@ After startup, you can test the SGLang OpenAI-compatible API with the following curl http://localhost:30000/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ - "model": "tencent/Hy3-preview-FP8", + "model": "tencent/Hy3-preview", "messages": [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Who won the world series in 2020?"} @@ -222,7 +214,7 @@ client = OpenAI( ) response = client.chat.completions.create( - model="tencent/Hy3-preview-FP8", + model="tencent/Hy3-preview", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Who won the world series in 2020?"} @@ -256,8 +248,8 @@ Enable the reasoning parser during deployment so that the thinking section (` - - - Metric - Score - - - - - Query-Success-Rate - 100.00% - - - ToolCalls-Match-Rate - 98.02% - - - ToolCalls-Schema-Accuracy - 96.43% - - - Response-Success-Rate - 100.00% - - - Language-Following-Success-Rate - 100.00% - - - +```text Output +TODO — replace with real tool-call accuracy after benchmark run on Hy3-preview (BF16). +``` ### 5.2 Speed Benchmark @@ -530,7 +491,7 @@ Total latency: 79.652 s ```bash Command python3 -m sglang.bench_serving \ --backend sglang \ - --model tencent/Hy3-preview-FP8 \ + --model tencent/Hy3-preview \ --dataset-name random \ --random-input-len 1000 \ --random-output-len 1000 \ @@ -541,32 +502,7 @@ python3 -m sglang.bench_serving \ - Test Results: ```text Output -============ Serving Benchmark Result ============ -Backend: sglang -Traffic request rate: inf -Max request concurrency: 1 -Successful requests: 10 -Benchmark duration (s): 33.24 -Total input tokens: 6101 -Total generated tokens: 4220 -Request throughput (req/s): 0.30 -Input token throughput (tok/s): 183.52 -Output token throughput (tok/s): 126.94 -Total token throughput (tok/s): 310.46 -Concurrency: 1.00 -----------------End-to-End Latency---------------- -Mean E2E Latency (ms): 3322.95 -Median E2E Latency (ms): 2665.29 ----------------Time to First Token---------------- -Mean TTFT (ms): 56.65 -Median TTFT (ms): 64.47 ------Time per Output Token (excl. 1st token)------ -Mean TPOT (ms): 7.74 -Median TPOT (ms): 7.76 ----------------Inter-Token Latency---------------- -Mean ITL (ms): 7.76 -Median ITL (ms): 7.77 -================================================== +TODO — replace with real low-concurrency output on Hy3-preview (BF16). ``` #### 5.2.2 High Concurrency @@ -576,7 +512,7 @@ Median ITL (ms): 7.77 ```bash Command python3 -m sglang.bench_serving \ --backend sglang \ - --model tencent/Hy3-preview-FP8 \ + --model tencent/Hy3-preview \ --dataset-name random \ --random-input-len 1000 \ --random-output-len 1000 \ @@ -587,30 +523,5 @@ python3 -m sglang.bench_serving \ - Test Results: ```text Output -============ Serving Benchmark Result ============ -Backend: sglang -Traffic request rate: inf -Max request concurrency: 100 -Successful requests: 500 -Benchmark duration (s): 102.58 -Total input tokens: 249831 -Total generated tokens: 252662 -Request throughput (req/s): 4.87 -Input token throughput (tok/s): 2435.54 -Output token throughput (tok/s): 2463.14 -Total token throughput (tok/s): 4898.68 -Concurrency: 90.35 -----------------End-to-End Latency---------------- -Mean E2E Latency (ms): 18535.76 -Median E2E Latency (ms): 18421.99 ----------------Time to First Token---------------- -Mean TTFT (ms): 273.79 -Median TTFT (ms): 101.47 ------Time per Output Token (excl. 1st token)------ -Mean TPOT (ms): 36.62 -Median TPOT (ms): 37.58 ----------------Inter-Token Latency---------------- -Mean ITL (ms): 36.21 -Median ITL (ms): 30.02 -================================================== +TODO — replace with real high-concurrency output on Hy3-preview (BF16). ``` diff --git a/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx b/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx index ae5664c32098..82abba7a8b21 100644 --- a/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/hunyuan3-preview-deployment.jsx @@ -1,7 +1,10 @@ export const Hunyuan3PreviewDeployment = () => { - // Hunyuan 3 Preview (~276B total / ~20B active MoE) — NVIDIA-only single-node TP layout. - // FP8 (~276GB): A100 tp=8, H100 tp=8, H200 tp=4, B200 tp=4, B300 tp=2, GB300 tp=4 - // BF16 (~552GB): H200 tp=8, B200 tp=8, B300 tp=4, GB300 tp=4 (80GB-class GPUs skipped) + // Hunyuan 3 Preview (~276B total / ~20B active MoE) — BF16 only. + // ~552GB weights; 80GB-class GPUs (A100/H100) cannot fit single-node. + // H200 (141GB): tp=8 + // B200 (180GB): tp=8 + // B300 (275GB): tp=4 + // GB300 (275GB, 4-GPU node): tp=4 const options = { hardware: { name: 'hardware', @@ -10,25 +13,9 @@ export const Hunyuan3PreviewDeployment = () => { { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, { id: 'b300', label: 'B300', default: false }, - { id: 'gb300', label: 'GB300', default: false }, - { id: 'h100', label: 'H100', default: false }, - { id: 'a100', label: 'A100', default: false } + { id: 'gb300', label: 'GB300', default: false } ] }, - quantization: { - name: 'quantization', - title: 'Quantization', - getDynamicItems: (values) => { - const hw = values.hardware; - const bf16Unsupported = hw === 'a100' || hw === 'h100'; - return [ - { id: 'fp8', label: 'FP8', subtitle: 'Recommended', default: true }, - { id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: false, - disabled: bf16Unsupported, - disabledReason: bf16Unsupported ? 'BF16 (~552GB) does not fit single-node on 80GB GPUs' : '' } - ]; - } - }, reasoning: { name: 'reasoning', title: 'Reasoning Parser', @@ -56,12 +43,10 @@ export const Hunyuan3PreviewDeployment = () => { }; const modelConfigs = { - a100: { fp8: { tp: 8, mem: 0.9 } }, - h100: { fp8: { tp: 8, mem: 0.9 } }, - h200: { fp8: { tp: 4, mem: 0.9 }, bf16: { tp: 8, mem: 0.9 } }, - b200: { fp8: { tp: 4, mem: 0.9 }, bf16: { tp: 8, mem: 0.9 } }, - b300: { fp8: { tp: 2, mem: 0.9 }, bf16: { tp: 4, mem: 0.9 } }, - gb300: { fp8: { tp: 4, mem: 0.9 }, bf16: { tp: 4, mem: 0.9 } } + h200: { tp: 8, mem: 0.9 }, + b200: { tp: 8, mem: 0.9 }, + b300: { tp: 4, mem: 0.9 }, + gb300: { tp: 4, mem: 0.9 } }; const resolveItems = (option, values) => { @@ -96,34 +81,17 @@ export const Hunyuan3PreviewDeployment = () => { return () => observer.disconnect(); }, []); - useEffect(() => { - setValues(prev => { - const next = { ...prev }; - for (const [key, option] of Object.entries(options)) { - if (typeof option.getDynamicItems !== 'function') continue; - const items = option.getDynamicItems(next); - const current = items.find(i => i.id === next[key]); - if (!current || current.disabled) { - const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled); - if (fallback) next[key] = fallback.id; - } - } - return next; - }); - }, [values.hardware]); - const handleRadioChange = (optionName, value) => { setValues(prev => ({ ...prev, [optionName]: value })); }; const generateCommand = () => { - const { hardware, quantization } = values; + const { hardware } = values; const isBlackwell = hardware === 'b200' || hardware === 'b300' || hardware === 'gb300'; - const hwConfig = modelConfigs[hardware] && modelConfigs[hardware][quantization]; - if (!hwConfig) return '# Configuration not available for the selected hardware and quantization.'; + const hwConfig = modelConfigs[hardware]; + if (!hwConfig) return '# Configuration not available for the selected hardware.'; - const suffix = quantization === 'fp8' ? '-FP8' : ''; - const modelName = `tencent/Hy3-preview${suffix}`; + const modelName = 'tencent/Hy3-preview'; const tpValue = hwConfig.tp; const memFraction = hwConfig.mem; const enableSpec = values.speculative === 'enabled';