Skip to content

[CI] Update perf json baselines #4289

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hsliuustc0106 merged 28 commits into from
Jun 17, 2026
+278 −236
Merged

[CI] Update perf json baselines #4289

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
5eafd32
Update baseline
congw729 Jun 9, 2026
9926116
update baseline (#8)
amy-why-3459 Jun 11, 2026
683da45
Updated Qwen-image and Qwen-image-layered baseline
herotai214 Jun 11, 2026
8ca992f
Update Wan2.2 perf baselines
bjf-frz Jun 11, 2026
2edd501
hunyuan baseline
Bounty-hunter Jun 11, 2026
b539266
adjust qwen image edit perf baseline
fhfuih Jun 11, 2026
e3f9e15
voxcpm2 baseline update
Shirley125 Jun 11, 2026
ec46d87
update tts baseline (#13)
R2-Y Jun 11, 2026
486a901
Update bagel baseline
congw729 Jun 12, 2026
8e7ecab
Merge branch 'main' into monthly-update-baseline
congw729 Jun 12, 2026
0b67de6
Merge branch 'main' into monthly-update-baseline
congw729 Jun 15, 2026
31ffb2f
Merge branch 'main' into monthly-update-baseline
congw729 Jun 15, 2026
4a97437
Merge branch 'main' into monthly-update-baseline
congw729 Jun 16, 2026
514da05
Merge branch 'main' into monthly-update-baseline
congw729 Jun 16, 2026
cdfdacc
Update qwen image edit
congw729 Jun 16, 2026
dbacac7
Revert bagel baseline due to regression. Remove hunyuan memory updation.
congw729 Jun 16, 2026
71cf59d
Update qwen image 2511 baseline
congw729 Jun 16, 2026
37502ca
Hunyuan metirc round to 4 decimals.
congw729 Jun 16, 2026
7a0a6de
Update qwen 3 tts baseline
congw729 Jun 16, 2026
53b750a
Update Qwen Image and Qwen Image Layered perf baselines
NumberWan Jun 16, 2026
7fda3e8
Update voxcpm2 baseline
congw729 Jun 16, 2026
9bda2d8
Merge remote-tracking branch 'origin/monthly-update-baseline' into mo…
congw729 Jun 16, 2026
6d27b9b
Fix Wan2.2 baseline update
bjf-frz Jun 16, 2026
39a587a
Merge branch 'main' into monthly-update-baseline
congw729 Jun 17, 2026
8b2b2b7
Update wan2.2
congw729 Jun 17, 2026
81760de
Merge remote-tracking branch 'refs/remotes/origin/monthly-update-base…
congw729 Jun 17, 2026
8352a34
Update qwen3omni and qwen3tts
congw729 Jun 17, 2026
80ba4c1
Merge branch 'main' into monthly-update-baseline
congw729 Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions tests/dfx/perf/tests/test_hunyuan_image3_it2i.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@
"tensor_parallel_size": 2,
"hf_overrides": {
"rope_parameters": {
"mrope_section": [0, 32, 32],
"mrope_section": [
0,
32,
32
],
"rope_type": "default"
}
},
Expand All @@ -58,7 +62,9 @@
"max_tokens": 8192,
"detokenize": true,
"skip_special_tokens": false,
"stop_token_ids": [128025]
"stop_token_ids": [
128025
]
}
},
{
Expand Down Expand Up @@ -106,7 +112,9 @@
"content": [
{
"prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。",
"image_urls": ["https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"],
"image_urls": [
"https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"
],
"num_inference_steps": 8,
"seed": 42,
"guidance_scale": 1.0
Expand All @@ -119,10 +127,9 @@
"num-prompts": 96,
"skip-performance-assertion": true,
"baseline": {
"throughput_qps": 0.1,
"latency_p99": 44.46,
"latency_mean": 35.57,
"latency_std_dev": 0.5,
"throughput_qps": 0.1874,
"latency_p99": 101.852,
"latency_mean": 79.0385,
"peak_memory_mb_max": 70000,
"peak_memory_mb_mean": 70000
}
Expand Down
9 changes: 5 additions & 4 deletions tests/dfx/perf/tests/test_hunyuan_image_tp4.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,11 @@
"max-concurrency": 1,
"skip-performance-assertion": true,
"baseline": {
"throughput_qps": 0.21,
"latency_p99": 4.9235,
"peak_memory_mb_max": 57576,
"peak_memory_mb_mean": 57576
"throughput_qps": 0.213,
"latency_mean": 4.6953,
"latency_p99": 4.8601,
"peak_memory_mb_max": 56912.0,
"peak_memory_mb_mean": 56912.0
}
}
]
Expand Down
78 changes: 39 additions & 39 deletions tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,25 +32,25 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
135.4379,
187.0896,
400,
300,
400,
600,
900
435.675,
724.6836
],
"mean_e2el_ms": [
26000,
43000,
52000,
72000,
109000
17619.9851,
34811.8335,
43656.9501,
56657.1548,
85097.6016
],
"mean_audio_rtf": [
0.15,
0.30,
0.35,
0.50,
0.75
0.1279,
0.2238,
0.2735,
0.3712,
0.5634
]
}
},
Expand Down Expand Up @@ -79,13 +79,13 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
600
340.329
],
"mean_e2el_ms": [
6000
4305.477
],
"mean_audio_rtf": [
0.20
0.1305
]
}
},
Expand Down Expand Up @@ -116,13 +116,13 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
300
220.0909
],
"mean_e2el_ms": [
7000
5475.9565
],
"mean_audio_rtf": [
0.25
0.1818
]
}
},
Expand Down Expand Up @@ -155,13 +155,13 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
700
450.0155
],
"mean_e2el_ms": [
11000
8516.8316
],
"mean_audio_rtf": [
0.35
0.2717
]
}
},
Expand Down Expand Up @@ -194,18 +194,18 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
100,
300,
500,
800,
1500
78.4278,
195.8037,
333.7318,
561.805,
1038.043
],
"mean_e2el_ms": [
5000,
8000,
10000,
14000,
20000
4288.3029,
7429.9429,
9451.6184,
12607.5292,
18153.2547
]
}
},
Expand Down Expand Up @@ -239,10 +239,10 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
500
275.3875
],
"mean_e2el_ms": [
1000
730.6995
]
}
},
Expand Down Expand Up @@ -278,10 +278,10 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
200
166.2201
],
"mean_e2el_ms": [
800
673.5789
]
}
},
Expand Down Expand Up @@ -319,10 +319,10 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
600
357.7781
],
"mean_e2el_ms": [
1300
953.3493
]
}
}
Expand Down
48 changes: 24 additions & 24 deletions tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,25 +32,25 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
400,
300,
400,
600,
900
109.1218,
247.4411,
354.9384,
447.7885,
694.7487
],
"mean_e2el_ms": [
32000,
34000,
45000,
61000,
90000
22413.667,
30808.9715,
35677.844,
46345.8978,
68350.4853
],
"mean_audio_rtf": [
0.20,
0.25,
0.30,
0.45,
0.60
0.1695,
0.2033,
0.2467,
0.3234,
0.4438
]
}
},
Expand Down Expand Up @@ -79,13 +79,13 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
500
323.3573
],
"mean_e2el_ms": [
6000
4623.4173
],
"mean_audio_rtf": [
0.20
0.1406
]
}
},
Expand Down Expand Up @@ -116,13 +116,13 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
300
215.4502
],
"mean_e2el_ms": [
7000
4924.149
],
"mean_audio_rtf": [
0.20
0.1637
]
}
},
Expand Down Expand Up @@ -155,13 +155,13 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
700
442.5815
],
"mean_e2el_ms": [
9000
6441.0986
],
"mean_audio_rtf": [
0.30
0.2091
]
}
}
Expand Down
32 changes: 16 additions & 16 deletions tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,18 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
300,
300,
1000,
600,
800
69.1276,
255.1882,
310.639,
422.0337,
532.0582
],
"mean_e2el_ms": [
6000,
8000,
11000,
13000,
17000
4382.7211,
7510.3815,
9440.4854,
12553.0186,
16932.6065
]
}
},
Expand Down Expand Up @@ -86,10 +86,10 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
30
26.3381
],
"mean_e2el_ms": [
540
485.2436
]
}
},
Expand Down Expand Up @@ -126,10 +126,10 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
30
26.4517
],
"mean_e2el_ms": [
600
517.2017
]
}
},
Expand Down Expand Up @@ -168,10 +168,10 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
40
27.6972
],
"mean_e2el_ms": [
700
561.3684
]
}
}
Expand Down
Loading
Loading