Skip to content

[Test] Reduce Perf test case and fix modify stage config #1449

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gaohan123 merged 9 commits into from
Feb 26, 2026
Merged

[Test] Reduce Perf test case and fix modify stage config #1449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b9cdfe7
add skip
yenuo26 Feb 24, 2026
ec5ef18
Merge branch 'vllm-project:main' into nightly
yenuo26 Feb 24, 2026
5875fb6
Remove commented-out H100 benchmark and diffusion image edit test ste…
yenuo26 Feb 24, 2026
8cad65f
Merge branch 'nightly' of https://github.com/yenuo26/vllm-omni into n…
yenuo26 Feb 24, 2026
4183763
debug for nightly
yenuo26 Feb 26, 2026
306ce49
Update CI Dockerfile to include 'jq' and remove skip markers from tes…
yenuo26 Feb 26, 2026
414040f
retry nightly ci
yenuo26 Feb 26, 2026
5a560b3
recover test-ready and reduce perf test
yenuo26 Feb 26, 2026
4e5bfcf
Merge branch 'main' into nightly
yenuo26 Feb 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .buildkite/test-nightly.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
steps:
- label: "Omni Model Test with H100"
timeout_in_minutes: 60
timeout_in_minutes: 90
depends_on: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
Expand Down Expand Up @@ -41,7 +41,6 @@ steps:
depends_on: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
- export VLLM_LOGGING_LEVEL=DEBUG
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v tests/examples/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
agents:
Expand All @@ -50,6 +49,7 @@ steps:
- docker#v5.2.0:
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
always-pull: true
shm-size: "8gb"
propagate-environment: true
environment:
- "HF_HOME=/fsx/hf_cache"
Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile.ci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ COPY . .

# Install system dependencies
RUN apt-get update && \
apt-get install -y ffmpeg sox libsox-fmt-all && \
apt-get install -y ffmpeg sox libsox-fmt-all jq && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

Expand Down
71 changes: 36 additions & 35 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -848,41 +848,42 @@ def delete_by_path(config_dict: dict, path: str) -> None:
del config[key]

# Apply updates
for key, value in updates.items():
if key == "stage_args":
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
if not stage_args:
raise ValueError("stage_args does not exist in config")

for stage_id, stage_updates in value.items():
# Find stage by ID
target_stage = None
for stage in stage_args:
if stage.get("stage_id") == int(stage_id):
target_stage = stage
break

if target_stage is None:
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")

# Apply updates to this stage
for path, val in stage_updates.items():
# Check if this is a simple key (not dot-separated)
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
if "." not in path:
# Direct key assignment (e.g., updating a list value)
target_stage[path] = val
else:
# Dot-separated path (e.g., nested dict access)
apply_update(target_stage, path, val)
elif "." in key:
# Apply using dot-separated path
apply_update(config, key, value)
else:
# Direct top-level key
config[key] = value
if updates:
for key, value in updates.items():
if key == "stage_args":
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
if not stage_args:
raise ValueError("stage_args does not exist in config")

for stage_id, stage_updates in value.items():
# Find stage by ID
target_stage = None
for stage in stage_args:
if stage.get("stage_id") == int(stage_id):
target_stage = stage
break

if target_stage is None:
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")

# Apply updates to this stage
for path, val in stage_updates.items():
# Check if this is a simple key (not dot-separated)
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
if "." not in path:
# Direct key assignment (e.g., updating a list value)
target_stage[path] = val
else:
# Dot-separated path (e.g., nested dict access)
apply_update(target_stage, path, val)
elif "." in key:
# Apply using dot-separated path
apply_update(config, key, value)
else:
# Direct top-level key
config[key] = value

# Save to new file with timestamp
timestamp = int(time.time())
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/online_serving/test_qwen3_omni.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"messages": messages,
"stream": True,
"key_words": {
"audio": ["water", "chirping", "crackling"],
"audio": ["water", "chirping", "crackling", "rain"],
"image": ["square", "quadrate"],
},
}
Expand Down
4 changes: 3 additions & 1 deletion tests/e2e/online_serving/test_qwen3_omni_expansion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]

AUDIO_KEY = ["water", "chirping", "crackling"]
AUDIO_KEY = ["water", "chirping", "crackling", "rain"]
IMAGE_KEY = ["square", "quadrate"]
VIDEO_KEY = ["sphere", "globe", "circle", "round", "ball"]

Expand Down Expand Up @@ -267,6 +267,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
openai_client.send_request(request_config)


@pytest.mark.skip(reason="There is a known issue with oom error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
Expand All @@ -287,6 +288,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
openai_client.send_request(request_config, request_num=get_max_batch_size())


@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
Expand Down
114 changes: 0 additions & 114 deletions tests/perf/tests/test.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,6 @@
"stage_config_name": "qwen3_omni.yaml"
},
"benchmark_params": [
{
"dataset_name": "random",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random",
"num_prompts": [
Expand Down Expand Up @@ -84,41 +62,6 @@
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random-mm",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(32, 32, 1)": 0.5,
"(0, 1, 1)": 0.1,
"(32, 32, 2)": 0.4
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
},
Expand Down Expand Up @@ -147,28 +90,6 @@
}
},
"benchmark_params": [
{
"dataset_name": "random",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random",
"num_prompts": [
Expand Down Expand Up @@ -225,41 +146,6 @@
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random-mm",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(32, 32, 1)": 0.5,
"(0, 1, 1)": 0.1,
"(32, 32, 2)": 0.4
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
}
Expand Down