anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Test] Reduce Perf test case and fix modify stage config (#1449)

Browse Source 
Signed-off-by: yenuo26 <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-02-26 14:20:57 +08:00
1 parent 5e7974c15b
commit 1ae2e1e4c4
6 files changed
+40 -151
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/test-nightly.yml
+2 -2
View File
@@ -1,6 +1,6 @@
steps:
- label: "Omni Model Test with H100"
timeout_in_minutes: 60
timeout_in_minutes: 90
depends_on: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
@@ -41,7 +41,6 @@ steps:
depends_on: upload-nightly-pipeline
if: build.env("NIGHTLY") == "1"
commands:
- export VLLM_LOGGING_LEVEL=DEBUG
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v tests/examples/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
agents:
@@ -50,6 +49,7 @@ steps:
- docker#v5.2.0:
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
always-pull: true
shm-size: "8gb"
propagate-environment: true
environment:
- "HF_HOME=/fsx/hf_cache"
docker/Dockerfile.ci
+1 -1
View File
@@ -8,7 +8,7 @@ COPY . .
# Install system dependencies
RUN apt-get update && \
apt-get install -y ffmpeg sox libsox-fmt-all && \
apt-get install -y ffmpeg sox libsox-fmt-all jq && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
tests/conftest.py
+33 -32
View File
@@ -848,41 +848,42 @@ def modify_stage_config(
del config[key]
# Apply updates
for key, value in updates.items():
if key == "stage_args":
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
if not stage_args:
raise ValueError("stage_args does not exist in config")
if updates:
for key, value in updates.items():
if key == "stage_args":
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
if not stage_args:
raise ValueError("stage_args does not exist in config")
for stage_id, stage_updates in value.items():
# Find stage by ID
target_stage = None
for stage in stage_args:
if stage.get("stage_id") == int(stage_id):
target_stage = stage
break
for stage_id, stage_updates in value.items():
# Find stage by ID
target_stage = None
for stage in stage_args:
if stage.get("stage_id") == int(stage_id):
target_stage = stage
break
if target_stage is None:
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")
if target_stage is None:
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")
# Apply updates to this stage
for path, val in stage_updates.items():
# Check if this is a simple key (not dot-separated)
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
if "." not in path:
# Direct key assignment (e.g., updating a list value)
target_stage[path] = val
else:
# Dot-separated path (e.g., nested dict access)
apply_update(target_stage, path, val)
elif "." in key:
# Apply using dot-separated path
apply_update(config, key, value)
else:
# Direct top-level key
config[key] = value
# Apply updates to this stage
for path, val in stage_updates.items():
# Check if this is a simple key (not dot-separated)
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
if "." not in path:
# Direct key assignment (e.g., updating a list value)
target_stage[path] = val
else:
# Dot-separated path (e.g., nested dict access)
apply_update(target_stage, path, val)
elif "." in key:
# Apply using dot-separated path
apply_update(config, key, value)
else:
# Direct top-level key
config[key] = value
# Save to new file with timestamp
timestamp = int(time.time())
tests/e2e/online_serving/test_qwen3_omni.py
+1 -1
View File
@@ -114,7 +114,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"messages": messages,
"stream": True,
"key_words": {
"audio": ["water", "chirping", "crackling"],
"audio": ["water", "chirping", "crackling", "rain"],
"image": ["square", "quadrate"],
},
}
tests/e2e/online_serving/test_qwen3_omni_expansion.py
+3 -1
View File
@@ -22,7 +22,7 @@ from tests.utils import hardware_test
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
AUDIO_KEY = ["water", "chirping", "crackling"]
AUDIO_KEY = ["water", "chirping", "crackling", "rain"]
IMAGE_KEY = ["square", "quadrate"]
VIDEO_KEY = ["sphere", "globe", "circle", "round", "ball"]
@@ -267,6 +267,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
openai_client.send_request(request_config)
@pytest.mark.skip(reason="There is a known issue with oom error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@@ -287,6 +288,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
openai_client.send_request(request_config, request_num=get_max_batch_size())
@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
tests/perf/tests/test.json
-114
View File
@@ -6,28 +6,6 @@
"stage_config_name": "qwen3_omni.yaml"
},
"benchmark_params": [
{
"dataset_name": "random",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random",
"num_prompts": [
@@ -84,41 +62,6 @@
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random-mm",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(32, 32, 1)": 0.5,
"(0, 1, 1)": 0.1,
"(32, 32, 2)": 0.4
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
},
@@ -147,28 +90,6 @@
}
},
"benchmark_params": [
{
"dataset_name": "random",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random",
"num_prompts": [
@@ -225,41 +146,6 @@
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random-mm",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(32, 32, 1)": 0.5,
"(0, 1, 1)": 0.1,
"(32, 32, 2)": 0.4
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
}