[Test] Add chunk e2e test case for CI (#956)

Signed-off-by: wangyu31577 <wangyu31577@hundsun.com>
Co-authored-by: wangyu31577 <wangyu31577@hundsun.com>
This commit is contained in:
wangyuandwangyu31577 authored and GitHub committed 2026-01-27 20:15:56 +08:00
1 parent 46e96e18e3
commit 2cc4375df8
7 files changed
+414 -89

No files matched your search

+2 -2
View File
@@ -10,7 +10,7 @@ import pytest
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm.sampling_params import SamplingParams
from tests.conftest import clean_gpu_memory
from tests.conftest import _run_post_test_cleanup
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.outputs import OmniRequestOutput
@@ -336,7 +336,7 @@ class OmniRunner:
self.close()
del self.omni
cleanup_dist_env_and_memory()
clean_gpu_memory()
_run_post_test_cleanup()
def close(self):
"""Close and cleanup the Omni instance."""
@@ -7,6 +7,8 @@ E2E offline tests for Omni model with video input and audio output.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
+57 -17
View File
@@ -7,8 +7,10 @@ E2E Online tests for Qwen3-Omni model with video input and audio output.
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import concurrent.futures
import threading
import time
from pathlib import Path
@@ -23,36 +25,67 @@ from tests.conftest import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from vllm_omni.utils import is_rocm
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
def get_default_config():
return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
def get_chunk_config():
path = modify_stage_config(
get_default_config(),
updates={
"async_chunk": True,
"stage_args": {
0: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
},
1: {
"engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
},
},
},
deletes={"stage_args": {2: ["custom_process_input_func"]}},
)
return path
CHUNK_CONFIG_PATH = get_chunk_config()
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
else:
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")]
stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
_omni_server_lock = threading.Lock()
@pytest.fixture(scope="module")
def omni_server(request):
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Multi-stage initialization can take 10-20+ minutes.
"""
model, stage_config_path = request.param
with _omni_server_lock:
model, stage_config_path = request.param
print(f"Starting OmniServer with model: {model}")
print("This may take 10-20+ minutes for initialization...")
print(f"Starting OmniServer with model: {model}")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
print("OmniServer started successfully")
yield server
print("OmniServer stopping...")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
print("OmniServer started successfully")
yield server
print("OmniServer stopped")
@@ -113,7 +146,7 @@ def get_max_batch_size(size_type="few"):
@pytest.mark.skipif(is_rocm(), reason="Test skipped on AMD environment due to known output issues")
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
"""
Test multi-modal input processing and text/audio output generation via OpenAI API.
Deploy Setting: default yaml
@@ -122,6 +155,11 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
Input Setting: stream=True
Datasets: single request
"""
# TODO:This skip will be removed when the chunk scenario supports multimodal input.
param = request.node.callspec.params.get("omni_server")
if param[1] == CHUNK_CONFIG_PATH:
pytest.skip("The current chunk scenario does not support multimodal.")
# Test single completion
e2e_list = list()
@@ -223,16 +261,18 @@ def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded."
for chat_completion in chat_completions:
# Verify audio output success
audio_message = chat_completion.choices[1].message
audio_data = audio_message.audio.data
assert audio_data is not None, "No audio output is generated"
assert audio_message.audio.expires_at > time.time(), "The generated audio has expired."
audio_data = None
text_content = None
for choice in chat_completion.choices:
if choice.message.audio is not None:
audio_message = choice.message
audio_data = audio_message.audio.data
assert audio_message.audio.expires_at > time.time(), "The generated audio has expired."
# Verify text output success
text_choice = chat_completion.choices[0]
text_content = text_choice.message.content
assert text_choice.message.content is not None, "No text output is generated"
assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords."
if choice.message.content is not None:
# Verify text output success
text_content = choice.message.content
assert "beijing" in text_content.lower(), "The output do not contain keywords."
# Verify text output same as audio output
audio_content = convert_audio_to_text(audio_data)
@@ -73,12 +73,14 @@ def get_max_batch_size(size_type="few"):
def get_deploy_config(deploy_type="TP1"):
result = {
"TP1": {
0: {
"engine_args.gpu_memory_utilization": 0.95,
"engine_args.tensor_parallel_size": 1,
"runtime.devices": "0",
},
2: {"runtime.devices": "1"},
"stage_args": {
0: {
"engine_args.gpu_memory_utilization": 0.95,
"engine_args.tensor_parallel_size": 1,
"runtime.devices": "0",
},
2: {"runtime.devices": "1"},
}
}
}
return result.get(deploy_type, result["TP1"])
@@ -151,8 +153,10 @@ def test_audio_to_text_audio_001(test_config: tuple[str, str]) -> None:
stage_config_path = modify_stage_config(
stage_config_path,
{
0: {"runtime.max_batch_size": num_concurrent_requests},
1: {"runtime.max_batch_size": num_concurrent_requests},
"stage_args": {
0: {"runtime.max_batch_size": num_concurrent_requests},
1: {"runtime.max_batch_size": num_concurrent_requests},
}
},
)
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
@@ -246,8 +250,10 @@ def test_image_to_text_audio_001(test_config: tuple[str, str]) -> None:
stage_config_path = modify_stage_config(
stage_config_path,
{
0: {"runtime.max_batch_size": num_concurrent_requests},
1: {"runtime.max_batch_size": num_concurrent_requests},
"stage_args": {
0: {"runtime.max_batch_size": num_concurrent_requests},
1: {"runtime.max_batch_size": num_concurrent_requests},
}
},
)
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server:
+1 -1
View File
@@ -79,7 +79,7 @@ stage_args:
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
max_num_batched_tokens: 100000
hf_config_name: thinker_config
async_scheduling: false
engine_input_source: [1]