[Test] L2 & L3 Test Case Stratification Design for Omni Model (#1272)

Signed-off-by: wangyu31577 <wangyu31577@hundsun.com>
Signed-off-by: yenuo26 <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: wangyu31577 <wangyu31577@hundsun.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-02-12 09:56:35 +08:00
1 parent 82fd2c3637
commit 20a8fe1dcc
20 files changed
+1193 -1183

No files matched your search

@@ -1,103 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type to launch OmniLLM
runtime:
devices: "0"
max_batch_size: 5
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
max_num_batched_tokens: 32768
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
stage_type: llm # Use llm stage type to launch OmniLLM
runtime:
devices: "1"
max_batch_size: 5
engine_args:
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 100
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
stage_type: llm # Use llm stage type to launch OmniLLM
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
load_format: dummy
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 200
seed: 42
detokenize: True
repetition_penalty: 1.1
@@ -1,31 +0,0 @@
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0,1"
max_batch_size: 5
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 2
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
@@ -1,95 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
max_batch_size: 5
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 1
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
max_batch_size: 5
engine_args:
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 1000
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
async_scheduling: false
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2000
seed: 42
detokenize: True
repetition_penalty: 1.1
-117
View File
@@ -1,117 +0,0 @@
import asyncio
import os
import sys
from contextlib import ExitStack
from pathlib import Path
import pytest
from vllm import SamplingParams
from vllm.inputs import PromptType
from tests.utils import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
SEED = 42
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen3_omni_thinker_ci.yaml")
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
async def generate(
engine: AsyncOmni,
request_id: str,
prompt: PromptType,
max_tokens: int,
) -> tuple[int, str]:
# Ensure generate doesn't complete too fast for cancellation test.
await asyncio.sleep(0.2)
thinker_sampling_params = SamplingParams(
temperature=0.4, # Deterministic
top_p=0.9,
top_k=1,
max_tokens=max_tokens,
repetition_penalty=1.05,
stop_token_ids=[151645], # Qwen EOS token <|im_end|>
seed=SEED,
)
sampling_params_list = [
thinker_sampling_params,
]
count = 0
async for omni_output in engine.generate(
prompt=prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
output_modalities=["text"],
):
stage_id = omni_output.stage_id
out = omni_output.request_output
if stage_id == 0:
num_tokens = sum(len(output.token_ids) for output in out.outputs)
count = num_tokens
await asyncio.sleep(0.0)
return count, request_id
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.asyncio
async def test_abort():
with ExitStack() as after:
# Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS.
engine = AsyncOmni(
model=model,
stage_configs_path=stage_config,
shm_threshold_bytes=sys.maxsize,
)
after.callback(engine.shutdown)
# Keep token counts modest to reduce flakiness on slow test hardware.
NUM_REQUESTS = 3
NUM_EXPECTED_TOKENS = 64
NUM_EXPECTED_TOKENS_LONG = 256
REQUEST_IDS_TO_ABORT = [1]
prompt = "Hello my name is Robert and "
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
# Create concurrent requests.
tasks: list[asyncio.Task] = []
for idx, request_id in enumerate(request_ids):
max_tokens = NUM_EXPECTED_TOKENS_LONG if (idx in REQUEST_IDS_TO_ABORT) else NUM_EXPECTED_TOKENS
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, max_tokens)))
# API server cancels requests when they disconnect.
# Explicitly abort in the engine to avoid orphaned requests hanging.
for idx in REQUEST_IDS_TO_ABORT:
tasks[idx].cancel()
await engine.abort(request_ids[idx])
await asyncio.sleep(0.1)
# Confirm the other requests are okay.
for idx, task in enumerate(tasks):
# Confirm that it was actually canceled.
if idx in REQUEST_IDS_TO_ABORT:
with pytest.raises((asyncio.CancelledError, GeneratorExit)):
await asyncio.wait_for(task, timeout=60)
else:
# Otherwise, make sure the request was not impacted.
num_generated_tokens, request_id = await asyncio.wait_for(task, timeout=180)
expected_tokens = NUM_EXPECTED_TOKENS
assert num_generated_tokens == expected_tokens, (
f"{request_id} generated {num_generated_tokens} but expected {expected_tokens}"
)
# Confirm we can do another generation.
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
task = asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS))
num_generated_tokens, request_id = await task
assert num_generated_tokens == NUM_EXPECTED_TOKENS
await asyncio.sleep(5)
+26 -158
View File
@@ -1,5 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E Online tests for Qwen3-Omni model with video input and audio output.
"""
@@ -9,23 +7,15 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import concurrent.futures
import threading
import time
from pathlib import Path
import openai
import pytest
from tests.conftest import (
OmniServer,
convert_audio_to_text,
cosine_similarity_text,
dummy_messages_from_mix_data,
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
merge_base64_and_convert_to_text,
modify_stage_config,
)
from tests.utils import hardware_test
@@ -34,13 +24,9 @@ from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
def get_default_config():
return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
def get_chunk_config():
path = modify_stage_config(
get_default_config(),
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
updates={
"async_chunk": True,
"stage_args": {
@@ -57,49 +43,17 @@ def get_chunk_config():
return path
CHUNK_CONFIG_PATH = get_chunk_config()
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
else:
stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
stage_configs = [get_chunk_config()]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
_omni_server_lock = threading.Lock()
@pytest.fixture(scope="module")
def omni_server(request):
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
model, stage_config_path = request.param
print(f"Starting OmniServer with model: {model}")
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
print("OmniServer started successfully")
yield server
print("OmniServer stopping...")
print("OmniServer stopped")
@pytest.fixture
def client(omni_server):
"""OpenAI client for the running vLLM-Omni server."""
return openai.OpenAI(
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
api_key="EMPTY",
)
def get_system_prompt():
return {
"role": "system",
@@ -116,23 +70,6 @@ def get_system_prompt():
}
def dummy_messages_from_video_data(
video_data_url: str,
content_text: str = "Describe the video briefly.",
):
"""Create messages with video data URL for OpenAI API."""
return [
get_system_prompt(),
{
"role": "user",
"content": [
{"type": "video_url", "video_url": {"url": video_data_url}},
{"type": "text", "text": content_text},
],
},
]
def get_prompt(prompt_type="text_only"):
prompts = {
"text_only": "What is the capital of China? Answer in 20 words.",
@@ -146,11 +83,12 @@ def get_max_batch_size(size_type="few"):
return batch_sizes.get(size_type, 5)
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"""
Test multi-modal input processing and text/audio output generation via OpenAI API.
Deploy Setting: default yaml
@@ -160,8 +98,6 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
Datasets: single request
"""
# Test single completion
e2e_list = list()
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
@@ -173,57 +109,25 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
content_text=get_prompt("mix"),
)
request_config = {
"model": omni_server.model,
"messages": messages,
"stream": True,
"key_words": {
"audio": ["water", "chirping"],
"image": ["square", "quadrate"],
},
}
# Test single completion
start_time = time.perf_counter()
chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True)
text_content = ""
audio_data = []
for chunk in chat_completion:
for choice in chunk.choices:
if hasattr(choice, "delta"):
content = getattr(choice.delta, "content", None)
else:
content = None
modality = getattr(chunk, "modality", None)
if modality == "audio" and content:
audio_data.append(content)
elif modality == "text" and content:
# Text chunk - accumulate text content
text_content += content if content else ""
# Verify E2E
current_e2e = time.perf_counter() - start_time
print(f"the request e2e is: {current_e2e}")
# TODO: Verify the E2E latency after confirmation baseline.
e2e_list.append(current_e2e)
print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
# Verify all completions succeeded
assert audio_data is not None, "No audio output is generated"
# Verify text output success
assert text_content is not None and len(text_content) >= 2, "No text output is generated"
assert any(
keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"]
), "The output does not contain any of the keywords."
# Verify text output same as audio output
audio_content = merge_base64_and_convert_to_text(audio_data)
print(f"text content is: {text_content}")
print(f"audio content is: {audio_content}")
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
print(f"similarity is: {similarity}")
assert similarity > 0.9, "The audio content is not same as the text"
openai_client.send_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
def test_text_to_text_001(omni_server, openai_client) -> None:
"""
Test text input processing and text/audio output generation via OpenAI API.
Deploy Setting: default yaml
@@ -231,50 +135,14 @@ def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
Output Modal: text + audio
Datasets: few requests
"""
num_concurrent_requests = get_max_batch_size()
messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt())
e2e_list = list()
with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
# Submit multiple completion requests concurrently
futures = [
executor.submit(client.chat.completions.create, model=omni_server.model, messages=messages)
for _ in range(num_concurrent_requests)
]
start_time = time.perf_counter()
# Wait for all requests to complete and collect results
chat_completions = list()
for future in concurrent.futures.as_completed(futures):
chat_completions.append(future.result())
# Verify E2E
current_e2e = time.perf_counter() - start_time
print(f"the request e2e is: {current_e2e}")
# TODO: Verify the E2E latency after confirmation baseline.
e2e_list.append(current_e2e)
request_config = {
"model": omni_server.model,
"messages": messages,
"stream": False,
"modalities": ["text"],
"key_words": {"text": ["beijing"]},
}
print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
# Verify all completions succeeded
assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded."
for chat_completion in chat_completions:
# Verify audio output success
audio_data = None
text_content = None
for choice in chat_completion.choices:
if choice.message.audio is not None:
audio_message = choice.message
audio_data = audio_message.audio.data
assert audio_message.audio.expires_at > time.time(), "The generated audio has expired."
if choice.message.content is not None:
# Verify text output success
text_content = choice.message.content
assert "beijing" in text_content.lower(), "The output do not contain keywords."
# Verify text output same as audio output
audio_content = convert_audio_to_text(audio_data)
print(f"text content is: {text_content}")
print(f"audio content is: {audio_content}")
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
print(f"similarity is: {similarity}")
assert similarity > 0.9, "The audio content is not same as the text"
openai_client.send_request(request_config, request_num=get_max_batch_size())