[Test] L2 & L3 Test Case Stratification Design for Omni Model (#1272)

Signed-off-by: wangyu31577 <wangyu31577@hundsun.com>
Signed-off-by: yenuo26 <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: wangyu31577 <wangyu31577@hundsun.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-02-12 09:56:35 +08:00
1 parent 82fd2c3637
commit 20a8fe1dcc
20 files changed
+1193 -1183

No files matched your search

-353
View File
@@ -1,353 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Pytest configuration and fixtures for vllm-omni tests.
"""
from typing import Any
import pytest
from vllm import TextPrompt
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import _run_post_test_cleanup, _run_pre_test_cleanup
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
PromptImageInput = list[Any] | Any | None
PromptVideoInput = list[Any] | Any | None
class OmniRunner:
"""
Test runner for Omni models.
"""
def __init__(
self,
model_name: str,
seed: int = 42,
stage_init_timeout: int = 300,
batch_timeout: int = 10,
init_timeout: int = 300,
shm_threshold_bytes: int = 65536,
log_stats: bool = False,
stage_configs_path: str | None = None,
**kwargs,
) -> None:
"""
Initialize an OmniRunner for testing.
Args:
model_name: The model name or path
seed: Random seed for reproducibility
stage_init_timeout: Timeout for initializing a single stage in seconds
batch_timeout: Timeout for batching in seconds
init_timeout: Timeout for initializing stages in seconds
shm_threshold_bytes: Threshold for using shared memory
log_stats: Enable detailed statistics logging
stage_configs_path: Optional path to YAML stage config file
**kwargs: Additional arguments passed to Omni
"""
cleanup_dist_env_and_memory()
_run_pre_test_cleanup(enable_force=True)
_run_post_test_cleanup(enable_force=True)
self.model_name = model_name
self.seed = seed
self.omni = Omni(
model=model_name,
log_stats=log_stats,
stage_init_timeout=stage_init_timeout,
batch_timeout=batch_timeout,
init_timeout=init_timeout,
shm_threshold_bytes=shm_threshold_bytes,
stage_configs_path=stage_configs_path,
**kwargs,
)
def get_default_sampling_params_list(self) -> list[OmniSamplingParams]:
"""
Get a list of default sampling parameters for all stages.
Returns:
List of SamplingParams with default decoding for each stage
"""
return [st.default_sampling_params for st in self.omni.stage_list]
def get_omni_inputs(
self,
prompts: list[str] | str,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
images: PromptImageInput = None,
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
modalities: list[str] | None = None,
) -> list[TextPrompt]:
"""
Construct Omni input format from prompts and multimodal data.
Args:
prompts: Text prompt(s) - either a single string or list of strings
system_prompt: Optional system prompt (defaults to Qwen system prompt)
audios: Audio input(s) - tuple of (audio_array, sample_rate) or list of tuples
images: Image input(s) - PIL Image or list of PIL Images
videos: Video input(s) - numpy array or list of numpy arrays
mm_processor_kwargs: Optional processor kwargs (e.g., use_audio_in_video)
Returns:
List of prompt dictionaries suitable for Omni.generate()
"""
if system_prompt is None:
system_prompt = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
"Group, capable of perceiving auditory and visual inputs, as well as "
"generating text and speech."
)
video_padding_token = "<|VIDEO|>"
image_padding_token = "<|IMAGE|>"
audio_padding_token = "<|AUDIO|>"
if self.model_name == "Qwen/Qwen3-Omni-30B-A3B-Instruct":
video_padding_token = "<|video_pad|>"
image_padding_token = "<|image_pad|>"
audio_padding_token = "<|audio_pad|>"
if isinstance(prompts, str):
prompts = [prompts]
def _normalize_mm_input(mm_input, num_prompts):
if mm_input is None:
return [None] * num_prompts
if isinstance(mm_input, list):
if len(mm_input) != num_prompts:
raise ValueError(
f"Multimodal input list length ({len(mm_input)}) must match prompts length ({num_prompts})"
)
return mm_input
return [mm_input] * num_prompts
num_prompts = len(prompts)
audios_list = _normalize_mm_input(audios, num_prompts)
images_list = _normalize_mm_input(images, num_prompts)
videos_list = _normalize_mm_input(videos, num_prompts)
omni_inputs = []
for i, prompt_text in enumerate(prompts):
user_content = ""
multi_modal_data = {}
audio = audios_list[i]
if audio is not None:
if isinstance(audio, list):
for _ in audio:
user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
multi_modal_data["audio"] = audio
else:
user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
multi_modal_data["audio"] = audio
image = images_list[i]
if image is not None:
if isinstance(image, list):
for _ in image:
user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
multi_modal_data["image"] = image
else:
user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
multi_modal_data["image"] = image
video = videos_list[i]
if video is not None:
if isinstance(video, list):
for _ in video:
user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
multi_modal_data["video"] = video
else:
user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
multi_modal_data["video"] = video
user_content += prompt_text
full_prompt = (
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
f"<|im_start|>user\n{user_content}<|im_end|>\n"
f"<|im_start|>assistant\n"
)
input_dict: TextPrompt = {"prompt": full_prompt}
if multi_modal_data:
input_dict["multi_modal_data"] = multi_modal_data
if modalities:
input_dict["modalities"] = modalities
if mm_processor_kwargs:
input_dict["mm_processor_kwargs"] = mm_processor_kwargs
omni_inputs.append(input_dict)
return omni_inputs
def generate(
self,
prompts: list[TextPrompt],
sampling_params_list: list[OmniSamplingParams] | None = None,
) -> list[OmniRequestOutput]:
"""
Generate outputs for the given prompts.
Args:
prompts: List of prompt dictionaries with 'prompt' and optionally
'multi_modal_data' keys
sampling_params_list: List of sampling parameters for each stage.
If None, uses default parameters.
Returns:
List of OmniRequestOutput objects from stages with final_output=True
"""
if sampling_params_list is None:
sampling_params_list = self.get_default_sampling_params_list()
return self.omni.generate(prompts, sampling_params_list)
def generate_multimodal(
self,
prompts: list[str] | str,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
images: PromptImageInput = None,
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
modalities: list[str] | None = None,
) -> list[OmniRequestOutput]:
"""
Convenience method to generate with multimodal inputs.
Args:
prompts: Text prompt(s)
sampling_params_list: List of sampling parameters for each stage
system_prompt: Optional system prompt
audios: Audio input(s)
images: Image input(s)
videos: Video input(s)
mm_processor_kwargs: Optional processor kwargs
Returns:
List of OmniRequestOutput objects from stages with final_output=True
"""
omni_inputs = self.get_omni_inputs(
prompts=prompts,
system_prompt=system_prompt,
audios=audios,
images=images,
videos=videos,
mm_processor_kwargs=mm_processor_kwargs,
modalities=modalities,
)
return self.generate(omni_inputs, sampling_params_list)
def generate_audio(
self,
prompts: list[str] | str,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[OmniRequestOutput]:
"""
Convenience method to generate with multimodal inputs.
Args:
prompts: Text prompt(s)
sampling_params_list: List of sampling parameters for each stage
system_prompt: Optional system prompt
audios: Audio input(s)
mm_processor_kwargs: Optional processor kwargs
Returns:
List of OmniRequestOutput objects from stages with final_output=True
"""
omni_inputs = self.get_omni_inputs(
prompts=prompts,
system_prompt=system_prompt,
audios=audios,
mm_processor_kwargs=mm_processor_kwargs,
)
return self.generate(omni_inputs, sampling_params_list)
def generate_video(
self,
prompts: list[str] | str,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[OmniRequestOutput]:
"""
Convenience method to generate with multimodal inputs.
Args:
prompts: Text prompt(s)
sampling_params_list: List of sampling parameters for each stage
system_prompt: Optional system prompt
videos: Video input(s)
mm_processor_kwargs: Optional processor kwargs
Returns:
List of OmniRequestOutput objects from stages with final_output=True
"""
omni_inputs = self.get_omni_inputs(
prompts=prompts,
system_prompt=system_prompt,
videos=videos,
mm_processor_kwargs=mm_processor_kwargs,
)
return self.generate(omni_inputs, sampling_params_list)
def generate_image(
self,
prompts: list[str] | str,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
images: PromptImageInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
) -> list[OmniRequestOutput]:
"""
Convenience method to generate with multimodal inputs.
Args:
prompts: Text prompt(s)
sampling_params_list: List of sampling parameters for each stage
system_prompt: Optional system prompt
images: Image input(s)
mm_processor_kwargs: Optional processor kwargs
Returns:
List of OmniRequestOutput objects from stages with final_output=True
"""
omni_inputs = self.get_omni_inputs(
prompts=prompts,
system_prompt=system_prompt,
images=images,
mm_processor_kwargs=mm_processor_kwargs,
)
return self.generate(omni_inputs, sampling_params_list)
def __enter__(self):
"""Context manager entry."""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Context manager exit - cleanup resources."""
self.close()
del self.omni
cleanup_dist_env_and_memory()
_run_post_test_cleanup(enable_force=True)
def close(self):
"""Close and cleanup the Omni instance."""
if hasattr(self.omni, "close"):
self.omni.close()
@pytest.fixture(scope="session")
def omni_runner():
return OmniRunner
@@ -1,106 +0,0 @@
# stage config for running qwen2.5-omni with architecture of OmniLLM.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
max_batch_size: 1
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
max_num_batched_tokens: 4069
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
defaults:
window_size: -1 # Simplified: trigger downstream only after full upstream completion
max_inflight: 1 # Simplified: process serially within each stage
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
window_size: -1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
window_size: -1
@@ -1,99 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 100
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
load_format: dummy
async_scheduling: false
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 200
seed: 42
detokenize: True
repetition_penalty: 1.1
@@ -1,105 +0,0 @@
# stage config for running qwen2.5-omni with architecture of OmniLLM.
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
max_batch_size: 1
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
defaults:
window_size: -1 # Simplified: trigger downstream only after full upstream completion
max_inflight: 1 # Simplified: process serially within each stage
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
window_size: -1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
window_size: -1
@@ -1,99 +0,0 @@
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
# Stage 0: Thinker (multimodal understanding + text generation)
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
# The following config has been verified on 2x H100-80G GPUs.
stage_args:
- stage_id: 0
runtime:
devices: "0"
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent # Output hidden states for talker
distributed_executor_backend: "mp"
enable_prefix_caching: false
hf_config_name: thinker_config
tensor_parallel_size: 1
load_format: dummy
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
top_k: 1
max_tokens: 100
seed: 42
detokenize: True
repetition_penalty: 1.05
- stage_id: 1
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: talker
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.6
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # Output codec codes for code2wav
# tensor_parallel_size: 2
enable_prefix_caching: false
distributed_executor_backend: "mp"
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
# final_output: true
# final_output_type: text
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 100
seed: 42
detokenize: False
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime:
devices: "1"
max_batch_size: 1
engine_args:
model_stage: code2wav
model_arch: Qwen3OmniMoeForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio # Final output: audio waveform
gpu_memory_utilization: 0.1
distributed_executor_backend: "mp"
max_num_batched_tokens: 1000000
hf_config_name: thinker_config
load_format: dummy
async_scheduling: false
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 200
seed: 42
detokenize: True
repetition_penalty: 1.1
+59 -109
View File
@@ -1,138 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E tests for Qwen2.5-Omni model with mixed modality inputs and audio output.
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
"""
from pathlib import Path
import pytest
from vllm.assets.audio import AudioAsset
from vllm.assets.image import ImageAsset
from vllm.assets.video import VideoAsset
from vllm.envs import VLLM_USE_MODELSCOPE
from vllm.multimodal.image import convert_image_mode
from tests.utils import create_new_process_for_each_test, hardware_test
from tests.conftest import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
)
from tests.utils import hardware_test
from vllm_omni.platforms import current_omni_platform
from .conftest import OmniRunner
models = ["Qwen/Qwen2.5-Omni-3B"]
models = ["Qwen/Qwen2.5-Omni-7B"]
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
if current_omni_platform.is_npu():
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_config = str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
else:
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen2_5_omni_ci.yaml")
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml")
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models]
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
@create_new_process_for_each_test("spawn")
@pytest.mark.parametrize("test_config", test_params)
def test_mixed_modalities_to_audio(omni_runner: type[OmniRunner], test_config: tuple[str, str]) -> None:
"""Test processing audio, image, and video together, generating audio output."""
model, stage_config_path = test_config
with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
# Prepare multimodal inputs
question = "What is recited in the audio? What is in this image? Describe the video briefly."
audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
audio = (audio[0][: 16000 * 5], audio[1]) # Trim to first 5 seconds
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB")
if not VLLM_USE_MODELSCOPE:
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
else:
# modelscope can't access raushan-testing-hf/videos-test, skip video input temporarily
video = None
outputs = runner.generate_multimodal(
prompts=question,
audios=audio,
images=image,
videos=video,
)
# Find and verify text output (thinker stage)
text_output = None
output_count = 0
for stage_output in outputs:
if stage_output.final_output_type == "text":
text_output = stage_output
output_count += 1
break
assert output_count > 0
assert text_output is not None
assert len(text_output.request_output) > 0
text_content = text_output.request_output[0].outputs[0].text
assert text_content is not None
assert len(text_content.strip()) > 0
# Find and verify audio output (code2wav stage)
audio_output = None
output_count = 0
for stage_output in outputs:
if stage_output.final_output_type == "audio":
audio_output = stage_output
output_count += 1
break
assert output_count > 0
assert audio_output is not None
assert len(audio_output.request_output) > 0
# Verify audio tensor exists and has content
audio_tensor = audio_output.request_output[0].outputs[0].multimodal_output["audio"]
assert audio_tensor is not None
assert audio_tensor.numel() > 0
def get_question(prompt_type="mix"):
prompts = {
"mix": "What is recited in the audio? What is in this image? Describe the video briefly.",
"text_only": "What is the capital of China?",
}
return prompts.get(prompt_type, prompts["mix"])
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
@create_new_process_for_each_test("spawn")
@pytest.mark.parametrize("test_config", test_params)
def test_mixed_modalities_to_text_only(omni_runner: type[OmniRunner], test_config: tuple[str, str]) -> None:
"""Test processing audio, image, and video together, generating audio output."""
model, stage_config_path = test_config
with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
# Prepare multimodal inputs
question = "What is recited in the audio? What is in this image? Describe the video briefly."
audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
audio = (audio[0][: 16000 * 5], audio[1]) # Trim to first 5 seconds
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB")
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
modalities = ["text"]
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
"""
Test multi-modal input processing and text/audio output generation via OpenAI API.
Deploy Setting: default yaml
Input Modal: text + audio + video + image
Output Modal: audio
Input Setting: stream=False
Datasets: single request
"""
video = generate_synthetic_video(16, 16, 30)["np_array"]
image = generate_synthetic_image(16, 16)["np_array"]
audio = generate_synthetic_audio(1, 1, 16000)["np_array"]
if len(audio.shape) == 2:
audio = audio.squeeze()
outputs = runner.generate_multimodal(
prompts=question,
audios=audio,
images=image,
videos=video,
modalities=modalities,
)
request_config = {
"prompts": get_question(),
"videos": video,
"images": image,
"audios": (audio, 16000),
"modalities": ["audio"],
}
# Find and verify text output (thinker stage)
text_output = None
output_count = 0
for stage_output in outputs:
assert stage_output.final_output_type != "audio"
if stage_output.final_output_type == "text":
text_output = stage_output
output_count += 1
break
assert output_count > 0
# Test single completion
omni_runner_handler.send_request(request_config)
assert text_output is not None
assert len(text_output.request_output) > 0
text_content = text_output.request_output[0].outputs[0].text
assert text_content is not None
assert len(text_content.strip()) > 0
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
"""
Test text input processing and text output generation via OpenAI API.
Deploy Setting: default yaml
Input Modal: text
Output Modal: text
Input Setting: stream=False
Datasets: single request
"""
request_config = {"prompts": get_question("text_only"), "modalities": ["text"]}
# Test single completion
omni_runner_handler.send_request(request_config)
+18 -51
View File
@@ -1,5 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E offline tests for Omni model with video input and audio output.
"""
@@ -12,73 +10,42 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from vllm.assets.video import VideoAsset
from tests.conftest import (
generate_synthetic_video,
)
from tests.utils import hardware_test
from vllm_omni.platforms import current_omni_platform
from .conftest import OmniRunner
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
else:
stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")]
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
def get_question(prompt_type="video"):
prompts = {
"video": "Describe the video briefly.",
}
return prompts.get(prompt_type, prompts["video"])
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("test_config", test_params)
def test_video_to_audio(omni_runner: type[OmniRunner], test_config) -> None:
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
"""Test processing video, generating audio output."""
model, stage_config_path = test_config
with omni_runner(model, seed=42, stage_configs_path=stage_config_path, stage_init_timeout=300) as runner:
# Prepare inputs
question = "Describe the video briefly."
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
video = generate_synthetic_video(224, 224, 300)["np_array"]
outputs = runner.generate_multimodal(
prompts=question,
videos=video,
)
request_config = {"prompts": get_question(), "videos": video, "modalities": ["audio"]}
# Find and verify text output (thinker stage)
text_output = None
output_count = 0
for stage_output in outputs:
if stage_output.final_output_type == "text":
text_output = stage_output
output_count += 1
break
assert output_count > 0
assert text_output is not None
assert len(text_output.request_output) > 0
text_content = text_output.request_output[0].outputs[0].text
assert text_content is not None
assert len(text_content.strip()) > 0
# Find and verify audio output (code2wav stage)
audio_output = None
output_count = 0
for stage_output in outputs:
if stage_output.final_output_type == "audio":
audio_output = stage_output
output_count += 1
break
assert output_count > 0
assert audio_output is not None
assert len(audio_output.request_output) > 0
# Verify audio tensor exists and has content
audio_tensor = audio_output.request_output[0].outputs[0].multimodal_output["audio"]
assert audio_tensor is not None
assert audio_tensor.numel() > 0
# Test single completion
omni_runner_handler.send_request(request_config)