[Test] L2 & L3 Test Case Stratification Design for Omni Model (#1272)
Signed-off-by: wangyu31577 <wangyu31577@hundsun.com> Signed-off-by: yenuo26 <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: wangyu31577 <wangyu31577@hundsun.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
20 files changed
+1193
-1183
No files matched your search
@@ -1,353 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Pytest configuration and fixtures for vllm-omni tests.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from vllm import TextPrompt
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.conftest import _run_post_test_cleanup, _run_pre_test_cleanup
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
|
||||
PromptImageInput = list[Any] | Any | None
|
||||
PromptVideoInput = list[Any] | Any | None
|
||||
|
||||
|
||||
class OmniRunner:
|
||||
"""
|
||||
Test runner for Omni models.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
seed: int = 42,
|
||||
stage_init_timeout: int = 300,
|
||||
batch_timeout: int = 10,
|
||||
init_timeout: int = 300,
|
||||
shm_threshold_bytes: int = 65536,
|
||||
log_stats: bool = False,
|
||||
stage_configs_path: str | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize an OmniRunner for testing.
|
||||
|
||||
Args:
|
||||
model_name: The model name or path
|
||||
seed: Random seed for reproducibility
|
||||
stage_init_timeout: Timeout for initializing a single stage in seconds
|
||||
batch_timeout: Timeout for batching in seconds
|
||||
init_timeout: Timeout for initializing stages in seconds
|
||||
shm_threshold_bytes: Threshold for using shared memory
|
||||
log_stats: Enable detailed statistics logging
|
||||
stage_configs_path: Optional path to YAML stage config file
|
||||
**kwargs: Additional arguments passed to Omni
|
||||
"""
|
||||
cleanup_dist_env_and_memory()
|
||||
_run_pre_test_cleanup(enable_force=True)
|
||||
_run_post_test_cleanup(enable_force=True)
|
||||
self.model_name = model_name
|
||||
self.seed = seed
|
||||
|
||||
self.omni = Omni(
|
||||
model=model_name,
|
||||
log_stats=log_stats,
|
||||
stage_init_timeout=stage_init_timeout,
|
||||
batch_timeout=batch_timeout,
|
||||
init_timeout=init_timeout,
|
||||
shm_threshold_bytes=shm_threshold_bytes,
|
||||
stage_configs_path=stage_configs_path,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_default_sampling_params_list(self) -> list[OmniSamplingParams]:
|
||||
"""
|
||||
Get a list of default sampling parameters for all stages.
|
||||
|
||||
Returns:
|
||||
List of SamplingParams with default decoding for each stage
|
||||
"""
|
||||
return [st.default_sampling_params for st in self.omni.stage_list]
|
||||
|
||||
def get_omni_inputs(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
images: PromptImageInput = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
modalities: list[str] | None = None,
|
||||
) -> list[TextPrompt]:
|
||||
"""
|
||||
Construct Omni input format from prompts and multimodal data.
|
||||
|
||||
Args:
|
||||
prompts: Text prompt(s) - either a single string or list of strings
|
||||
system_prompt: Optional system prompt (defaults to Qwen system prompt)
|
||||
audios: Audio input(s) - tuple of (audio_array, sample_rate) or list of tuples
|
||||
images: Image input(s) - PIL Image or list of PIL Images
|
||||
videos: Video input(s) - numpy array or list of numpy arrays
|
||||
mm_processor_kwargs: Optional processor kwargs (e.g., use_audio_in_video)
|
||||
|
||||
Returns:
|
||||
List of prompt dictionaries suitable for Omni.generate()
|
||||
"""
|
||||
if system_prompt is None:
|
||||
system_prompt = (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
|
||||
"Group, capable of perceiving auditory and visual inputs, as well as "
|
||||
"generating text and speech."
|
||||
)
|
||||
|
||||
video_padding_token = "<|VIDEO|>"
|
||||
image_padding_token = "<|IMAGE|>"
|
||||
audio_padding_token = "<|AUDIO|>"
|
||||
|
||||
if self.model_name == "Qwen/Qwen3-Omni-30B-A3B-Instruct":
|
||||
video_padding_token = "<|video_pad|>"
|
||||
image_padding_token = "<|image_pad|>"
|
||||
audio_padding_token = "<|audio_pad|>"
|
||||
|
||||
if isinstance(prompts, str):
|
||||
prompts = [prompts]
|
||||
|
||||
def _normalize_mm_input(mm_input, num_prompts):
|
||||
if mm_input is None:
|
||||
return [None] * num_prompts
|
||||
if isinstance(mm_input, list):
|
||||
if len(mm_input) != num_prompts:
|
||||
raise ValueError(
|
||||
f"Multimodal input list length ({len(mm_input)}) must match prompts length ({num_prompts})"
|
||||
)
|
||||
return mm_input
|
||||
return [mm_input] * num_prompts
|
||||
|
||||
num_prompts = len(prompts)
|
||||
audios_list = _normalize_mm_input(audios, num_prompts)
|
||||
images_list = _normalize_mm_input(images, num_prompts)
|
||||
videos_list = _normalize_mm_input(videos, num_prompts)
|
||||
|
||||
omni_inputs = []
|
||||
for i, prompt_text in enumerate(prompts):
|
||||
user_content = ""
|
||||
multi_modal_data = {}
|
||||
|
||||
audio = audios_list[i]
|
||||
if audio is not None:
|
||||
if isinstance(audio, list):
|
||||
for _ in audio:
|
||||
user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
|
||||
multi_modal_data["audio"] = audio
|
||||
else:
|
||||
user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
|
||||
multi_modal_data["audio"] = audio
|
||||
|
||||
image = images_list[i]
|
||||
if image is not None:
|
||||
if isinstance(image, list):
|
||||
for _ in image:
|
||||
user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["image"] = image
|
||||
else:
|
||||
user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["image"] = image
|
||||
|
||||
video = videos_list[i]
|
||||
if video is not None:
|
||||
if isinstance(video, list):
|
||||
for _ in video:
|
||||
user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["video"] = video
|
||||
else:
|
||||
user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["video"] = video
|
||||
|
||||
user_content += prompt_text
|
||||
|
||||
full_prompt = (
|
||||
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_content}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
input_dict: TextPrompt = {"prompt": full_prompt}
|
||||
if multi_modal_data:
|
||||
input_dict["multi_modal_data"] = multi_modal_data
|
||||
if modalities:
|
||||
input_dict["modalities"] = modalities
|
||||
if mm_processor_kwargs:
|
||||
input_dict["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
omni_inputs.append(input_dict)
|
||||
|
||||
return omni_inputs
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: list[TextPrompt],
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
) -> list[OmniRequestOutput]:
|
||||
"""
|
||||
Generate outputs for the given prompts.
|
||||
|
||||
Args:
|
||||
prompts: List of prompt dictionaries with 'prompt' and optionally
|
||||
'multi_modal_data' keys
|
||||
sampling_params_list: List of sampling parameters for each stage.
|
||||
If None, uses default parameters.
|
||||
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
if sampling_params_list is None:
|
||||
sampling_params_list = self.get_default_sampling_params_list()
|
||||
|
||||
return self.omni.generate(prompts, sampling_params_list)
|
||||
|
||||
def generate_multimodal(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
images: PromptImageInput = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
modalities: list[str] | None = None,
|
||||
) -> list[OmniRequestOutput]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
audios: Audio input(s)
|
||||
images: Image input(s)
|
||||
videos: Video input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
audios=audios,
|
||||
images=images,
|
||||
videos=videos,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
modalities=modalities,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def generate_audio(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[OmniRequestOutput]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
audios: Audio input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
audios=audios,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def generate_video(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[OmniRequestOutput]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
videos: Video input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
videos=videos,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def generate_image(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
images: PromptImageInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[OmniRequestOutput]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
images: Image input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
images=images,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry."""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Context manager exit - cleanup resources."""
|
||||
self.close()
|
||||
del self.omni
|
||||
cleanup_dist_env_and_memory()
|
||||
_run_post_test_cleanup(enable_force=True)
|
||||
|
||||
def close(self):
|
||||
"""Close and cleanup the Omni instance."""
|
||||
if hasattr(self.omni, "close"):
|
||||
self.omni.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def omni_runner():
|
||||
return OmniRunner
|
||||
@@ -1,106 +0,0 @@
|
||||
# stage config for running qwen2.5-omni with architecture of OmniLLM.
|
||||
|
||||
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
|
||||
# This config is optimized for CI e2e tests.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
max_num_batched_tokens: 4069
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1 # Simplified: trigger downstream only after full upstream completion
|
||||
max_inflight: 1 # Simplified: process serially within each stage
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
window_size: -1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
window_size: -1
|
||||
@@ -1,99 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
load_format: dummy
|
||||
async_scheduling: false
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 200
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,138 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E tests for Qwen2.5-Omni model with mixed modality inputs and audio output.
|
||||
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.envs import VLLM_USE_MODELSCOPE
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
from tests.utils import create_new_process_for_each_test, hardware_test
|
||||
from tests.conftest import (
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
generate_synthetic_video,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .conftest import OmniRunner
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-3B"]
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
|
||||
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
|
||||
if current_omni_platform.is_npu():
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
|
||||
elif current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
|
||||
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
|
||||
else:
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen2_5_omni_ci.yaml")
|
||||
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml")
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
@create_new_process_for_each_test("spawn")
|
||||
@pytest.mark.parametrize("test_config", test_params)
|
||||
def test_mixed_modalities_to_audio(omni_runner: type[OmniRunner], test_config: tuple[str, str]) -> None:
|
||||
"""Test processing audio, image, and video together, generating audio output."""
|
||||
model, stage_config_path = test_config
|
||||
with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
|
||||
# Prepare multimodal inputs
|
||||
question = "What is recited in the audio? What is in this image? Describe the video briefly."
|
||||
audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
audio = (audio[0][: 16000 * 5], audio[1]) # Trim to first 5 seconds
|
||||
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB")
|
||||
if not VLLM_USE_MODELSCOPE:
|
||||
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
|
||||
else:
|
||||
# modelscope can't access raushan-testing-hf/videos-test, skip video input temporarily
|
||||
video = None
|
||||
|
||||
outputs = runner.generate_multimodal(
|
||||
prompts=question,
|
||||
audios=audio,
|
||||
images=image,
|
||||
videos=video,
|
||||
)
|
||||
|
||||
# Find and verify text output (thinker stage)
|
||||
text_output = None
|
||||
output_count = 0
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "text":
|
||||
text_output = stage_output
|
||||
output_count += 1
|
||||
break
|
||||
assert output_count > 0
|
||||
|
||||
assert text_output is not None
|
||||
assert len(text_output.request_output) > 0
|
||||
text_content = text_output.request_output[0].outputs[0].text
|
||||
assert text_content is not None
|
||||
assert len(text_content.strip()) > 0
|
||||
|
||||
# Find and verify audio output (code2wav stage)
|
||||
audio_output = None
|
||||
output_count = 0
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "audio":
|
||||
audio_output = stage_output
|
||||
output_count += 1
|
||||
break
|
||||
assert output_count > 0
|
||||
|
||||
assert audio_output is not None
|
||||
assert len(audio_output.request_output) > 0
|
||||
|
||||
# Verify audio tensor exists and has content
|
||||
audio_tensor = audio_output.request_output[0].outputs[0].multimodal_output["audio"]
|
||||
assert audio_tensor is not None
|
||||
assert audio_tensor.numel() > 0
|
||||
def get_question(prompt_type="mix"):
|
||||
prompts = {
|
||||
"mix": "What is recited in the audio? What is in this image? Describe the video briefly.",
|
||||
"text_only": "What is the capital of China?",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["mix"])
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
@create_new_process_for_each_test("spawn")
|
||||
@pytest.mark.parametrize("test_config", test_params)
|
||||
def test_mixed_modalities_to_text_only(omni_runner: type[OmniRunner], test_config: tuple[str, str]) -> None:
|
||||
"""Test processing audio, image, and video together, generating audio output."""
|
||||
model, stage_config_path = test_config
|
||||
with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
|
||||
# Prepare multimodal inputs
|
||||
question = "What is recited in the audio? What is in this image? Describe the video briefly."
|
||||
audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
audio = (audio[0][: 16000 * 5], audio[1]) # Trim to first 5 seconds
|
||||
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB")
|
||||
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
|
||||
modalities = ["text"]
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test multi-modal input processing and text/audio output generation via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
Input Modal: text + audio + video + image
|
||||
Output Modal: audio
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
video = generate_synthetic_video(16, 16, 30)["np_array"]
|
||||
image = generate_synthetic_image(16, 16)["np_array"]
|
||||
audio = generate_synthetic_audio(1, 1, 16000)["np_array"]
|
||||
if len(audio.shape) == 2:
|
||||
audio = audio.squeeze()
|
||||
|
||||
outputs = runner.generate_multimodal(
|
||||
prompts=question,
|
||||
audios=audio,
|
||||
images=image,
|
||||
videos=video,
|
||||
modalities=modalities,
|
||||
)
|
||||
request_config = {
|
||||
"prompts": get_question(),
|
||||
"videos": video,
|
||||
"images": image,
|
||||
"audios": (audio, 16000),
|
||||
"modalities": ["audio"],
|
||||
}
|
||||
|
||||
# Find and verify text output (thinker stage)
|
||||
text_output = None
|
||||
output_count = 0
|
||||
for stage_output in outputs:
|
||||
assert stage_output.final_output_type != "audio"
|
||||
if stage_output.final_output_type == "text":
|
||||
text_output = stage_output
|
||||
output_count += 1
|
||||
break
|
||||
assert output_count > 0
|
||||
# Test single completion
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
assert text_output is not None
|
||||
assert len(text_output.request_output) > 0
|
||||
text_content = text_output.request_output[0].outputs[0].text
|
||||
assert text_content is not None
|
||||
assert len(text_content.strip()) > 0
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test text input processing and text output generation via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
Input Modal: text
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
|
||||
request_config = {"prompts": get_question("text_only"), "modalities": ["text"]}
|
||||
|
||||
# Test single completion
|
||||
omni_runner_handler.send_request(request_config)
|
||||
@@ -1,5 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E offline tests for Omni model with video input and audio output.
|
||||
"""
|
||||
@@ -12,73 +10,42 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.assets.video import VideoAsset
|
||||
|
||||
from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .conftest import OmniRunner
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
|
||||
if current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
else:
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")]
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
def get_question(prompt_type="video"):
|
||||
prompts = {
|
||||
"video": "Describe the video briefly.",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["video"])
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("test_config", test_params)
|
||||
def test_video_to_audio(omni_runner: type[OmniRunner], test_config) -> None:
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing video, generating audio output."""
|
||||
model, stage_config_path = test_config
|
||||
with omni_runner(model, seed=42, stage_configs_path=stage_config_path, stage_init_timeout=300) as runner:
|
||||
# Prepare inputs
|
||||
question = "Describe the video briefly."
|
||||
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
|
||||
video = generate_synthetic_video(224, 224, 300)["np_array"]
|
||||
|
||||
outputs = runner.generate_multimodal(
|
||||
prompts=question,
|
||||
videos=video,
|
||||
)
|
||||
request_config = {"prompts": get_question(), "videos": video, "modalities": ["audio"]}
|
||||
|
||||
# Find and verify text output (thinker stage)
|
||||
text_output = None
|
||||
output_count = 0
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "text":
|
||||
text_output = stage_output
|
||||
output_count += 1
|
||||
break
|
||||
|
||||
assert output_count > 0
|
||||
assert text_output is not None
|
||||
assert len(text_output.request_output) > 0
|
||||
text_content = text_output.request_output[0].outputs[0].text
|
||||
assert text_content is not None
|
||||
assert len(text_content.strip()) > 0
|
||||
|
||||
# Find and verify audio output (code2wav stage)
|
||||
audio_output = None
|
||||
output_count = 0
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "audio":
|
||||
audio_output = stage_output
|
||||
output_count += 1
|
||||
break
|
||||
|
||||
assert output_count > 0
|
||||
assert audio_output is not None
|
||||
assert len(audio_output.request_output) > 0
|
||||
|
||||
# Verify audio tensor exists and has content
|
||||
audio_tensor = audio_output.request_output[0].outputs[0].multimodal_output["audio"]
|
||||
assert audio_tensor is not None
|
||||
assert audio_tensor.numel() > 0
|
||||
# Test single completion
|
||||
omni_runner_handler.send_request(request_config)
|
||||
@@ -1,103 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 200
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,95 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 5
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 1000
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
async_scheduling: false
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 2000
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,117 +0,0 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import PromptType
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
SEED = 42
|
||||
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "qwen3_omni_thinker_ci.yaml")
|
||||
model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncOmni,
|
||||
request_id: str,
|
||||
prompt: PromptType,
|
||||
max_tokens: int,
|
||||
) -> tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.4, # Deterministic
|
||||
top_p=0.9,
|
||||
top_k=1,
|
||||
max_tokens=max_tokens,
|
||||
repetition_penalty=1.05,
|
||||
stop_token_ids=[151645], # Qwen EOS token <|im_end|>
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
]
|
||||
count = 0
|
||||
async for omni_output in engine.generate(
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params_list,
|
||||
output_modalities=["text"],
|
||||
):
|
||||
stage_id = omni_output.stage_id
|
||||
out = omni_output.request_output
|
||||
if stage_id == 0:
|
||||
num_tokens = sum(len(output.token_ids) for output in out.outputs)
|
||||
count = num_tokens
|
||||
|
||||
await asyncio.sleep(0.0)
|
||||
|
||||
return count, request_id
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort():
|
||||
with ExitStack() as after:
|
||||
# Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS.
|
||||
engine = AsyncOmni(
|
||||
model=model,
|
||||
stage_configs_path=stage_config,
|
||||
shm_threshold_bytes=sys.maxsize,
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
# Keep token counts modest to reduce flakiness on slow test hardware.
|
||||
NUM_REQUESTS = 3
|
||||
NUM_EXPECTED_TOKENS = 64
|
||||
NUM_EXPECTED_TOKENS_LONG = 256
|
||||
REQUEST_IDS_TO_ABORT = [1]
|
||||
|
||||
prompt = "Hello my name is Robert and "
|
||||
|
||||
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
|
||||
|
||||
# Create concurrent requests.
|
||||
tasks: list[asyncio.Task] = []
|
||||
for idx, request_id in enumerate(request_ids):
|
||||
max_tokens = NUM_EXPECTED_TOKENS_LONG if (idx in REQUEST_IDS_TO_ABORT) else NUM_EXPECTED_TOKENS
|
||||
tasks.append(asyncio.create_task(generate(engine, request_id, prompt, max_tokens)))
|
||||
|
||||
# API server cancels requests when they disconnect.
|
||||
# Explicitly abort in the engine to avoid orphaned requests hanging.
|
||||
for idx in REQUEST_IDS_TO_ABORT:
|
||||
tasks[idx].cancel()
|
||||
await engine.abort(request_ids[idx])
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Confirm the other requests are okay.
|
||||
for idx, task in enumerate(tasks):
|
||||
# Confirm that it was actually canceled.
|
||||
if idx in REQUEST_IDS_TO_ABORT:
|
||||
with pytest.raises((asyncio.CancelledError, GeneratorExit)):
|
||||
await asyncio.wait_for(task, timeout=60)
|
||||
else:
|
||||
# Otherwise, make sure the request was not impacted.
|
||||
num_generated_tokens, request_id = await asyncio.wait_for(task, timeout=180)
|
||||
expected_tokens = NUM_EXPECTED_TOKENS
|
||||
assert num_generated_tokens == expected_tokens, (
|
||||
f"{request_id} generated {num_generated_tokens} but expected {expected_tokens}"
|
||||
)
|
||||
|
||||
# Confirm we can do another generation.
|
||||
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
|
||||
task = asyncio.create_task(generate(engine, request_id, prompt, NUM_EXPECTED_TOKENS))
|
||||
num_generated_tokens, request_id = await task
|
||||
assert num_generated_tokens == NUM_EXPECTED_TOKENS
|
||||
await asyncio.sleep(5)
|
||||
@@ -1,5 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for Qwen3-Omni model with video input and audio output.
|
||||
"""
|
||||
@@ -9,23 +7,15 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import concurrent.futures
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServer,
|
||||
convert_audio_to_text,
|
||||
cosine_similarity_text,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
generate_synthetic_video,
|
||||
merge_base64_and_convert_to_text,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
@@ -34,13 +24,9 @@ from vllm_omni.platforms import current_omni_platform
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
|
||||
def get_default_config():
|
||||
return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
|
||||
|
||||
def get_chunk_config():
|
||||
path = modify_stage_config(
|
||||
get_default_config(),
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
|
||||
updates={
|
||||
"async_chunk": True,
|
||||
"stage_args": {
|
||||
@@ -57,49 +43,17 @@ def get_chunk_config():
|
||||
return path
|
||||
|
||||
|
||||
CHUNK_CONFIG_PATH = get_chunk_config()
|
||||
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
|
||||
if current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
else:
|
||||
stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
|
||||
stage_configs = [get_chunk_config()]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
_omni_server_lock = threading.Lock()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server(request):
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
||||
Uses session scope so the server starts only once for the entire test session.
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
model, stage_config_path = request.param
|
||||
|
||||
print(f"Starting OmniServer with model: {model}")
|
||||
|
||||
with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server:
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(omni_server):
|
||||
"""OpenAI client for the running vLLM-Omni server."""
|
||||
return openai.OpenAI(
|
||||
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
|
||||
api_key="EMPTY",
|
||||
)
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
return {
|
||||
"role": "system",
|
||||
@@ -116,23 +70,6 @@ def get_system_prompt():
|
||||
}
|
||||
|
||||
|
||||
def dummy_messages_from_video_data(
|
||||
video_data_url: str,
|
||||
content_text: str = "Describe the video briefly.",
|
||||
):
|
||||
"""Create messages with video data URL for OpenAI API."""
|
||||
return [
|
||||
get_system_prompt(),
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_data_url}},
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text_only"):
|
||||
prompts = {
|
||||
"text_only": "What is the capital of China? Answer in 20 words.",
|
||||
@@ -146,11 +83,12 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
|
||||
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test multi-modal input processing and text/audio output generation via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
@@ -160,8 +98,6 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
|
||||
Datasets: single request
|
||||
"""
|
||||
|
||||
# Test single completion
|
||||
e2e_list = list()
|
||||
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
|
||||
audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
|
||||
@@ -173,57 +109,25 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N
|
||||
content_text=get_prompt("mix"),
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {
|
||||
"audio": ["water", "chirping"],
|
||||
"image": ["square", "quadrate"],
|
||||
},
|
||||
}
|
||||
|
||||
# Test single completion
|
||||
start_time = time.perf_counter()
|
||||
chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True)
|
||||
|
||||
text_content = ""
|
||||
audio_data = []
|
||||
for chunk in chat_completion:
|
||||
for choice in chunk.choices:
|
||||
if hasattr(choice, "delta"):
|
||||
content = getattr(choice.delta, "content", None)
|
||||
else:
|
||||
content = None
|
||||
|
||||
modality = getattr(chunk, "modality", None)
|
||||
|
||||
if modality == "audio" and content:
|
||||
audio_data.append(content)
|
||||
elif modality == "text" and content:
|
||||
# Text chunk - accumulate text content
|
||||
text_content += content if content else ""
|
||||
|
||||
# Verify E2E
|
||||
current_e2e = time.perf_counter() - start_time
|
||||
print(f"the request e2e is: {current_e2e}")
|
||||
# TODO: Verify the E2E latency after confirmation baseline.
|
||||
e2e_list.append(current_e2e)
|
||||
|
||||
print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
|
||||
# Verify all completions succeeded
|
||||
assert audio_data is not None, "No audio output is generated"
|
||||
|
||||
# Verify text output success
|
||||
assert text_content is not None and len(text_content) >= 2, "No text output is generated"
|
||||
assert any(
|
||||
keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"]
|
||||
), "The output does not contain any of the keywords."
|
||||
|
||||
# Verify text output same as audio output
|
||||
audio_content = merge_base64_and_convert_to_text(audio_data)
|
||||
print(f"text content is: {text_content}")
|
||||
print(f"audio content is: {audio_content}")
|
||||
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
|
||||
print(f"similarity is: {similarity}")
|
||||
assert similarity > 0.9, "The audio content is not same as the text"
|
||||
openai_client.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
|
||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text input processing and text/audio output generation via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
@@ -231,50 +135,14 @@ def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None:
|
||||
Output Modal: text + audio
|
||||
Datasets: few requests
|
||||
"""
|
||||
|
||||
num_concurrent_requests = get_max_batch_size()
|
||||
messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt())
|
||||
|
||||
e2e_list = list()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
|
||||
# Submit multiple completion requests concurrently
|
||||
futures = [
|
||||
executor.submit(client.chat.completions.create, model=omni_server.model, messages=messages)
|
||||
for _ in range(num_concurrent_requests)
|
||||
]
|
||||
start_time = time.perf_counter()
|
||||
# Wait for all requests to complete and collect results
|
||||
chat_completions = list()
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
chat_completions.append(future.result())
|
||||
# Verify E2E
|
||||
current_e2e = time.perf_counter() - start_time
|
||||
print(f"the request e2e is: {current_e2e}")
|
||||
# TODO: Verify the E2E latency after confirmation baseline.
|
||||
e2e_list.append(current_e2e)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
|
||||
# Verify all completions succeeded
|
||||
assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded."
|
||||
for chat_completion in chat_completions:
|
||||
# Verify audio output success
|
||||
audio_data = None
|
||||
text_content = None
|
||||
for choice in chat_completion.choices:
|
||||
if choice.message.audio is not None:
|
||||
audio_message = choice.message
|
||||
audio_data = audio_message.audio.data
|
||||
assert audio_message.audio.expires_at > time.time(), "The generated audio has expired."
|
||||
|
||||
if choice.message.content is not None:
|
||||
# Verify text output success
|
||||
text_content = choice.message.content
|
||||
assert "beijing" in text_content.lower(), "The output do not contain keywords."
|
||||
|
||||
# Verify text output same as audio output
|
||||
audio_content = convert_audio_to_text(audio_data)
|
||||
print(f"text content is: {text_content}")
|
||||
print(f"audio content is: {audio_content}")
|
||||
similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
|
||||
print(f"similarity is: {similarity}")
|
||||
assert similarity > 0.9, "The audio content is not same as the text"
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -11,10 +11,10 @@ stage_args:
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 32768
|
||||
max_num_batched_tokens: 32768
|
||||
max_model_len: 2400
|
||||
max_num_batched_tokens: 2400
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
@@ -41,10 +41,10 @@ stage_args:
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 32768
|
||||
max_num_batched_tokens: 32768
|
||||
max_model_len: 2400
|
||||
max_num_batched_tokens: 2400
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
@@ -71,7 +71,7 @@ stage_args:
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
|
||||
@@ -24,6 +24,7 @@ stage_args:
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 1
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
@@ -55,6 +56,7 @@ stage_args:
|
||||
max_model_len: 32768
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
|
||||
File renamed without changes.
+4
-4
@@ -13,8 +13,8 @@ stage_args:
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_model_len: 2400
|
||||
max_num_batched_tokens: 2400
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
@@ -43,8 +43,8 @@ stage_args:
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_model_len: 2400
|
||||
max_num_batched_tokens: 2400
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
File renamed without changes.
Reference in new issue
Block a user