Merge branch 'main' into feat/moss-tts-nano

Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-04-21 17:50:21 -04:00
commit 3bd77d3f8f
611 files changed
+48775 -14783

No files matched your search

@@ -26,7 +26,7 @@ from contextlib import ExitStack
import pytest
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -19,7 +19,7 @@ import numpy as np
import pytest
from transformers import AutoTokenizer
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -10,7 +10,7 @@ import pytest
from tests.e2e.offline_inference.custom_pipeline.worker_extension import (
vLLMOmniColocateWorkerExtensionForTest,
)
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension
from vllm_omni.entrypoints.async_omni import AsyncOmni
@@ -64,9 +64,6 @@ stage_args:
# Top-level runtime config with Mooncake connector
runtime:
enabled: true
defaults:
window_size: -1
max_inflight: 1
connectors:
mooncake_connector:
name: MooncakeConnector
@@ -80,4 +77,3 @@ runtime:
edges:
- from: 0
to: 1
window_size: -1
@@ -62,10 +62,6 @@ stage_args:
# Runtime edges
runtime:
enabled: true
defaults:
window_size: -1
max_inflight: 1
# Distributed connectors configuration (optional)
# More connectors will be supported in the future.
connectors:
@@ -78,4 +74,3 @@ runtime:
edges:
- from: 0
to: 1
window_size: -1
@@ -1,103 +0,0 @@
# stage config for running qwen2.5-omni for multi-stage omni runtime.
# This config is optimized for CI e2e tests.
stage_args:
- stage_id: 0
runtime:
process: true # Run this stage in a separate process
devices: "0"
engine_args:
model_stage: thinker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
mm_processor_cache_gb: 0
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
runtime:
process: true
devices: "1"
engine_args:
model_stage: talker
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
max_model_len: 896
max_num_batched_tokens: 896
max_num_seqs: 1
gpu_memory_utilization: 0.8
skip_mm_profiling: true
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: latent
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime:
process: true
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
engine_args:
model_stage: code2wav
max_num_seqs: 1
model_arch: Qwen2_5OmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.15
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
engine_output_type: audio
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 128
seed: 42
detokenize: True
repetition_penalty: 1.1
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
defaults:
window_size: -1 # Simplified: trigger downstream only after full upstream completion
max_inflight: 1 # Simplified: process serially within each stage
edges:
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
to: 1
window_size: -1
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
to: 2
window_size: -1
@@ -22,9 +22,10 @@ import pytest
from PIL import Image
from vllm.assets.image import ImageAsset
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from vllm_omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
# Reference pixel data extracted from the known-good output image
@@ -32,30 +33,30 @@ from vllm_omni.platforms import current_omni_platform
# prompt='Change the grass color to red',
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (157, 172, 217)},
{"position": (400, 50), "rgb": (105, 144, 218)},
{"position": (700, 100), "rgb": (118, 159, 233)},
{"position": (150, 400), "rgb": (195, 34, 60)},
{"position": (512, 336), "rgb": (222, 214, 193)},
{"position": (700, 400), "rgb": (197, 15, 43)},
{"position": (100, 600), "rgb": (105, 13, 18)},
{"position": (400, 600), "rgb": (169, 33, 44)},
{"position": (700, 600), "rgb": (101, 86, 93)},
{"position": (256, 256), "rgb": (181, 202, 222)},
{"position": (100, 100), "rgb": (156, 172, 217)},
{"position": (400, 50), "rgb": (105, 144, 217)},
{"position": (700, 100), "rgb": (118, 159, 232)},
{"position": (150, 400), "rgb": (180, 22, 52)},
{"position": (512, 336), "rgb": (221, 211, 194)},
{"position": (700, 400), "rgb": (192, 10, 46)},
{"position": (100, 600), "rgb": (102, 12, 22)},
{"position": (400, 600), "rgb": (161, 28, 47)},
{"position": (700, 600), "rgb": (100, 87, 94)},
{"position": (256, 256), "rgb": (181, 201, 221)},
]
if current_omni_platform.is_rocm():
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (156, 172, 215)},
{"position": (400, 50), "rgb": (106, 144, 216)},
{"position": (700, 100), "rgb": (118, 158, 231)},
{"position": (150, 400), "rgb": (183, 23, 48)},
{"position": (512, 336), "rgb": (218, 215, 191)},
{"position": (700, 400), "rgb": (194, 14, 42)},
{"position": (100, 600), "rgb": (105, 10, 16)},
{"position": (400, 600), "rgb": (167, 33, 46)},
{"position": (700, 600), "rgb": (102, 86, 92)},
{"position": (256, 256), "rgb": (181, 201, 220)},
{"position": (100, 100), "rgb": (156, 172, 217)},
{"position": (400, 50), "rgb": (105, 144, 217)},
{"position": (700, 100), "rgb": (118, 159, 232)},
{"position": (150, 400), "rgb": (180, 22, 52)},
{"position": (512, 336), "rgb": (221, 211, 194)},
{"position": (700, 400), "rgb": (192, 10, 46)},
{"position": (100, 600), "rgb": (102, 12, 22)},
{"position": (400, 600), "rgb": (161, 28, 47)},
{"position": (700, 600), "rgb": (100, 87, 94)},
{"position": (256, 256), "rgb": (181, 201, 221)},
]
PIXEL_TOLERANCE = 10
@@ -31,9 +31,10 @@ import torch
from PIL import Image
from safetensors.torch import save_file
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from vllm_omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
@@ -27,9 +27,10 @@ from typing import Any
import pytest
from PIL import Image
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from vllm_omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
# Reference pixel data extracted from the known-good output image
@@ -37,30 +38,30 @@ from vllm_omni.platforms import current_omni_platform
# "Generated with seed=52, num_inference_steps=15,
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (121, 118, 100)},
{"position": (400, 50), "rgb": (163, 162, 143)},
{"position": (700, 100), "rgb": (170, 156, 127)},
{"position": (150, 400), "rgb": (129, 127, 112)},
{"position": (512, 512), "rgb": (135, 61, 59)},
{"position": (700, 400), "rgb": (205, 107, 43)},
{"position": (100, 700), "rgb": (197, 177, 157)},
{"position": (400, 700), "rgb": (139, 107, 86)},
{"position": (700, 700), "rgb": (247, 205, 146)},
{"position": (256, 256), "rgb": (171, 160, 153)},
{"position": (100, 100), "rgb": (115, 113, 94)},
{"position": (400, 50), "rgb": (159, 160, 144)},
{"position": (700, 100), "rgb": (164, 151, 123)},
{"position": (150, 400), "rgb": (120, 121, 107)},
{"position": (512, 512), "rgb": (165, 133, 127)},
{"position": (700, 400), "rgb": (217, 130, 66)},
{"position": (100, 700), "rgb": (191, 168, 152)},
{"position": (400, 700), "rgb": (130, 96, 77)},
{"position": (700, 700), "rgb": (247, 203, 140)},
{"position": (256, 256), "rgb": (167, 156, 150)},
]
if current_omni_platform.is_rocm():
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (123, 119, 100)},
{"position": (400, 50), "rgb": (162, 161, 142)},
{"position": (700, 100), "rgb": (171, 156, 127)},
{"position": (150, 400), "rgb": (131, 128, 112)},
{"position": (512, 512), "rgb": (134, 61, 59)},
{"position": (700, 400), "rgb": (204, 107, 43)},
{"position": (100, 700), "rgb": (201, 180, 165)},
{"position": (400, 700), "rgb": (140, 108, 87)},
{"position": (700, 700), "rgb": (247, 205, 145)},
{"position": (256, 256), "rgb": (171, 160, 153)},
{"position": (100, 100), "rgb": (115, 113, 94)},
{"position": (400, 50), "rgb": (159, 160, 144)},
{"position": (700, 100), "rgb": (164, 151, 123)},
{"position": (150, 400), "rgb": (120, 121, 107)},
{"position": (512, 512), "rgb": (165, 133, 127)},
{"position": (700, 400), "rgb": (217, 130, 66)},
{"position": (100, 700), "rgb": (191, 168, 152)},
{"position": (400, 700), "rgb": (130, 96, 77)},
{"position": (700, 700), "rgb": (247, 203, 140)},
{"position": (256, 256), "rgb": (167, 156, 150)},
]
# Maximum allowed difference per color channel
@@ -26,8 +26,9 @@ from pathlib import Path
import pytest
from vllm.assets.image import ImageAsset
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import modify_stage_config
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "bagel_sharedmemory_ci.yaml")
@@ -11,8 +11,8 @@ It uses minimal settings to keep test time short for CI.
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
+50 -68
View File
@@ -15,19 +15,18 @@ from __future__ import annotations
import functools
import io
import os
import tempfile
from pathlib import Path
from urllib.request import urlopen
import numpy as np
import pytest
import soundfile as sf
import yaml
from huggingface_hub import snapshot_download
from vllm.sampling_params import SamplingParams
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
@@ -44,13 +43,9 @@ REFERENCE_STAGE0_TEMPERATURE = 1.0
REFERENCE_STAGE0_REPETITION_PENALTY = 2.0
def _stage_config(name: str) -> str:
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
STAGE_CONFIGS = [
_stage_config("cosyvoice3.yaml"),
_stage_config("cosyvoice3_async_chunk.yaml"),
ASYNC_CHUNK_MODES = [
pytest.param(False, id="sync"),
pytest.param(True, id="async_chunk"),
]
@@ -138,19 +133,6 @@ def _get_stage_engine_outputs(omni_runner: OmniRunner, stage_id: int):
raise AttributeError("Unable to locate stage outputs on Omni runner")
def _patched_stage_config(base_stage_config: str, model_dir: Path, tmp_dir: Path) -> str:
cfg = yaml.safe_load(Path(base_stage_config).read_text(encoding="utf-8"))
tokenizer_path = str(model_dir / "CosyVoice-BlankEN")
for stage in cfg.get("stage_args", []):
engine_args = stage.setdefault("engine_args", {})
engine_args["tokenizer"] = tokenizer_path
engine_args["enforce_eager"] = True
engine_args["hf_overrides"] = {"architectures": ["CosyVoice3Model"]}
out_path = tmp_dir / Path(base_stage_config).name
out_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8")
return str(out_path)
def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[str, object]]:
return [
{
@@ -165,54 +147,54 @@ def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[s
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("base_stage_config", STAGE_CONFIGS)
def test_cosyvoice3_offline_reference_zero_shot(base_stage_config: str) -> None:
@pytest.mark.parametrize("async_chunk", ASYNC_CHUNK_MODES)
def test_cosyvoice3_offline_reference_zero_shot(async_chunk: bool) -> None:
"""CosyVoice3 zero-shot reference inference should stop cleanly and produce sane audio."""
prompt_audio, prompt_sr = _load_reference_prompt_wav()
model_dir = _resolve_model_dir()
expected_stop_token = int(CosyVoice3Config().llm["eos_token_id"])
with tempfile.TemporaryDirectory(prefix="cv3-e2e-") as tmp:
stage_config = _patched_stage_config(base_stage_config, model_dir, Path(tmp))
with OmniRunner(
str(model_dir), seed=42, stage_configs_path=stage_config, stage_init_timeout=300
) as omni_runner:
sampling_params_list = omni_runner.get_default_sampling_params_list()
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
with OmniRunner(
str(model_dir),
seed=42,
stage_configs_path=get_deploy_config_path("cosyvoice3.yaml"),
async_chunk=async_chunk,
stage_init_timeout=300,
) as omni_runner:
sampling_params_list = omni_runner.get_default_sampling_params_list()
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
outputs = omni_runner.omni.generate(
_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list
outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list)
assert outputs, "No outputs returned"
audio_mm = outputs[0].multimodal_output
assert "audio" in audio_mm, "No audio output found"
audio = _concat_audio(audio_mm["audio"])
assert audio.size > 0, "Generated audio is empty"
sr_val = audio_mm.get("sr", 24000)
if isinstance(sr_val, list) and sr_val:
sr_val = sr_val[-1]
if hasattr(sr_val, "item"):
sr_val = sr_val.item()
sr = int(sr_val)
assert sr == 24000, f"Unexpected sample_rate={sr}"
duration_s = audio.size / sr
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
if stage0_outputs:
completion = stage0_outputs[0].outputs[0]
finish_reason = getattr(completion, "finish_reason", None)
stop_reason = getattr(completion, "stop_reason", None)
num_tokens = len(getattr(completion, "token_ids", []) or [])
assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
assert int(stop_reason) == expected_stop_token, (
f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
)
assert outputs, "No outputs returned"
audio_mm = outputs[0].multimodal_output
assert "audio" in audio_mm, "No audio output found"
audio = _concat_audio(audio_mm["audio"])
assert audio.size > 0, "Generated audio is empty"
sr_val = audio_mm.get("sr", 24000)
if isinstance(sr_val, list) and sr_val:
sr_val = sr_val[-1]
if hasattr(sr_val, "item"):
sr_val = sr_val.item()
sr = int(sr_val)
assert sr == 24000, f"Unexpected sample_rate={sr}"
duration_s = audio.size / sr
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
if stage0_outputs:
completion = stage0_outputs[0].outputs[0]
finish_reason = getattr(completion, "finish_reason", None)
stop_reason = getattr(completion, "stop_reason", None)
num_tokens = len(getattr(completion, "token_ids", []) or [])
assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
assert int(stop_reason) == expected_stop_token, (
f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
)
assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
else:
assert "async_chunk" in Path(base_stage_config).name, "Stage-0 produced no engine outputs"
assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
else:
assert async_chunk, "Stage-0 produced no engine outputs"
@@ -4,8 +4,9 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor, hardware_test
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -2,8 +2,8 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -7,7 +7,7 @@ import pytest
import torch
from safetensors.torch import save_file
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -18,7 +18,7 @@ import pytest
import torch
from transformers import AutoTokenizer
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -18,8 +18,8 @@ import torch
import torch.distributed as dist
from PIL import Image
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
+37
View File
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Flux1 Schnell."""
import pytest
from PIL import Image
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "black-forest-labs/FLUX.1-schnell"
@pytest.mark.core_model
@pytest.mark.diffusion
def test_flux_schnell_text_to_image():
"""Test FLUX.1-schnell text-to-image generation."""
omni = Omni(model=MODEL)
omni_outputs = list(
omni.generate(
prompts=["A photo of a cat sitting on a laptop"],
sampling_params_list=OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
seed=42,
),
)
)
assert len(omni_outputs) > 0
images = omni_outputs[0].images
assert len(images) == 1
assert isinstance(images[0], Image.Image)
assert images[0].size == (512, 512)
@@ -0,0 +1,216 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for Flux2 Klein inpainting.
"""
# ruff: noqa: E402
import os
import sys
from pathlib import Path
import pytest
import torch
from PIL import Image, ImageDraw
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from vllm_omni import Omni
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
MODEL = "black-forest-labs/FLUX.2-klein-4B"
_HEIGHT = 512
_WIDTH = 512
_NUM_INFERENCE_STEPS = 4
def _create_test_image(width: int = _WIDTH, height: int = _HEIGHT, color: tuple = (128, 128, 128)) -> Image.Image:
return Image.new("RGB", (width, height), color)
def _create_test_mask(width: int = _WIDTH, height: int = _HEIGHT) -> Image.Image:
mask = Image.new("L", (width, height), 0)
draw = ImageDraw.Draw(mask)
draw.rectangle([width // 4, height // 4, width * 3 // 4, height * 3 // 4], fill=255)
return mask
def _create_test_inputs(color: tuple = (100, 150, 200)):
return _create_test_image(_WIDTH, _HEIGHT, color), _create_test_mask(_WIDTH, _HEIGHT)
def _extract_images_from_output(outputs: list) -> list[Image.Image]:
images = []
for req_output in outputs:
if hasattr(req_output, "images") and req_output.images:
images.extend(req_output.images)
elif hasattr(req_output, "request_output") and req_output.request_output:
stage_out = req_output.request_output
if isinstance(stage_out, OmniRequestOutput) and hasattr(stage_out, "images"):
images.extend(stage_out.images)
elif isinstance(stage_out, list):
for s in stage_out:
if hasattr(s, "images") and s.images:
images.extend(s.images)
return images
@pytest.mark.core_model
@pytest.mark.diffusion
def test_flux2_klein_inpaint_basic():
m = None
try:
m = Omni(model=MODEL)
input_image, mask_image = _create_test_inputs()
outputs = m.generate(
prompts=[
{
"prompt": "Fill in the masked area with a beautiful garden",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=1,
),
)
images = _extract_images_from_output(list(outputs))
assert len(images) == 1
assert images[0].size == (_WIDTH, _HEIGHT)
finally:
if m is not None and hasattr(m, "close"):
m.close()
@pytest.mark.diffusion
def test_flux2_klein_inpaint_deterministic():
m = None
try:
m = Omni(model=MODEL)
input_image, mask_image = _create_test_inputs()
seed = 12345
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
outputs1 = m.generate(
prompts=[
{
"prompt": "A red flower in a field",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen1,
num_outputs_per_prompt=1,
),
)
outputs2 = m.generate(
prompts=[
{
"prompt": "A red flower in a field",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen2,
num_outputs_per_prompt=1,
),
)
images1 = _extract_images_from_output(list(outputs1))
images2 = _extract_images_from_output(list(outputs2))
assert len(images1) == 1
assert len(images2) == 1
assert list(images1[0].getdata()) == list(images2[0].getdata()), (
"Same input with same seed should produce identical output. "
"This is critical for offline/online consistency."
)
finally:
if m is not None and hasattr(m, "close"):
m.close()
@pytest.mark.diffusion
def test_flux2_klein_inpaint_different_seeds_different_output():
m = None
try:
m = Omni(model=MODEL)
input_image, mask_image = _create_test_inputs()
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(42)
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(99999)
outputs1 = m.generate(
prompts=[
{
"prompt": "A beautiful landscape",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen1,
num_outputs_per_prompt=1,
),
)
outputs2 = m.generate(
prompts=[
{
"prompt": "A beautiful landscape",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen2,
num_outputs_per_prompt=1,
),
)
images1 = _extract_images_from_output(list(outputs1))
images2 = _extract_images_from_output(list(outputs2))
assert len(images1) == 1
assert len(images2) == 1
different_pixel_count = sum(1 for p1, p2 in zip(images1[0].getdata(), images2[0].getdata()) if p1 != p2)
assert different_pixel_count > 0, "Different seeds should produce different outputs"
finally:
if m is not None and hasattr(m, "close"):
m.close()
@@ -14,8 +14,9 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor, hardware_test
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -13,7 +13,7 @@ import pytest
from PIL import Image
from vllm.assets.image import ImageAsset
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -8,7 +8,7 @@ import torch.nn.functional as F
from PIL import Image
from transformers import CLIPModel, CLIPProcessor
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -17,7 +17,7 @@ PROMPT = "A brown and white dog is running on the grass"
MODEL_NAME = "tencent/HunyuanImage-3.0"
LOCAL_CLIP_PATH = "openai/clip-vit-base-patch32"
REPO_ROOT = Path(__file__).resolve().parents[3]
STAGE_CONFIG_PATH = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "hunyuan_image_3_moe.yaml"
STAGE_CONFIG_PATH = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "hunyuan_image3_t2i.yaml"
pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion]
@@ -11,7 +11,7 @@ import numpy as np
import pytest
from PIL import Image
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
REPO_ROOT = Path(__file__).resolve().parents[3]
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
@@ -8,8 +8,8 @@ import av
import numpy as np
import pytest
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -23,8 +23,10 @@ import pytest
import torch
from vllm.sampling_params import SamplingParams
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
# ---------------------------------------------------------------------------
# Constants
@@ -0,0 +1,142 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
)
from tests.helpers.stage_config import modify_stage_config
models = ["Jonathan1909/Ming-flash-omni-2.0"]
# Ming-specific
SYSTEM_PROMPT = "你是一个友好的AI助手。\n\ndetailed thinking off"
EOS_TOKEN = "<|role_end|>"
IMAGE_TOKEN = "<IMAGE>"
VIDEO_TOKEN = "<VIDEO>"
AUDIO_TOKEN = "<AUDIO>"
def build_prompt(user_text: str) -> str:
"""Build a Ming chat prompt."""
return (
f"<role>SYSTEM</role>{SYSTEM_PROMPT}{EOS_TOKEN}<role>HUMAN</role>{user_text}{EOS_TOKEN}<role>ASSISTANT</role>"
)
def get_eager_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "bailingmm_moe_v2_lite_ci.yaml"),
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
},
},
)
return path
stage_configs = [get_eager_config()]
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
"""
Test text-only input processing and text output generation.
Input Modal: text
Output Modal: text
"""
prompt = build_prompt("请详细介绍鹦鹉的生活习性。")
request_config = {"prompts": prompt, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_image_to_text(omni_runner, omni_runner_handler) -> None:
"""
Test image understanding with text output.
Input Modal: image + text
Output Modal: text
"""
image = generate_synthetic_image(224, 224)["np_array"]
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
request_config = {"prompts": prompt, "images": image, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
"""
Test audio understanding with text output.
Input Modal: audio + text
Output Modal: text
"""
audio = generate_synthetic_audio(2, 1, 16000)["np_array"]
if len(audio.shape) == 2:
audio = audio.squeeze()
prompt = build_prompt(f"{AUDIO_TOKEN}Please recognize the language of this speech and transcribe it. Format: oral.")
request_config = {"prompts": prompt, "audios": audio, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_video_to_text(omni_runner, omni_runner_handler) -> None:
"""
Test video understanding with text output.
Input Modal: video + text
Output Modal: text
"""
video = generate_synthetic_video(224, 224, 30)["np_array"]
prompt = build_prompt(f"{VIDEO_TOKEN}Describe what is happening in this video.")
request_config = {"prompts": prompt, "videos": video, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_mixed_to_text(omni_runner, omni_runner_handler) -> None:
"""
Test mixed modality input (image + audio) with text output.
Input Modal: image + audio + text
Output Modal: text
"""
image = generate_synthetic_image(224, 224)["np_array"]
audio = generate_synthetic_audio(2, 1, 16000)["np_array"]
if len(audio.shape) == 2:
audio = audio.squeeze()
prompt = build_prompt(f"{IMAGE_TOKEN}{AUDIO_TOKEN}Describe the image and transcribe the audio.")
request_config = {"prompts": prompt, "images": image, "audios": audio, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
@@ -0,0 +1,159 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end tests for Omni Sleep Mode across various model architectures.
"""
import pytest
import torch
from vllm import SamplingParams
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
MODEL_DIFF = "riverclouds/qwen_image_random"
def get_ack_info(ack, key, default=None):
if hasattr(ack, key):
return getattr(ack, key)
if isinstance(ack, dict):
return ack.get(key, default)
return default
def get_dynamic_devices(stage_idx, num_stages, tp_size):
total_gpus = torch.cuda.device_count()
gpus_per_stage = tp_size
start_idx = stage_idx * gpus_per_stage
if start_idx + gpus_per_stage > total_gpus:
start_idx = start_idx % total_gpus
device_ids = [str(start_idx + i) for i in range(gpus_per_stage)]
return ",".join(device_ids)
# Test 1: Diffusion Model (2-Stage BAGEL)
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.parametrize("tp_size", [1, 2])
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.asyncio
async def test_diffusion_model_sleep_tp(tp_size):
num_gpus = torch.cuda.device_count()
if num_gpus < tp_size:
pytest.skip(f"Skipping TP={tp_size}")
engine_args = {
"model": MODEL,
"enable_sleep_mode": True,
"tensor_parallel_size": tp_size,
"enforce_eager": True,
"trust_remote_code": True,
"dtype": "bfloat16",
"gpu_memory_utilization": 0.5,
}
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
try:
# BAGEL requires 2 params
diff_sp = OmniDiffusionSamplingParams(num_inference_steps=2, height=256, width=256)
llm_sp = SamplingParams()
# Warmup
async for _ in engine.generate("test", sampling_params=[llm_sp, diff_sp]):
pass
# Sleep all
acks = await engine.sleep(level=2)
statuses = [get_ack_info(ack, "status") for ack in acks]
assert all(s == "SUCCESS" for s in statuses), f"Sleep failed. Statuses: {statuses}"
# Wakeup & Verify
await engine.wake_up()
async for _ in engine.generate("verify", sampling_params=[llm_sp, diff_sp]):
pass
print(f"Diffusion TP={tp_size} Lifecycle OK")
finally:
engine.shutdown()
# Test 2: Multi-stage Manual Config
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.parametrize("tp_size", [1, 2])
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.asyncio
async def test_multistage_sleep_h100(tp_size):
num_gpus = torch.cuda.device_count()
if num_gpus < tp_size * 2:
pytest.skip("Not enough GPUs")
stages = []
for i in range(2):
devs = get_dynamic_devices(i, 2, tp_size)
stages.append(
{
"stage_id": i,
"stage_type": "llm" if i == 0 else "diffusion",
"runtime": {"process": True, "devices": devs},
"engine_args": {
"model": MODEL,
"model_stage": "thinker" if i == 0 else "base",
"tensor_parallel_size": tp_size,
"gpu_memory_utilization": 0.4,
"dtype": "bfloat16",
"enable_sleep_mode": True,
"trust_remote_code": True,
},
}
)
connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
engine = AsyncOmni(
model=MODEL, stages=stages, connectors=connectors, enable_sleep_mode=True, stage_init_timeout=1200
)
try:
sp = OmniDiffusionSamplingParams(num_inference_steps=2)
async for _ in engine.generate("warmup", sampling_params=[SamplingParams(), sp]):
pass
acks = await engine.sleep(stage_ids=[0, 1], level=2)
assert len(acks) == 2 * tp_size
await engine.wake_up(stage_ids=[0, 1])
async for _ in engine.generate("verify", sampling_params=[SamplingParams(), sp]):
pass
finally:
engine.shutdown()
# Test 3: Pure Diffusion Single-Stage
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.parametrize("tp_size", [1, 2])
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.asyncio
async def test_pure_diffusion_scenario(tp_size):
engine_args = {
"model": MODEL_DIFF,
"enable_sleep_mode": True,
"tensor_parallel_size": tp_size,
"enforce_eager": True,
"dtype": "bfloat16",
"gpu_memory_utilization": 0.5,
}
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
try:
await engine.sleep(level=1)
await engine.wake_up()
async for _ in engine.generate("test", sampling_params=[SamplingParams()]):
pass
print("Pure Diffusion OK")
finally:
engine.shutdown()
@@ -16,8 +16,8 @@ from pathlib import Path
import numpy as np
import pytest
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
MODEL = "k2-fsa/OmniVoice"
@@ -16,7 +16,7 @@ import pytest
import torch
from pytest_mock import MockerFixture
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
# Mock the OvisImageTransformer2DModel to avoid complex init if needed,
@@ -36,8 +36,8 @@ from typing import Any
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -2,46 +2,39 @@
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
"""
from pathlib import Path
import pytest
from tests.conftest import (
from tests.helpers.mark import hardware_test
from tests.helpers.media import (
generate_synthetic_audio,
generate_synthetic_image,
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen2.5-Omni-7B"]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section. NPU still uses the legacy per-platform YAML until it
# also migrates to the new schema.
_CI_DEPLOY = get_deploy_config_path("ci/qwen2_5_omni.yaml")
def get_cuda_graph_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
return modify_stage_config(
_CI_DEPLOY,
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
1: {"engine_args.enforce_eager": "true"},
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
},
},
)
return path
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
if current_omni_platform.is_npu():
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
elif current_omni_platform.is_xpu():
# Intel XPU stage config optimized for B60 GPU
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml")
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu() or current_omni_platform.is_npu():
stage_config = _CI_DEPLOY
else:
stage_config = get_cuda_graph_config()
+15 -21
View File
@@ -7,41 +7,35 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import (
generate_synthetic_video,
modify_stage_config,
)
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_video
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.platforms import current_omni_platform
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
def get_cuda_graph_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
return modify_stage_config(
_CI_DEPLOY,
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
1: {"engine_args.enforce_eager": "true"},
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
},
},
)
return path
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
if current_omni_platform.is_rocm():
# ROCm stage config optimized for MI325 GPU
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
elif current_omni_platform.is_xpu():
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")]
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
stage_configs = [_CI_DEPLOY]
else:
stage_configs = [get_cuda_graph_config()]
@@ -13,12 +13,10 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import modify_stage_config
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
@@ -26,23 +24,31 @@ REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know wh
def get_cuda_graph_config():
path = modify_stage_config(
get_stage_config(),
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
Composes the synchronous (no-async-chunk) variant on top of the bundled
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
standalone variant yaml; same effective config, no checked-in file needed.
"""
return modify_stage_config(
get_deploy_config_path("qwen3_tts.yaml"),
updates={
"stage_args": {
"async_chunk": False,
"stages": {
0: {
"engine_args.enforce_eager": "true",
"max_num_seqs": 1,
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {"engine_args.enforce_eager": "true"},
},
},
)
return path
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
# Same structure as test_qwen3_omni: models, stage_configs, test_params
@@ -13,34 +13,40 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import modify_stage_config
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
def get_cuda_graph_config():
path = modify_stage_config(
get_stage_config(),
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
Composes the synchronous (no-async-chunk) variant on top of the bundled
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
standalone variant yaml; same effective config, no checked-in file needed.
"""
return modify_stage_config(
get_deploy_config_path("qwen3_tts.yaml"),
updates={
"stage_args": {
"async_chunk": False,
"stages": {
0: {
"engine_args.enforce_eager": "true",
"max_num_seqs": 1,
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {
"gpu_memory_utilization": 0.2,
"enforce_eager": True,
"async_scheduling": False,
},
1: {"engine_args.enforce_eager": "true"},
},
},
)
return path
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
# Same structure as test_qwen3_omni: models, stage_configs, test_params
@@ -36,8 +36,8 @@ from pathlib import Path
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -20,8 +20,8 @@ import torch
import torch.distributed as dist
from PIL import Image
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -15,8 +15,8 @@ import numpy as np
import pytest
import torch
from tests.conftest import assert_audio_valid
from tests.utils import hardware_test
from tests.helpers.assertions import assert_audio_valid
from tests.helpers.mark import hardware_test
from vllm_omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -1,7 +1,7 @@
import pytest
import torch
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -3,7 +3,7 @@ import os
import pytest
import torch
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
+2 -2
View File
@@ -11,8 +11,8 @@ It uses minimal settings to keep test time short for CI.
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
@@ -18,7 +18,7 @@ if str(REPO_ROOT) not in sys.path:
import time
from tests.conftest import OmniRunner
from tests.helpers.runtime import OmniRunner
from vllm_omni.platforms import current_omni_platform
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
+156
View File
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test for VoxCPM offline inference."""
import json
import os
from pathlib import Path
from typing import Any
import numpy as np
import pytest
import torch
import tests.helpers.runtime as omni_runtime
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.model_executor.models.voxcpm.voxcpm_runtime_utils import (
prepare_voxcpm_hf_config_dir,
resolve_voxcpm_model_dir,
)
VOXCPM_MODEL = os.environ.get("VOXCPM_MODEL", "OpenBMB/VoxCPM1.5")
STAGE_CONFIG = str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
)
SAMPLE_RATE = 24000
@pytest.fixture(autouse=True)
def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
"""Limit the NPU cleanup workaround to this VoxCPM test module only."""
original_cleanup = omni_runtime.cleanup_dist_env_and_memory
def _safe_cleanup() -> None:
try:
original_cleanup()
except RuntimeError as exc:
if "Allocator for npu is not a DeviceAllocator" in str(exc):
return
raise
monkeypatch.setattr(omni_runtime, "cleanup_dist_env_and_memory", _safe_cleanup)
def _build_prompt(text: str) -> dict[str, Any]:
return {
"prompt_token_ids": [1],
"additional_information": {
"text": [text],
"cfg_value": [2.0],
"inference_timesteps": [10],
"min_len": [2],
"max_new_tokens": [1024],
},
}
def _extract_audio_tensor(multimodal_output: dict[str, Any]) -> torch.Tensor:
audio = multimodal_output.get("audio", multimodal_output.get("model_outputs"))
assert audio is not None, f"No audio output found, keys={list(multimodal_output.keys())}"
if isinstance(audio, list):
parts: list[torch.Tensor] = []
for item in audio:
if item is None:
continue
tensor = torch.as_tensor(item)
if tensor.numel() == 0:
continue
parts.append(tensor.float().cpu().reshape(-1))
return torch.cat(parts, dim=-1) if parts else torch.zeros((0,), dtype=torch.float32)
return torch.as_tensor(audio).float().cpu().reshape(-1)
def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
for item in reversed(outputs):
request_output = getattr(item, "request_output", None)
if request_output is not None:
multimodal_output = getattr(request_output, "multimodal_output", None)
if isinstance(multimodal_output, dict):
return multimodal_output
completions = getattr(request_output, "outputs", None) or []
for completion in completions:
multimodal_output = getattr(completion, "multimodal_output", None)
if isinstance(multimodal_output, dict):
return multimodal_output
multimodal_output = getattr(item, "multimodal_output", None)
if isinstance(multimodal_output, dict):
return multimodal_output
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
@pytest.fixture
def voxcpm_model_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> str:
model_dir = resolve_voxcpm_model_dir(VOXCPM_MODEL)
hf_config_env = os.environ.get("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH")
if hf_config_env:
hf_config_dir = Path(hf_config_env).expanduser()
else:
hf_config_dir = tmp_path / "voxcpm_hf_config"
if not (hf_config_dir / "config.json").exists():
prepare_voxcpm_hf_config_dir(model_dir, hf_config_dir)
monkeypatch.setenv("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH", str(hf_config_dir))
return str(model_dir)
def test_prepare_voxcpm_hf_config_dir(tmp_path: Path):
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "config.json").write_text(json.dumps({"hidden_size": 1024}), encoding="utf-8")
(model_dir / "generation_config.json").write_text(json.dumps({"do_sample": False}), encoding="utf-8")
hf_config_dir = prepare_voxcpm_hf_config_dir(model_dir, tmp_path / "voxcpm_hf_config")
prepared_config = json.loads((hf_config_dir / "config.json").read_text(encoding="utf-8"))
assert prepared_config["model_type"] == "voxcpm"
assert prepared_config["architectures"] == ["VoxCPMForConditionalGeneration"]
assert (hf_config_dir / "generation_config.json").exists()
def test_resolve_voxcpm_model_dir_local_path(tmp_path: Path):
model_dir = tmp_path / "OpenBMB" / "VoxCPM1.5"
model_dir.mkdir(parents=True)
assert resolve_voxcpm_model_dir(str(model_dir)) == model_dir
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm_zero_shot_001(voxcpm_model_path: str):
with OmniRunner(voxcpm_model_path, stage_configs_path=STAGE_CONFIG) as runner:
outputs = list(runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
assert outputs, "No outputs returned"
multimodal_output = _extract_final_multimodal_output(outputs)
audio = _extract_audio_tensor(multimodal_output)
assert audio.numel() > SAMPLE_RATE // 2, f"Audio too short: {audio.numel()} samples"
duration_s = audio.shape[0] / SAMPLE_RATE
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
peak = float(torch.max(torch.abs(audio)).item()) if audio.numel() > 0 else 0.0
assert peak > 0.01, "Generated audio appears to be silence"
audio_np = audio.numpy()
rms = float(np.sqrt(np.mean(np.square(audio_np)))) if audio_np.size else 0.0
assert rms > 1e-4, "Generated audio RMS too low"
+39 -17
View File
@@ -5,27 +5,19 @@ import os
import pytest
import torch
from tests.conftest import OmniRunner
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
VOXCPM2_MODEL = "openbmb/VoxCPM2"
STAGE_CONFIG = os.path.join(
os.path.dirname(__file__),
"..",
"..",
"..",
"vllm_omni",
"model_executor",
"stage_configs",
"voxcpm2.yaml",
)
DEPLOY_CONFIG = get_deploy_config_path("voxcpm2.yaml")
SAMPLE_RATE = 48000
@pytest.fixture(scope="module")
def voxcpm2_engine():
"""Create VoxCPM2 engine for testing."""
with OmniRunner(VOXCPM2_MODEL, stage_configs_path=STAGE_CONFIG) as runner:
with OmniRunner(VOXCPM2_MODEL, stage_configs_path=DEPLOY_CONFIG) as runner:
yield runner.omni
@@ -33,14 +25,16 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor:
"""Extract the final complete audio tensor from multimodal output."""
assert isinstance(multimodal_output, dict), f"Expected dict, got {type(multimodal_output)}"
# Output processor accumulates per-step full audio under "audio".
audio = multimodal_output.get("audio") or multimodal_output.get("model_outputs")
# Output processor accumulates per-step audio chunks under "audio".
audio = multimodal_output.get("audio")
if audio is None:
audio = multimodal_output.get("model_outputs")
assert audio is not None, f"No audio key, got {list(multimodal_output.keys())}"
if isinstance(audio, list):
valid = [x for x in audio if isinstance(x, torch.Tensor) and x.numel() > 100]
valid = [torch.as_tensor(x).float().cpu().reshape(-1) for x in audio if x is not None]
assert valid, "No valid audio tensors in output list"
audio = valid[-1]
audio = torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}"
return audio
@@ -98,3 +92,31 @@ def test_voxcpm2_voice_clone_002(voxcpm2_engine):
audio = _extract_audio(outputs[0].outputs[0].multimodal_output)
duration_s = audio.shape[0] / SAMPLE_RATE
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm2_prefill_decode_mixed_batch_003(voxcpm2_engine):
"""Regression: prefill+decode mixed batch must not crash (PR #2903)."""
long_prompt = (
"This is a deliberately long prompt that will stay in the decode "
"phase for many steps so that subsequent shorter prompts keep "
"entering prefill alongside it, reproducing the prefill plus "
"decode mixed batch scheduling pattern."
)
short_prompts = [
"Hello one.",
"Hello two.",
"Hello three.",
"Hello four.",
]
requests = [{"prompt": long_prompt}] + [{"prompt": p} for p in short_prompts]
outputs = voxcpm2_engine.generate(requests)
assert len(outputs) == len(requests)
for i, out in enumerate(outputs):
audio = _extract_audio(out.outputs[0].multimodal_output)
duration_s = audio.shape[0] / SAMPLE_RATE
assert 0.1 < duration_s < 30.0, f"Request {i} audio duration out of range: {duration_s:.2f}s"
@@ -29,14 +29,13 @@ from mistral_common.protocol.speech.request import SpeechRequest
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
from vllm import SamplingParams
from tests.conftest import OmniRunner, modify_stage_config
from tests.utils import hardware_test
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.entrypoints.async_omni import AsyncOmni
MODEL = "mistralai/Voxtral-4B-TTS-2603"
STAGE_CONFIG = str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxtral_tts.yaml"
)
STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml")
SAMPLE_RATE = 24000
# Minimum expected audio samples for a short sentence (~0.04s of 24kHz audio)
MIN_AUDIO_SAMPLES = 1000
@@ -65,9 +64,9 @@ def _resolve_stage_config(run_level: str) -> str:
return modify_stage_config(
STAGE_CONFIG,
deletes={
"stage_args": {
0: ["engine_args.load_format"],
1: ["engine_args.load_format"],
"stages": {
0: ["load_format"],
1: ["load_format"],
}
},
)
@@ -20,8 +20,9 @@ import pytest
import torch
from PIL import Image
from tests.conftest import OmniRunner
from tests.utils import DeviceMemoryMonitor, hardware_test
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -213,7 +214,10 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
)
@pytest.mark.integration
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
if current_omni_platform.is_npu():
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA and ROCm for now.")
-210
View File
@@ -1,210 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import functools
import os
import signal
import subprocess
import sys
import tempfile
from collections.abc import Callable
from contextlib import ExitStack, suppress
from pathlib import Path
from typing import Any, Literal
import cloudpickle
from typing_extensions import ParamSpec
from vllm.platforms import current_platform
VLLM_PATH = Path(__file__).parent.parent.parent
"""Path to root of the vLLM repository."""
_P = ParamSpec("_P")
def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, None]:
"""Decorator to fork a new process for each test function.
See https://github.com/vllm-project/vllm/issues/7053 for more details.
"""
@functools.wraps(func)
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
# Make the process the leader of its own process group
# to avoid sending SIGTERM to the parent process
os.setpgrp()
from _pytest.outcomes import Skipped
# Create a unique temporary file to store exception info from child
# process. Use test function name and process ID to avoid collisions.
with (
tempfile.NamedTemporaryFile(
delete=False,
mode="w+b",
prefix=f"vllm_test_{func.__name__}_{os.getpid()}_",
suffix=".exc",
) as exc_file,
ExitStack() as delete_after,
):
exc_file_path = exc_file.name
delete_after.callback(os.remove, exc_file_path)
pid = os.fork()
print(f"Fork a new process to run a test {pid}")
if pid == 0:
# Parent process responsible for deleting, don't delete
# in child.
delete_after.pop_all()
try:
func(*args, **kwargs)
except Skipped as e:
# convert Skipped to exit code 0
print(str(e))
os._exit(0)
except Exception as e:
import traceback
tb_string = traceback.format_exc()
# Try to serialize the exception object first
exc_to_serialize: dict[str, Any]
try:
# First, try to pickle the actual exception with
# its traceback.
exc_to_serialize = {"pickled_exception": e}
# Test if it can be pickled
cloudpickle.dumps(exc_to_serialize)
except (Exception, KeyboardInterrupt):
# Fall back to string-based approach.
exc_to_serialize = {
"exception_type": type(e).__name__,
"exception_msg": str(e),
"traceback": tb_string,
}
try:
with open(exc_file_path, "wb") as f:
cloudpickle.dump(exc_to_serialize, f)
except Exception:
# Fallback: just print the traceback.
print(tb_string)
os._exit(1)
else:
os._exit(0)
else:
pgid = os.getpgid(pid)
_pid, _exitcode = os.waitpid(pid, 0)
# ignore SIGTERM signal itself
old_signal_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
# kill all child processes
os.killpg(pgid, signal.SIGTERM)
# restore the signal handler
signal.signal(signal.SIGTERM, old_signal_handler)
if _exitcode != 0:
# Try to read the exception from the child process
exc_info = {}
if os.path.exists(exc_file_path):
with (
contextlib.suppress(Exception),
open(exc_file_path, "rb") as f,
):
exc_info = cloudpickle.load(f)
original_exception = exc_info.get("pickled_exception")
if original_exception is not None and isinstance(original_exception, Exception):
# Re-raise the actual exception object if it was
# successfully pickled.
raise original_exception
if (original_tb := exc_info.get("traceback")) is not None:
# Use string-based traceback for fallback case
raise AssertionError(
f"Test {func.__name__} failed when called with"
f" args {args} and kwargs {kwargs}"
f" (exit code: {_exitcode}):\n{original_tb}"
) from None
# Fallback to the original generic error
raise AssertionError(
f"function {func.__name__} failed when called with"
f" args {args} and kwargs {kwargs}"
f" (exit code: {_exitcode})"
) from None
return wrapper
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
"""Decorator to spawn a new process for each test function."""
@functools.wraps(f)
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
# Check if we're already in a subprocess
if os.environ.get("RUNNING_IN_SUBPROCESS") == "1":
# If we are, just run the function directly
return f(*args, **kwargs)
import torch.multiprocessing as mp
with suppress(RuntimeError):
mp.set_start_method("spawn")
# Get the module
module_name = f.__module__
# Create a process with environment variable set
env = os.environ.copy()
env["RUNNING_IN_SUBPROCESS"] = "1"
with tempfile.TemporaryDirectory() as tempdir:
output_filepath = os.path.join(tempdir, "new_process.tmp")
# `cloudpickle` allows pickling complex functions directly
input_bytes = cloudpickle.dumps((f, output_filepath))
repo_root = str(VLLM_PATH.resolve())
env = dict(env or os.environ)
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
cmd = [sys.executable, "-m", f"{module_name}"]
returned = subprocess.run(cmd, input=input_bytes, capture_output=True, env=env)
# check if the subprocess is successful
try:
returned.check_returncode()
except Exception as e:
# wrap raised exception to provide more information
raise RuntimeError(f"Error raised in subprocess:\n{returned.stderr.decode()}") from e
return wrapper
def create_new_process_for_each_test(
method: Literal["spawn", "fork"] | None = None,
) -> Callable[[Callable[_P, None]], Callable[_P, None]]:
"""Creates a decorator that runs each test function in a new process.
Args:
method: The process creation method. Can be either "spawn" or "fork".
If not specified, it defaults to "spawn" on ROCm and XPU
platforms and "fork" otherwise.
Returns:
A decorator to run test functions in separate processes.
"""
if method is None:
# TODO: Find out why spawn is not working correctly on ROCm
# The test content will not run and tests passed immediately.
# For now, using `fork` for ROCm as it can run with `fork`
# and tests are running correctly.
use_spawn = current_platform.is_xpu()
method = "spawn" if use_spawn else "fork"
assert method in ["spawn", "fork"], "Method must be either 'spawn' or 'fork'"
if method == "fork":
return fork_new_process_for_each_test
return spawn_new_process_for_each_test