Merge branch 'main' into feat/moss-tts-nano
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
611 files changed
+48775
-14783
No files matched your search
@@ -26,7 +26,7 @@ from contextlib import ExitStack
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -19,7 +19,7 @@ import numpy as np
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -10,7 +10,7 @@ import pytest
|
||||
from tests.e2e.offline_inference.custom_pipeline.worker_extension import (
|
||||
vLLMOmniColocateWorkerExtensionForTest,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import CustomPipelineWorkerExtension
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
|
||||
@@ -64,9 +64,6 @@ stage_args:
|
||||
# Top-level runtime config with Mooncake connector
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1
|
||||
max_inflight: 1
|
||||
connectors:
|
||||
mooncake_connector:
|
||||
name: MooncakeConnector
|
||||
@@ -80,4 +77,3 @@ runtime:
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
window_size: -1
|
||||
@@ -62,10 +62,6 @@ stage_args:
|
||||
# Runtime edges
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1
|
||||
max_inflight: 1
|
||||
|
||||
# Distributed connectors configuration (optional)
|
||||
# More connectors will be supported in the future.
|
||||
connectors:
|
||||
@@ -78,4 +74,3 @@ runtime:
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
window_size: -1
|
||||
@@ -1,103 +0,0 @@
|
||||
# stage config for running qwen2.5-omni for multi-stage omni runtime.
|
||||
|
||||
# This config is optimized for CI e2e tests.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1 # Simplified: trigger downstream only after full upstream completion
|
||||
max_inflight: 1 # Simplified: process serially within each stage
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
window_size: -1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
window_size: -1
|
||||
@@ -22,9 +22,10 @@ import pytest
|
||||
from PIL import Image
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.conftest import OmniRunner, modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni import Omni
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
@@ -32,30 +33,30 @@ from vllm_omni.platforms import current_omni_platform
|
||||
# prompt='Change the grass color to red',
|
||||
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (157, 172, 217)},
|
||||
{"position": (400, 50), "rgb": (105, 144, 218)},
|
||||
{"position": (700, 100), "rgb": (118, 159, 233)},
|
||||
{"position": (150, 400), "rgb": (195, 34, 60)},
|
||||
{"position": (512, 336), "rgb": (222, 214, 193)},
|
||||
{"position": (700, 400), "rgb": (197, 15, 43)},
|
||||
{"position": (100, 600), "rgb": (105, 13, 18)},
|
||||
{"position": (400, 600), "rgb": (169, 33, 44)},
|
||||
{"position": (700, 600), "rgb": (101, 86, 93)},
|
||||
{"position": (256, 256), "rgb": (181, 202, 222)},
|
||||
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||
]
|
||||
|
||||
if current_omni_platform.is_rocm():
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (156, 172, 215)},
|
||||
{"position": (400, 50), "rgb": (106, 144, 216)},
|
||||
{"position": (700, 100), "rgb": (118, 158, 231)},
|
||||
{"position": (150, 400), "rgb": (183, 23, 48)},
|
||||
{"position": (512, 336), "rgb": (218, 215, 191)},
|
||||
{"position": (700, 400), "rgb": (194, 14, 42)},
|
||||
{"position": (100, 600), "rgb": (105, 10, 16)},
|
||||
{"position": (400, 600), "rgb": (167, 33, 46)},
|
||||
{"position": (700, 600), "rgb": (102, 86, 92)},
|
||||
{"position": (256, 256), "rgb": (181, 201, 220)},
|
||||
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||
]
|
||||
|
||||
PIXEL_TOLERANCE = 10
|
||||
|
||||
@@ -31,9 +31,10 @@ import torch
|
||||
from PIL import Image
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from tests.conftest import OmniRunner, modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni import Omni
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
|
||||
|
||||
@@ -27,9 +27,10 @@ from typing import Any
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.conftest import OmniRunner, modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni import Omni
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
@@ -37,30 +38,30 @@ from vllm_omni.platforms import current_omni_platform
|
||||
# "Generated with seed=52, num_inference_steps=15,
|
||||
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (121, 118, 100)},
|
||||
{"position": (400, 50), "rgb": (163, 162, 143)},
|
||||
{"position": (700, 100), "rgb": (170, 156, 127)},
|
||||
{"position": (150, 400), "rgb": (129, 127, 112)},
|
||||
{"position": (512, 512), "rgb": (135, 61, 59)},
|
||||
{"position": (700, 400), "rgb": (205, 107, 43)},
|
||||
{"position": (100, 700), "rgb": (197, 177, 157)},
|
||||
{"position": (400, 700), "rgb": (139, 107, 86)},
|
||||
{"position": (700, 700), "rgb": (247, 205, 146)},
|
||||
{"position": (256, 256), "rgb": (171, 160, 153)},
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||
]
|
||||
|
||||
if current_omni_platform.is_rocm():
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (123, 119, 100)},
|
||||
{"position": (400, 50), "rgb": (162, 161, 142)},
|
||||
{"position": (700, 100), "rgb": (171, 156, 127)},
|
||||
{"position": (150, 400), "rgb": (131, 128, 112)},
|
||||
{"position": (512, 512), "rgb": (134, 61, 59)},
|
||||
{"position": (700, 400), "rgb": (204, 107, 43)},
|
||||
{"position": (100, 700), "rgb": (201, 180, 165)},
|
||||
{"position": (400, 700), "rgb": (140, 108, 87)},
|
||||
{"position": (700, 700), "rgb": (247, 205, 145)},
|
||||
{"position": (256, 256), "rgb": (171, 160, 153)},
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||
]
|
||||
|
||||
# Maximum allowed difference per color channel
|
||||
|
||||
@@ -26,8 +26,9 @@ from pathlib import Path
|
||||
import pytest
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.conftest import OmniRunner, modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
|
||||
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "bagel_sharedmemory_ci.yaml")
|
||||
|
||||
@@ -11,8 +11,8 @@ It uses minimal settings to keep test time short for CI.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -15,19 +15,18 @@ from __future__ import annotations
|
||||
import functools
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
import yaml
|
||||
from huggingface_hub import snapshot_download
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
|
||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||
|
||||
@@ -44,13 +43,9 @@ REFERENCE_STAGE0_TEMPERATURE = 1.0
|
||||
REFERENCE_STAGE0_REPETITION_PENALTY = 2.0
|
||||
|
||||
|
||||
def _stage_config(name: str) -> str:
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
STAGE_CONFIGS = [
|
||||
_stage_config("cosyvoice3.yaml"),
|
||||
_stage_config("cosyvoice3_async_chunk.yaml"),
|
||||
ASYNC_CHUNK_MODES = [
|
||||
pytest.param(False, id="sync"),
|
||||
pytest.param(True, id="async_chunk"),
|
||||
]
|
||||
|
||||
|
||||
@@ -138,19 +133,6 @@ def _get_stage_engine_outputs(omni_runner: OmniRunner, stage_id: int):
|
||||
raise AttributeError("Unable to locate stage outputs on Omni runner")
|
||||
|
||||
|
||||
def _patched_stage_config(base_stage_config: str, model_dir: Path, tmp_dir: Path) -> str:
|
||||
cfg = yaml.safe_load(Path(base_stage_config).read_text(encoding="utf-8"))
|
||||
tokenizer_path = str(model_dir / "CosyVoice-BlankEN")
|
||||
for stage in cfg.get("stage_args", []):
|
||||
engine_args = stage.setdefault("engine_args", {})
|
||||
engine_args["tokenizer"] = tokenizer_path
|
||||
engine_args["enforce_eager"] = True
|
||||
engine_args["hf_overrides"] = {"architectures": ["CosyVoice3Model"]}
|
||||
out_path = tmp_dir / Path(base_stage_config).name
|
||||
out_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8")
|
||||
return str(out_path)
|
||||
|
||||
|
||||
def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
@@ -165,54 +147,54 @@ def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[s
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("base_stage_config", STAGE_CONFIGS)
|
||||
def test_cosyvoice3_offline_reference_zero_shot(base_stage_config: str) -> None:
|
||||
@pytest.mark.parametrize("async_chunk", ASYNC_CHUNK_MODES)
|
||||
def test_cosyvoice3_offline_reference_zero_shot(async_chunk: bool) -> None:
|
||||
"""CosyVoice3 zero-shot reference inference should stop cleanly and produce sane audio."""
|
||||
prompt_audio, prompt_sr = _load_reference_prompt_wav()
|
||||
model_dir = _resolve_model_dir()
|
||||
expected_stop_token = int(CosyVoice3Config().llm["eos_token_id"])
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="cv3-e2e-") as tmp:
|
||||
stage_config = _patched_stage_config(base_stage_config, model_dir, Path(tmp))
|
||||
with OmniRunner(
|
||||
str(model_dir), seed=42, stage_configs_path=stage_config, stage_init_timeout=300
|
||||
) as omni_runner:
|
||||
sampling_params_list = omni_runner.get_default_sampling_params_list()
|
||||
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
|
||||
with OmniRunner(
|
||||
str(model_dir),
|
||||
seed=42,
|
||||
stage_configs_path=get_deploy_config_path("cosyvoice3.yaml"),
|
||||
async_chunk=async_chunk,
|
||||
stage_init_timeout=300,
|
||||
) as omni_runner:
|
||||
sampling_params_list = omni_runner.get_default_sampling_params_list()
|
||||
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
|
||||
|
||||
outputs = omni_runner.omni.generate(
|
||||
_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list
|
||||
outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list)
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, "No audio output found"
|
||||
|
||||
audio = _concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, "Generated audio is empty"
|
||||
|
||||
sr_val = audio_mm.get("sr", 24000)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
sr = int(sr_val)
|
||||
assert sr == 24000, f"Unexpected sample_rate={sr}"
|
||||
|
||||
duration_s = audio.size / sr
|
||||
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
|
||||
|
||||
stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
|
||||
if stage0_outputs:
|
||||
completion = stage0_outputs[0].outputs[0]
|
||||
finish_reason = getattr(completion, "finish_reason", None)
|
||||
stop_reason = getattr(completion, "stop_reason", None)
|
||||
num_tokens = len(getattr(completion, "token_ids", []) or [])
|
||||
|
||||
assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
|
||||
assert int(stop_reason) == expected_stop_token, (
|
||||
f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
|
||||
)
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, "No audio output found"
|
||||
|
||||
audio = _concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, "Generated audio is empty"
|
||||
|
||||
sr_val = audio_mm.get("sr", 24000)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
sr = int(sr_val)
|
||||
assert sr == 24000, f"Unexpected sample_rate={sr}"
|
||||
|
||||
duration_s = audio.size / sr
|
||||
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
|
||||
|
||||
stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
|
||||
if stage0_outputs:
|
||||
completion = stage0_outputs[0].outputs[0]
|
||||
finish_reason = getattr(completion, "finish_reason", None)
|
||||
stop_reason = getattr(completion, "stop_reason", None)
|
||||
num_tokens = len(getattr(completion, "token_ids", []) or [])
|
||||
|
||||
assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
|
||||
assert int(stop_reason) == expected_stop_token, (
|
||||
f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
|
||||
)
|
||||
assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
|
||||
else:
|
||||
assert "async_chunk" in Path(base_stage_config).name, "Stage-0 produced no engine outputs"
|
||||
assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
|
||||
else:
|
||||
assert async_chunk, "Stage-0 produced no engine outputs"
|
||||
@@ -4,8 +4,9 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import DeviceMemoryMonitor
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -18,7 +18,7 @@ import pytest
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
@@ -18,8 +18,8 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from PIL import Image
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for Flux1 Schnell."""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.1-schnell"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux_schnell_text_to_image():
|
||||
"""Test FLUX.1-schnell text-to-image generation."""
|
||||
omni = Omni(model=MODEL)
|
||||
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=["A photo of a cat sitting on a laptop"],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0
|
||||
images = omni_outputs[0].images
|
||||
assert len(images) == 1
|
||||
assert isinstance(images[0], Image.Image)
|
||||
assert images[0].size == (512, 512)
|
||||
@@ -0,0 +1,216 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for Flux2 Klein inpainting.
|
||||
|
||||
"""
|
||||
|
||||
# ruff: noqa: E402
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.2-klein-4B"
|
||||
|
||||
_HEIGHT = 512
|
||||
_WIDTH = 512
|
||||
_NUM_INFERENCE_STEPS = 4
|
||||
|
||||
|
||||
def _create_test_image(width: int = _WIDTH, height: int = _HEIGHT, color: tuple = (128, 128, 128)) -> Image.Image:
|
||||
return Image.new("RGB", (width, height), color)
|
||||
|
||||
|
||||
def _create_test_mask(width: int = _WIDTH, height: int = _HEIGHT) -> Image.Image:
|
||||
mask = Image.new("L", (width, height), 0)
|
||||
draw = ImageDraw.Draw(mask)
|
||||
draw.rectangle([width // 4, height // 4, width * 3 // 4, height * 3 // 4], fill=255)
|
||||
return mask
|
||||
|
||||
|
||||
def _create_test_inputs(color: tuple = (100, 150, 200)):
|
||||
return _create_test_image(_WIDTH, _HEIGHT, color), _create_test_mask(_WIDTH, _HEIGHT)
|
||||
|
||||
|
||||
def _extract_images_from_output(outputs: list) -> list[Image.Image]:
|
||||
images = []
|
||||
for req_output in outputs:
|
||||
if hasattr(req_output, "images") and req_output.images:
|
||||
images.extend(req_output.images)
|
||||
elif hasattr(req_output, "request_output") and req_output.request_output:
|
||||
stage_out = req_output.request_output
|
||||
if isinstance(stage_out, OmniRequestOutput) and hasattr(stage_out, "images"):
|
||||
images.extend(stage_out.images)
|
||||
elif isinstance(stage_out, list):
|
||||
for s in stage_out:
|
||||
if hasattr(s, "images") and s.images:
|
||||
images.extend(s.images)
|
||||
return images
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_inpaint_basic():
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=MODEL)
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
|
||||
outputs = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "Fill in the masked area with a beautiful garden",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
images = _extract_images_from_output(list(outputs))
|
||||
assert len(images) == 1
|
||||
assert images[0].size == (_WIDTH, _HEIGHT)
|
||||
finally:
|
||||
if m is not None and hasattr(m, "close"):
|
||||
m.close()
|
||||
|
||||
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_inpaint_deterministic():
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=MODEL)
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
seed = 12345
|
||||
|
||||
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
|
||||
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
|
||||
|
||||
outputs1 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A red flower in a field",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen1,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
outputs2 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A red flower in a field",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen2,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
images1 = _extract_images_from_output(list(outputs1))
|
||||
images2 = _extract_images_from_output(list(outputs2))
|
||||
|
||||
assert len(images1) == 1
|
||||
assert len(images2) == 1
|
||||
|
||||
assert list(images1[0].getdata()) == list(images2[0].getdata()), (
|
||||
"Same input with same seed should produce identical output. "
|
||||
"This is critical for offline/online consistency."
|
||||
)
|
||||
finally:
|
||||
if m is not None and hasattr(m, "close"):
|
||||
m.close()
|
||||
|
||||
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_inpaint_different_seeds_different_output():
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=MODEL)
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
|
||||
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(42)
|
||||
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(99999)
|
||||
|
||||
outputs1 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A beautiful landscape",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen1,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
outputs2 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A beautiful landscape",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen2,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
images1 = _extract_images_from_output(list(outputs1))
|
||||
images2 = _extract_images_from_output(list(outputs2))
|
||||
|
||||
assert len(images1) == 1
|
||||
assert len(images2) == 1
|
||||
|
||||
different_pixel_count = sum(1 for p1, p2 in zip(images1[0].getdata(), images2[0].getdata()) if p1 != p2)
|
||||
assert different_pixel_count > 0, "Different seeds should produce different outputs"
|
||||
finally:
|
||||
if m is not None and hasattr(m, "close"):
|
||||
m.close()
|
||||
@@ -14,8 +14,9 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -13,7 +13,7 @@ import pytest
|
||||
from PIL import Image
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import CLIPModel, CLIPProcessor
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -17,7 +17,7 @@ PROMPT = "A brown and white dog is running on the grass"
|
||||
MODEL_NAME = "tencent/HunyuanImage-3.0"
|
||||
LOCAL_CLIP_PATH = "openai/clip-vit-base-patch32"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
STAGE_CONFIG_PATH = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "hunyuan_image_3_moe.yaml"
|
||||
STAGE_CONFIG_PATH = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "hunyuan_image3_t2i.yaml"
|
||||
|
||||
pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
|
||||
|
||||
@@ -8,8 +8,8 @@ import av
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
@@ -23,8 +23,10 @@ import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import (
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
generate_synthetic_video,
|
||||
)
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
|
||||
models = ["Jonathan1909/Ming-flash-omni-2.0"]
|
||||
|
||||
# Ming-specific
|
||||
SYSTEM_PROMPT = "你是一个友好的AI助手。\n\ndetailed thinking off"
|
||||
EOS_TOKEN = "<|role_end|>"
|
||||
IMAGE_TOKEN = "<IMAGE>"
|
||||
VIDEO_TOKEN = "<VIDEO>"
|
||||
AUDIO_TOKEN = "<AUDIO>"
|
||||
|
||||
|
||||
def build_prompt(user_text: str) -> str:
|
||||
"""Build a Ming chat prompt."""
|
||||
return (
|
||||
f"<role>SYSTEM</role>{SYSTEM_PROMPT}{EOS_TOKEN}<role>HUMAN</role>{user_text}{EOS_TOKEN}<role>ASSISTANT</role>"
|
||||
)
|
||||
|
||||
|
||||
def get_eager_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "bailingmm_moe_v2_lite_ci.yaml"),
|
||||
updates={
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
stage_configs = [get_eager_config()]
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test text-only input processing and text output generation.
|
||||
Input Modal: text
|
||||
Output Modal: text
|
||||
"""
|
||||
prompt = build_prompt("请详细介绍鹦鹉的生活习性。")
|
||||
request_config = {"prompts": prompt, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test image understanding with text output.
|
||||
Input Modal: image + text
|
||||
Output Modal: text
|
||||
"""
|
||||
image = generate_synthetic_image(224, 224)["np_array"]
|
||||
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
|
||||
request_config = {"prompts": prompt, "images": image, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test audio understanding with text output.
|
||||
Input Modal: audio + text
|
||||
Output Modal: text
|
||||
"""
|
||||
audio = generate_synthetic_audio(2, 1, 16000)["np_array"]
|
||||
if len(audio.shape) == 2:
|
||||
audio = audio.squeeze()
|
||||
prompt = build_prompt(f"{AUDIO_TOKEN}Please recognize the language of this speech and transcribe it. Format: oral.")
|
||||
request_config = {"prompts": prompt, "audios": audio, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test video understanding with text output.
|
||||
Input Modal: video + text
|
||||
Output Modal: text
|
||||
"""
|
||||
video = generate_synthetic_video(224, 224, 30)["np_array"]
|
||||
prompt = build_prompt(f"{VIDEO_TOKEN}Describe what is happening in this video.")
|
||||
request_config = {"prompts": prompt, "videos": video, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_mixed_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test mixed modality input (image + audio) with text output.
|
||||
Input Modal: image + audio + text
|
||||
Output Modal: text
|
||||
"""
|
||||
image = generate_synthetic_image(224, 224)["np_array"]
|
||||
audio = generate_synthetic_audio(2, 1, 16000)["np_array"]
|
||||
if len(audio.shape) == 2:
|
||||
audio = audio.squeeze()
|
||||
prompt = build_prompt(f"{IMAGE_TOKEN}{AUDIO_TOKEN}Describe the image and transcribe the audio.")
|
||||
request_config = {"prompts": prompt, "images": image, "audios": audio, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
@@ -0,0 +1,159 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end tests for Omni Sleep Mode across various model architectures.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
MODEL_DIFF = "riverclouds/qwen_image_random"
|
||||
|
||||
|
||||
def get_ack_info(ack, key, default=None):
|
||||
if hasattr(ack, key):
|
||||
return getattr(ack, key)
|
||||
if isinstance(ack, dict):
|
||||
return ack.get(key, default)
|
||||
return default
|
||||
|
||||
|
||||
def get_dynamic_devices(stage_idx, num_stages, tp_size):
|
||||
total_gpus = torch.cuda.device_count()
|
||||
gpus_per_stage = tp_size
|
||||
start_idx = stage_idx * gpus_per_stage
|
||||
if start_idx + gpus_per_stage > total_gpus:
|
||||
start_idx = start_idx % total_gpus
|
||||
device_ids = [str(start_idx + i) for i in range(gpus_per_stage)]
|
||||
return ",".join(device_ids)
|
||||
|
||||
|
||||
# Test 1: Diffusion Model (2-Stage BAGEL)
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_diffusion_model_sleep_tp(tp_size):
|
||||
num_gpus = torch.cuda.device_count()
|
||||
if num_gpus < tp_size:
|
||||
pytest.skip(f"Skipping TP={tp_size}")
|
||||
|
||||
engine_args = {
|
||||
"model": MODEL,
|
||||
"enable_sleep_mode": True,
|
||||
"tensor_parallel_size": tp_size,
|
||||
"enforce_eager": True,
|
||||
"trust_remote_code": True,
|
||||
"dtype": "bfloat16",
|
||||
"gpu_memory_utilization": 0.5,
|
||||
}
|
||||
|
||||
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
|
||||
try:
|
||||
# BAGEL requires 2 params
|
||||
diff_sp = OmniDiffusionSamplingParams(num_inference_steps=2, height=256, width=256)
|
||||
llm_sp = SamplingParams()
|
||||
|
||||
# Warmup
|
||||
async for _ in engine.generate("test", sampling_params=[llm_sp, diff_sp]):
|
||||
pass
|
||||
|
||||
# Sleep all
|
||||
acks = await engine.sleep(level=2)
|
||||
statuses = [get_ack_info(ack, "status") for ack in acks]
|
||||
assert all(s == "SUCCESS" for s in statuses), f"Sleep failed. Statuses: {statuses}"
|
||||
|
||||
# Wakeup & Verify
|
||||
await engine.wake_up()
|
||||
async for _ in engine.generate("verify", sampling_params=[llm_sp, diff_sp]):
|
||||
pass
|
||||
|
||||
print(f"Diffusion TP={tp_size} Lifecycle OK")
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
# Test 2: Multi-stage Manual Config
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_multistage_sleep_h100(tp_size):
|
||||
num_gpus = torch.cuda.device_count()
|
||||
if num_gpus < tp_size * 2:
|
||||
pytest.skip("Not enough GPUs")
|
||||
|
||||
stages = []
|
||||
for i in range(2):
|
||||
devs = get_dynamic_devices(i, 2, tp_size)
|
||||
stages.append(
|
||||
{
|
||||
"stage_id": i,
|
||||
"stage_type": "llm" if i == 0 else "diffusion",
|
||||
"runtime": {"process": True, "devices": devs},
|
||||
"engine_args": {
|
||||
"model": MODEL,
|
||||
"model_stage": "thinker" if i == 0 else "base",
|
||||
"tensor_parallel_size": tp_size,
|
||||
"gpu_memory_utilization": 0.4,
|
||||
"dtype": "bfloat16",
|
||||
"enable_sleep_mode": True,
|
||||
"trust_remote_code": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
|
||||
|
||||
engine = AsyncOmni(
|
||||
model=MODEL, stages=stages, connectors=connectors, enable_sleep_mode=True, stage_init_timeout=1200
|
||||
)
|
||||
try:
|
||||
sp = OmniDiffusionSamplingParams(num_inference_steps=2)
|
||||
async for _ in engine.generate("warmup", sampling_params=[SamplingParams(), sp]):
|
||||
pass
|
||||
|
||||
acks = await engine.sleep(stage_ids=[0, 1], level=2)
|
||||
assert len(acks) == 2 * tp_size
|
||||
|
||||
await engine.wake_up(stage_ids=[0, 1])
|
||||
async for _ in engine.generate("verify", sampling_params=[SamplingParams(), sp]):
|
||||
pass
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
# Test 3: Pure Diffusion Single-Stage
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_pure_diffusion_scenario(tp_size):
|
||||
engine_args = {
|
||||
"model": MODEL_DIFF,
|
||||
"enable_sleep_mode": True,
|
||||
"tensor_parallel_size": tp_size,
|
||||
"enforce_eager": True,
|
||||
"dtype": "bfloat16",
|
||||
"gpu_memory_utilization": 0.5,
|
||||
}
|
||||
|
||||
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
|
||||
try:
|
||||
await engine.sleep(level=1)
|
||||
await engine.wake_up()
|
||||
async for _ in engine.generate("test", sampling_params=[SamplingParams()]):
|
||||
pass
|
||||
print("Pure Diffusion OK")
|
||||
finally:
|
||||
engine.shutdown()
|
||||
@@ -16,8 +16,8 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
|
||||
MODEL = "k2-fsa/OmniVoice"
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
|
||||
# Mock the OvisImageTransformer2DModel to avoid complex init if needed,
|
||||
|
||||
@@ -36,8 +36,8 @@ from typing import Any
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -2,46 +2,39 @@
|
||||
E2E tests for Qwen2.5-Omni model with mixed modality inputs, audio and text output.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import (
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
|
||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||
# platforms: section. NPU still uses the legacy per-platform YAML until it
|
||||
# also migrates to the new schema.
|
||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen2_5_omni.yaml")
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen2_5_omni_ci.yaml"),
|
||||
return modify_stage_config(
|
||||
_CI_DEPLOY,
|
||||
updates={
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
"stages": {
|
||||
0: {"enforce_eager": True},
|
||||
1: {"enforce_eager": True},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
# CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU
|
||||
if current_omni_platform.is_npu():
|
||||
stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml")
|
||||
elif current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml")
|
||||
elif current_omni_platform.is_xpu():
|
||||
# Intel XPU stage config optimized for B60 GPU
|
||||
stage_config = str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen2_5_omni_ci.yaml")
|
||||
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu() or current_omni_platform.is_npu():
|
||||
stage_config = _CI_DEPLOY
|
||||
else:
|
||||
stage_config = get_cuda_graph_config()
|
||||
|
||||
|
||||
@@ -7,41 +7,35 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
generate_synthetic_video,
|
||||
modify_stage_config,
|
||||
)
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import generate_synthetic_video
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
|
||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
|
||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml"),
|
||||
return modify_stage_config(
|
||||
_CI_DEPLOY,
|
||||
updates={
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
"stages": {
|
||||
0: {"enforce_eager": True},
|
||||
1: {"enforce_eager": True},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
# CI stage config for 2xH100-80G GPUs or AMD GPU MI325
|
||||
if current_omni_platform.is_rocm():
|
||||
# ROCm stage config optimized for MI325 GPU
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")]
|
||||
elif current_omni_platform.is_xpu():
|
||||
stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "xpu" / "qwen3_omni_ci.yaml")]
|
||||
if current_omni_platform.is_rocm() or current_omni_platform.is_xpu():
|
||||
stage_configs = [_CI_DEPLOY]
|
||||
else:
|
||||
stage_configs = [get_cuda_graph_config()]
|
||||
|
||||
|
||||
@@ -13,12 +13,10 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
|
||||
@@ -26,23 +24,31 @@ REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know wh
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
get_stage_config(),
|
||||
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
|
||||
|
||||
Composes the synchronous (no-async-chunk) variant on top of the bundled
|
||||
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
|
||||
standalone variant yaml; same effective config, no checked-in file needed.
|
||||
"""
|
||||
return modify_stage_config(
|
||||
get_deploy_config_path("qwen3_tts.yaml"),
|
||||
updates={
|
||||
"stage_args": {
|
||||
"async_chunk": False,
|
||||
"stages": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
|
||||
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
# Same structure as test_qwen3_omni: models, stage_configs, test_params
|
||||
|
||||
@@ -13,34 +13,40 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
|
||||
|
||||
|
||||
def get_cuda_graph_config():
|
||||
path = modify_stage_config(
|
||||
get_stage_config(),
|
||||
"""Build a temp deploy yaml mirroring the deleted qwen3_tts_no_async_chunk.yaml.
|
||||
|
||||
Composes the synchronous (no-async-chunk) variant on top of the bundled
|
||||
qwen3_tts.yaml prod default, with cudagraphs disabled. Replaces the deleted
|
||||
standalone variant yaml; same effective config, no checked-in file needed.
|
||||
"""
|
||||
return modify_stage_config(
|
||||
get_deploy_config_path("qwen3_tts.yaml"),
|
||||
updates={
|
||||
"stage_args": {
|
||||
"async_chunk": False,
|
||||
"stages": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {
|
||||
"gpu_memory_utilization": 0.2,
|
||||
"enforce_eager": True,
|
||||
"async_scheduling": False,
|
||||
},
|
||||
1: {"engine_args.enforce_eager": "true"},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
|
||||
"""Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
# Same structure as test_qwen3_omni: models, stage_configs, test_params
|
||||
|
||||
@@ -36,8 +36,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -20,8 +20,8 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from PIL import Image
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -15,8 +15,8 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import assert_audio_valid
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.assertions import assert_audio_valid
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -3,7 +3,7 @@ import os
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -11,8 +11,8 @@ It uses minimal settings to keep test time short for CI.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -18,7 +18,7 @@ if str(REPO_ROOT) not in sys.path:
|
||||
|
||||
import time
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E test for VoxCPM offline inference."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import tests.helpers.runtime as omni_runtime
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.model_executor.models.voxcpm.voxcpm_runtime_utils import (
|
||||
prepare_voxcpm_hf_config_dir,
|
||||
resolve_voxcpm_model_dir,
|
||||
)
|
||||
|
||||
VOXCPM_MODEL = os.environ.get("VOXCPM_MODEL", "OpenBMB/VoxCPM1.5")
|
||||
STAGE_CONFIG = str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
|
||||
)
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Limit the NPU cleanup workaround to this VoxCPM test module only."""
|
||||
original_cleanup = omni_runtime.cleanup_dist_env_and_memory
|
||||
|
||||
def _safe_cleanup() -> None:
|
||||
try:
|
||||
original_cleanup()
|
||||
except RuntimeError as exc:
|
||||
if "Allocator for npu is not a DeviceAllocator" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
monkeypatch.setattr(omni_runtime, "cleanup_dist_env_and_memory", _safe_cleanup)
|
||||
|
||||
|
||||
def _build_prompt(text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": {
|
||||
"text": [text],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [1024],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _extract_audio_tensor(multimodal_output: dict[str, Any]) -> torch.Tensor:
|
||||
audio = multimodal_output.get("audio", multimodal_output.get("model_outputs"))
|
||||
assert audio is not None, f"No audio output found, keys={list(multimodal_output.keys())}"
|
||||
|
||||
if isinstance(audio, list):
|
||||
parts: list[torch.Tensor] = []
|
||||
for item in audio:
|
||||
if item is None:
|
||||
continue
|
||||
tensor = torch.as_tensor(item)
|
||||
if tensor.numel() == 0:
|
||||
continue
|
||||
parts.append(tensor.float().cpu().reshape(-1))
|
||||
return torch.cat(parts, dim=-1) if parts else torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
|
||||
for item in reversed(outputs):
|
||||
request_output = getattr(item, "request_output", None)
|
||||
if request_output is not None:
|
||||
multimodal_output = getattr(request_output, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
completions = getattr(request_output, "outputs", None) or []
|
||||
for completion in completions:
|
||||
multimodal_output = getattr(completion, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
|
||||
multimodal_output = getattr(item, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
|
||||
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def voxcpm_model_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> str:
|
||||
model_dir = resolve_voxcpm_model_dir(VOXCPM_MODEL)
|
||||
|
||||
hf_config_env = os.environ.get("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH")
|
||||
if hf_config_env:
|
||||
hf_config_dir = Path(hf_config_env).expanduser()
|
||||
else:
|
||||
hf_config_dir = tmp_path / "voxcpm_hf_config"
|
||||
|
||||
if not (hf_config_dir / "config.json").exists():
|
||||
prepare_voxcpm_hf_config_dir(model_dir, hf_config_dir)
|
||||
|
||||
monkeypatch.setenv("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH", str(hf_config_dir))
|
||||
return str(model_dir)
|
||||
|
||||
|
||||
def test_prepare_voxcpm_hf_config_dir(tmp_path: Path):
|
||||
model_dir = tmp_path / "model"
|
||||
model_dir.mkdir()
|
||||
(model_dir / "config.json").write_text(json.dumps({"hidden_size": 1024}), encoding="utf-8")
|
||||
(model_dir / "generation_config.json").write_text(json.dumps({"do_sample": False}), encoding="utf-8")
|
||||
|
||||
hf_config_dir = prepare_voxcpm_hf_config_dir(model_dir, tmp_path / "voxcpm_hf_config")
|
||||
|
||||
prepared_config = json.loads((hf_config_dir / "config.json").read_text(encoding="utf-8"))
|
||||
assert prepared_config["model_type"] == "voxcpm"
|
||||
assert prepared_config["architectures"] == ["VoxCPMForConditionalGeneration"]
|
||||
assert (hf_config_dir / "generation_config.json").exists()
|
||||
|
||||
|
||||
def test_resolve_voxcpm_model_dir_local_path(tmp_path: Path):
|
||||
model_dir = tmp_path / "OpenBMB" / "VoxCPM1.5"
|
||||
model_dir.mkdir(parents=True)
|
||||
|
||||
assert resolve_voxcpm_model_dir(str(model_dir)) == model_dir
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm_zero_shot_001(voxcpm_model_path: str):
|
||||
with OmniRunner(voxcpm_model_path, stage_configs_path=STAGE_CONFIG) as runner:
|
||||
outputs = list(runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
|
||||
multimodal_output = _extract_final_multimodal_output(outputs)
|
||||
audio = _extract_audio_tensor(multimodal_output)
|
||||
assert audio.numel() > SAMPLE_RATE // 2, f"Audio too short: {audio.numel()} samples"
|
||||
|
||||
duration_s = audio.shape[0] / SAMPLE_RATE
|
||||
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
|
||||
|
||||
peak = float(torch.max(torch.abs(audio)).item()) if audio.numel() > 0 else 0.0
|
||||
assert peak > 0.01, "Generated audio appears to be silence"
|
||||
|
||||
audio_np = audio.numpy()
|
||||
rms = float(np.sqrt(np.mean(np.square(audio_np)))) if audio_np.size else 0.0
|
||||
assert rms > 1e-4, "Generated audio RMS too low"
|
||||
@@ -5,27 +5,19 @@ import os
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
VOXCPM2_MODEL = "openbmb/VoxCPM2"
|
||||
STAGE_CONFIG = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"vllm_omni",
|
||||
"model_executor",
|
||||
"stage_configs",
|
||||
"voxcpm2.yaml",
|
||||
)
|
||||
DEPLOY_CONFIG = get_deploy_config_path("voxcpm2.yaml")
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def voxcpm2_engine():
|
||||
"""Create VoxCPM2 engine for testing."""
|
||||
with OmniRunner(VOXCPM2_MODEL, stage_configs_path=STAGE_CONFIG) as runner:
|
||||
with OmniRunner(VOXCPM2_MODEL, stage_configs_path=DEPLOY_CONFIG) as runner:
|
||||
yield runner.omni
|
||||
|
||||
|
||||
@@ -33,14 +25,16 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor:
|
||||
"""Extract the final complete audio tensor from multimodal output."""
|
||||
assert isinstance(multimodal_output, dict), f"Expected dict, got {type(multimodal_output)}"
|
||||
|
||||
# Output processor accumulates per-step full audio under "audio".
|
||||
audio = multimodal_output.get("audio") or multimodal_output.get("model_outputs")
|
||||
# Output processor accumulates per-step audio chunks under "audio".
|
||||
audio = multimodal_output.get("audio")
|
||||
if audio is None:
|
||||
audio = multimodal_output.get("model_outputs")
|
||||
assert audio is not None, f"No audio key, got {list(multimodal_output.keys())}"
|
||||
|
||||
if isinstance(audio, list):
|
||||
valid = [x for x in audio if isinstance(x, torch.Tensor) and x.numel() > 100]
|
||||
valid = [torch.as_tensor(x).float().cpu().reshape(-1) for x in audio if x is not None]
|
||||
assert valid, "No valid audio tensors in output list"
|
||||
audio = valid[-1]
|
||||
audio = torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
||||
|
||||
assert isinstance(audio, torch.Tensor), f"Expected Tensor, got {type(audio)}"
|
||||
return audio
|
||||
@@ -98,3 +92,31 @@ def test_voxcpm2_voice_clone_002(voxcpm2_engine):
|
||||
audio = _extract_audio(outputs[0].outputs[0].multimodal_output)
|
||||
duration_s = audio.shape[0] / SAMPLE_RATE
|
||||
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_prefill_decode_mixed_batch_003(voxcpm2_engine):
|
||||
"""Regression: prefill+decode mixed batch must not crash (PR #2903)."""
|
||||
long_prompt = (
|
||||
"This is a deliberately long prompt that will stay in the decode "
|
||||
"phase for many steps so that subsequent shorter prompts keep "
|
||||
"entering prefill alongside it, reproducing the prefill plus "
|
||||
"decode mixed batch scheduling pattern."
|
||||
)
|
||||
short_prompts = [
|
||||
"Hello one.",
|
||||
"Hello two.",
|
||||
"Hello three.",
|
||||
"Hello four.",
|
||||
]
|
||||
requests = [{"prompt": long_prompt}] + [{"prompt": p} for p in short_prompts]
|
||||
|
||||
outputs = voxcpm2_engine.generate(requests)
|
||||
assert len(outputs) == len(requests)
|
||||
|
||||
for i, out in enumerate(outputs):
|
||||
audio = _extract_audio(out.outputs[0].multimodal_output)
|
||||
duration_s = audio.shape[0] / SAMPLE_RATE
|
||||
assert 0.1 < duration_s < 30.0, f"Request {i} audio duration out of range: {duration_s:.2f}s"
|
||||
@@ -29,14 +29,13 @@ from mistral_common.protocol.speech.request import SpeechRequest
|
||||
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.conftest import OmniRunner, modify_stage_config
|
||||
from tests.utils import hardware_test
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
MODEL = "mistralai/Voxtral-4B-TTS-2603"
|
||||
STAGE_CONFIG = str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxtral_tts.yaml"
|
||||
)
|
||||
STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml")
|
||||
SAMPLE_RATE = 24000
|
||||
# Minimum expected audio samples for a short sentence (~0.04s of 24kHz audio)
|
||||
MIN_AUDIO_SAMPLES = 1000
|
||||
@@ -65,9 +64,9 @@ def _resolve_stage_config(run_level: str) -> str:
|
||||
return modify_stage_config(
|
||||
STAGE_CONFIG,
|
||||
deletes={
|
||||
"stage_args": {
|
||||
0: ["engine_args.load_format"],
|
||||
1: ["engine_args.load_format"],
|
||||
"stages": {
|
||||
0: ["load_format"],
|
||||
1: ["load_format"],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
@@ -20,8 +20,9 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
@@ -213,7 +214,10 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu():
|
||||
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA and ROCm for now.")
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack, suppress
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import cloudpickle
|
||||
from typing_extensions import ParamSpec
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
VLLM_PATH = Path(__file__).parent.parent.parent
|
||||
"""Path to root of the vLLM repository."""
|
||||
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
|
||||
def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, None]:
|
||||
"""Decorator to fork a new process for each test function.
|
||||
See https://github.com/vllm-project/vllm/issues/7053 for more details.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||
# Make the process the leader of its own process group
|
||||
# to avoid sending SIGTERM to the parent process
|
||||
os.setpgrp()
|
||||
from _pytest.outcomes import Skipped
|
||||
|
||||
# Create a unique temporary file to store exception info from child
|
||||
# process. Use test function name and process ID to avoid collisions.
|
||||
with (
|
||||
tempfile.NamedTemporaryFile(
|
||||
delete=False,
|
||||
mode="w+b",
|
||||
prefix=f"vllm_test_{func.__name__}_{os.getpid()}_",
|
||||
suffix=".exc",
|
||||
) as exc_file,
|
||||
ExitStack() as delete_after,
|
||||
):
|
||||
exc_file_path = exc_file.name
|
||||
delete_after.callback(os.remove, exc_file_path)
|
||||
|
||||
pid = os.fork()
|
||||
print(f"Fork a new process to run a test {pid}")
|
||||
if pid == 0:
|
||||
# Parent process responsible for deleting, don't delete
|
||||
# in child.
|
||||
delete_after.pop_all()
|
||||
try:
|
||||
func(*args, **kwargs)
|
||||
except Skipped as e:
|
||||
# convert Skipped to exit code 0
|
||||
print(str(e))
|
||||
os._exit(0)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
tb_string = traceback.format_exc()
|
||||
|
||||
# Try to serialize the exception object first
|
||||
exc_to_serialize: dict[str, Any]
|
||||
try:
|
||||
# First, try to pickle the actual exception with
|
||||
# its traceback.
|
||||
exc_to_serialize = {"pickled_exception": e}
|
||||
# Test if it can be pickled
|
||||
cloudpickle.dumps(exc_to_serialize)
|
||||
except (Exception, KeyboardInterrupt):
|
||||
# Fall back to string-based approach.
|
||||
exc_to_serialize = {
|
||||
"exception_type": type(e).__name__,
|
||||
"exception_msg": str(e),
|
||||
"traceback": tb_string,
|
||||
}
|
||||
try:
|
||||
with open(exc_file_path, "wb") as f:
|
||||
cloudpickle.dump(exc_to_serialize, f)
|
||||
except Exception:
|
||||
# Fallback: just print the traceback.
|
||||
print(tb_string)
|
||||
os._exit(1)
|
||||
else:
|
||||
os._exit(0)
|
||||
else:
|
||||
pgid = os.getpgid(pid)
|
||||
_pid, _exitcode = os.waitpid(pid, 0)
|
||||
# ignore SIGTERM signal itself
|
||||
old_signal_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
# kill all child processes
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
# restore the signal handler
|
||||
signal.signal(signal.SIGTERM, old_signal_handler)
|
||||
if _exitcode != 0:
|
||||
# Try to read the exception from the child process
|
||||
exc_info = {}
|
||||
if os.path.exists(exc_file_path):
|
||||
with (
|
||||
contextlib.suppress(Exception),
|
||||
open(exc_file_path, "rb") as f,
|
||||
):
|
||||
exc_info = cloudpickle.load(f)
|
||||
|
||||
original_exception = exc_info.get("pickled_exception")
|
||||
if original_exception is not None and isinstance(original_exception, Exception):
|
||||
# Re-raise the actual exception object if it was
|
||||
# successfully pickled.
|
||||
raise original_exception
|
||||
|
||||
if (original_tb := exc_info.get("traceback")) is not None:
|
||||
# Use string-based traceback for fallback case
|
||||
raise AssertionError(
|
||||
f"Test {func.__name__} failed when called with"
|
||||
f" args {args} and kwargs {kwargs}"
|
||||
f" (exit code: {_exitcode}):\n{original_tb}"
|
||||
) from None
|
||||
|
||||
# Fallback to the original generic error
|
||||
raise AssertionError(
|
||||
f"function {func.__name__} failed when called with"
|
||||
f" args {args} and kwargs {kwargs}"
|
||||
f" (exit code: {_exitcode})"
|
||||
) from None
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
|
||||
"""Decorator to spawn a new process for each test function."""
|
||||
|
||||
@functools.wraps(f)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||
# Check if we're already in a subprocess
|
||||
if os.environ.get("RUNNING_IN_SUBPROCESS") == "1":
|
||||
# If we are, just run the function directly
|
||||
return f(*args, **kwargs)
|
||||
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
with suppress(RuntimeError):
|
||||
mp.set_start_method("spawn")
|
||||
|
||||
# Get the module
|
||||
module_name = f.__module__
|
||||
|
||||
# Create a process with environment variable set
|
||||
env = os.environ.copy()
|
||||
env["RUNNING_IN_SUBPROCESS"] = "1"
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
output_filepath = os.path.join(tempdir, "new_process.tmp")
|
||||
|
||||
# `cloudpickle` allows pickling complex functions directly
|
||||
input_bytes = cloudpickle.dumps((f, output_filepath))
|
||||
|
||||
repo_root = str(VLLM_PATH.resolve())
|
||||
|
||||
env = dict(env or os.environ)
|
||||
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
|
||||
|
||||
cmd = [sys.executable, "-m", f"{module_name}"]
|
||||
|
||||
returned = subprocess.run(cmd, input=input_bytes, capture_output=True, env=env)
|
||||
|
||||
# check if the subprocess is successful
|
||||
try:
|
||||
returned.check_returncode()
|
||||
except Exception as e:
|
||||
# wrap raised exception to provide more information
|
||||
raise RuntimeError(f"Error raised in subprocess:\n{returned.stderr.decode()}") from e
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def create_new_process_for_each_test(
|
||||
method: Literal["spawn", "fork"] | None = None,
|
||||
) -> Callable[[Callable[_P, None]], Callable[_P, None]]:
|
||||
"""Creates a decorator that runs each test function in a new process.
|
||||
|
||||
Args:
|
||||
method: The process creation method. Can be either "spawn" or "fork".
|
||||
If not specified, it defaults to "spawn" on ROCm and XPU
|
||||
platforms and "fork" otherwise.
|
||||
|
||||
Returns:
|
||||
A decorator to run test functions in separate processes.
|
||||
"""
|
||||
if method is None:
|
||||
# TODO: Find out why spawn is not working correctly on ROCm
|
||||
# The test content will not run and tests passed immediately.
|
||||
# For now, using `fork` for ROCm as it can run with `fork`
|
||||
# and tests are running correctly.
|
||||
use_spawn = current_platform.is_xpu()
|
||||
method = "spawn" if use_spawn else "fork"
|
||||
|
||||
assert method in ["spawn", "fork"], "Method must be either 'spawn' or 'fork'"
|
||||
|
||||
if method == "fork":
|
||||
return fork_new_process_for_each_test
|
||||
|
||||
return spawn_new_process_for_each_test
|
||||
Reference in new issue
Block a user