Merge branch 'main' into dev/vllm-align
This commit is contained in:
commit
d3db476f2d
144 files changed
+17247
-1024
No files matched your search
@@ -3,13 +3,16 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServerParams
|
||||
from tests.utils import hardware_test
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
stage_configs = [str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen2_5_omni_ci.yaml")]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
|
||||
@@ -63,6 +63,12 @@ def _setup_comfyui_test_environment():
|
||||
mock_comfy_api_input.AudioInput = AudioInput
|
||||
mock_comfy_api_input.VideoInput = VideoInput
|
||||
mock_comfy_api.input = mock_comfy_api_input
|
||||
mock_comfy_api_latest = MagicMock()
|
||||
mock_comfy_api_latest.Types.VideoComponents = MagicMock(side_effect=lambda **kwargs: kwargs)
|
||||
mock_comfy_api_latest.InputImpl.VideoFromComponents = MagicMock(
|
||||
side_effect=lambda _: VideoInput(b"mock_video_from_components")
|
||||
)
|
||||
mock_comfy_api.latest = mock_comfy_api_latest
|
||||
|
||||
def mock_load(_: str | BinaryIO):
|
||||
"""Mock nodes_audio.load that returns a waveform tensor (channels, samples) and sample rate."""
|
||||
@@ -78,5 +84,6 @@ def _setup_comfyui_test_environment():
|
||||
# Install mock modules BEFORE importing any comfyui_vllm_omni code
|
||||
sys.modules["comfy_api"] = mock_comfy_api
|
||||
sys.modules["comfy_api.input"] = mock_comfy_api_input
|
||||
sys.modules["comfy_api.latest"] = mock_comfy_api_latest
|
||||
sys.modules["comfy_extras"] = mock_comfy_extras
|
||||
sys.modules["comfy_extras.nodes_audio"] = mock_nodes_audio
|
||||
@@ -20,11 +20,12 @@ import torch
|
||||
from comfy_api.input import AudioInput, VideoInput
|
||||
from comfyui_vllm_omni.nodes import (
|
||||
VLLMOmniGenerateImage,
|
||||
VLLMOmniGenerateVideo,
|
||||
VLLMOmniTTS,
|
||||
VLLMOmniUnderstanding,
|
||||
VLLMOmniVoiceClone,
|
||||
)
|
||||
from comfyui_vllm_omni.utils.types import AutoregressionSamplingParams, DiffusionSamplingParams
|
||||
from comfyui_vllm_omni.utils.types import AutoregressionSamplingParams, DiffusionSamplingParams, WanModelSpecificParams
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
@@ -34,6 +35,8 @@ from vllm_omni.entrypoints.cli.serve import OmniServeCommand
|
||||
from vllm_omni.inputs.data import OmniSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class ServerCase(NamedTuple):
|
||||
"""Parametrizing the model to serve."""
|
||||
@@ -49,6 +52,7 @@ class SamplingCase(NamedTuple):
|
||||
|
||||
kind: "SamplingKind"
|
||||
sampling_params: dict | list[dict] | None
|
||||
lora: dict | None = None
|
||||
|
||||
|
||||
class SamplingKind(StrEnum):
|
||||
@@ -58,11 +62,17 @@ class SamplingKind(StrEnum):
|
||||
UNDERSTANDING_AR_LIST = auto()
|
||||
TTS_NONE = auto()
|
||||
TTS_DIFFUSION_SINGLE = auto()
|
||||
VIDEO_NONE = auto()
|
||||
VIDEO_DIFFUSION_SINGLE = auto()
|
||||
|
||||
|
||||
# Pre-defined arguments to be used in function calls during the tests
|
||||
IMAGE_WIDTH = 64
|
||||
IMAGE_HEIGHT = 64
|
||||
VIDEO_WIDTH = 32
|
||||
VIDEO_HEIGHT = 32
|
||||
VIDEO_FPS = 8
|
||||
VIDEO_NUM_FRAMES = 5
|
||||
DIFFUSION_SINGLE_SAMPLING_PARAMS = DiffusionSamplingParams(
|
||||
{
|
||||
"n": 2,
|
||||
@@ -102,6 +112,10 @@ AR_LIST_SAMPLING_PARAMS = [
|
||||
),
|
||||
]
|
||||
|
||||
VIDEO_MODEL_PARAMS = WanModelSpecificParams({"guidance_scale_2": 5.0, "boundary_ratio": 0.98, "flow_shift": 12.0})
|
||||
|
||||
LORA_PARAMS = {"local_path": "test_lora_path", "name": "test_name", "scale": 0.7, "int_id": 10}
|
||||
|
||||
|
||||
def _build_image_output(size: tuple[int, int] = (IMAGE_WIDTH, IMAGE_HEIGHT), color: str = "red") -> Image.Image:
|
||||
return Image.new("RGB", size, color=color)
|
||||
@@ -181,6 +195,16 @@ def _build_diffusion_image_output_for_images_endpoint() -> OmniRequestOutput:
|
||||
)
|
||||
|
||||
|
||||
def _build_diffusion_video_output() -> OmniRequestOutput:
|
||||
# Small video: VIDEO_NUM_FRAMES frames of (VIDEO_HEIGHT x VIDEO_WIDTH) RGB, shape (F, H, W, C)
|
||||
video_frames = torch.zeros((VIDEO_NUM_FRAMES, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
|
||||
return OmniRequestOutput.from_diffusion(
|
||||
request_id="test_req_video",
|
||||
images=[video_frames],
|
||||
final_output_type="video",
|
||||
)
|
||||
|
||||
|
||||
def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
|
||||
request_output = MagicMock()
|
||||
request_output.images = [_build_image_output(color="blue")]
|
||||
@@ -193,12 +217,40 @@ def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
|
||||
)
|
||||
|
||||
|
||||
def _assert_sampling_param_values(received: OmniSamplingParams, expected: dict[str, Any]):
|
||||
def _assert_sampling_param_values(
|
||||
received: OmniSamplingParams, expected: dict[str, Any], expected_lora: dict | None = None
|
||||
):
|
||||
for key, expected_value in expected.items():
|
||||
actual_value = getattr(received, key, None)
|
||||
actual_value = getattr(received, key)
|
||||
assert actual_value == expected_value, (
|
||||
f"Expected sampling param '{key}'={expected_value}, got {actual_value}. The received sampling params: {received}"
|
||||
)
|
||||
if expected_lora:
|
||||
assert received.lora_request.lora_name == expected_lora["name"], (
|
||||
f"Expected lora name={(expected_lora['name'])}, got {received.lora_request.lora_name}. The received sampling params: {received}"
|
||||
)
|
||||
assert received.lora_request.lora_int_id == expected_lora["int_id"], (
|
||||
f"Expected lora int_id={expected_lora['int_id']}, got {received.lora_request.lora_int_id}. The received sampling params: {received}"
|
||||
)
|
||||
assert received.lora_request.lora_path == expected_lora["local_path"], (
|
||||
f"Expected lora path={expected_lora['local_path']}, got {received.lora_request.lora_path}. The received sampling params: {received}"
|
||||
)
|
||||
assert received.lora_scale == expected_lora["scale"], (
|
||||
f"Expected lora scale={expected_lora['scale']}, got {received.lora_scale}. The received sampling params: {received}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_model_param_values(received: OmniSamplingParams, expected: dict):
|
||||
for key, expected_value in expected.items():
|
||||
try:
|
||||
actual_value = getattr(received, key)
|
||||
expected_param_name = key
|
||||
except AttributeError:
|
||||
actual_value = received.extra_args.get(key, None)
|
||||
expected_param_name = f'extra_args["{key}"]'
|
||||
assert actual_value == expected_value, (
|
||||
f"Expected model param '{expected_param_name}'={expected_value}, got {actual_value}. The received sampling params: {received}"
|
||||
)
|
||||
|
||||
|
||||
def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: SamplingCase, server_case: ServerCase):
|
||||
@@ -235,6 +287,7 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam
|
||||
"height": IMAGE_HEIGHT,
|
||||
**expected,
|
||||
},
|
||||
LORA_PARAMS,
|
||||
)
|
||||
elif sampling_case.kind is SamplingKind.UNDERSTANDING_NONE:
|
||||
assert len(received_sampling_params_list) == 3
|
||||
@@ -244,6 +297,33 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam
|
||||
_assert_sampling_param_values(received_sampling_params_list[i], expected)
|
||||
elif sampling_case.kind in {SamplingKind.TTS_NONE, SamplingKind.TTS_DIFFUSION_SINGLE}:
|
||||
assert len(received_sampling_params_list) == 1
|
||||
elif sampling_case.kind is SamplingKind.VIDEO_NONE:
|
||||
assert len(received_sampling_params_list) == 1
|
||||
_assert_sampling_param_values(
|
||||
received_sampling_params_list[0],
|
||||
{
|
||||
"width": VIDEO_WIDTH,
|
||||
"height": VIDEO_HEIGHT,
|
||||
"num_frames": VIDEO_NUM_FRAMES,
|
||||
"fps": VIDEO_FPS,
|
||||
},
|
||||
)
|
||||
elif sampling_case.kind is SamplingKind.VIDEO_DIFFUSION_SINGLE:
|
||||
assert len(received_sampling_params_list) == 1
|
||||
expected = DIFFUSION_SINGLE_SAMPLING_PARAMS.copy()
|
||||
expected["num_outputs_per_prompt"] = expected.pop("n") # convert from n to num_outputs_per_prompt
|
||||
_assert_sampling_param_values(
|
||||
received_sampling_params_list[0],
|
||||
{
|
||||
"width": VIDEO_WIDTH,
|
||||
"height": VIDEO_HEIGHT,
|
||||
"num_frames": VIDEO_NUM_FRAMES,
|
||||
"fps": VIDEO_FPS,
|
||||
**expected,
|
||||
},
|
||||
LORA_PARAMS,
|
||||
)
|
||||
_assert_model_param_values(received_sampling_params_list[0], VIDEO_MODEL_PARAMS)
|
||||
else:
|
||||
raise AssertionError(f"Unknown sampling case: {sampling_case.kind}")
|
||||
|
||||
@@ -401,9 +481,15 @@ def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni
|
||||
@pytest.mark.parametrize(
|
||||
"sampling_case",
|
||||
[
|
||||
pytest.param(SamplingCase(kind=SamplingKind.IMAGE_NONE, sampling_params=None), id="no-sampling-params"),
|
||||
pytest.param(
|
||||
SamplingCase(kind=SamplingKind.IMAGE_DIFFUSION_SINGLE, sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS),
|
||||
SamplingCase(kind=SamplingKind.IMAGE_NONE, sampling_params=None, lora=None), id="no-sampling-params"
|
||||
),
|
||||
pytest.param(
|
||||
SamplingCase(
|
||||
kind=SamplingKind.IMAGE_DIFFUSION_SINGLE,
|
||||
sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS,
|
||||
lora=LORA_PARAMS,
|
||||
),
|
||||
id="single-diffusion-sampling-params",
|
||||
),
|
||||
],
|
||||
@@ -423,7 +509,8 @@ async def test_image_generation_node(api_server: str, model: str, image_input: b
|
||||
kwargs["image"] = torch.zeros((1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), dtype=torch.float32)
|
||||
if sampling_case.sampling_params is not None:
|
||||
kwargs["sampling_params"] = sampling_case.sampling_params
|
||||
print(f"!!!!!! Calling {model} node.generate with kwargs: {sampling_case.sampling_params}")
|
||||
if sampling_case.lora:
|
||||
kwargs["lora"] = sampling_case.lora
|
||||
|
||||
result = await node.generate(**kwargs)
|
||||
|
||||
@@ -565,3 +652,75 @@ async def test_tts_nodes(api_server: str, node_cls, call_kwargs: dict, sampling_
|
||||
assert result[0]["sample_rate"] == 24000
|
||||
assert isinstance(result[0]["waveform"], torch.Tensor)
|
||||
assert result[0]["waveform"].shape == (1, 1, 24000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server_case,model,image_input",
|
||||
[
|
||||
pytest.param(
|
||||
ServerCase(
|
||||
served_model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||
stage_list=["diffusion"],
|
||||
stage_configs=[{"stage_type": "diffusion"}],
|
||||
outputs=[_build_diffusion_video_output()],
|
||||
),
|
||||
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||
False,
|
||||
id="text-to-video",
|
||||
),
|
||||
pytest.param(
|
||||
ServerCase(
|
||||
served_model="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
stage_list=["diffusion"],
|
||||
stage_configs=[{"stage_type": "diffusion"}],
|
||||
outputs=[_build_diffusion_video_output()],
|
||||
),
|
||||
"Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
True,
|
||||
id="image-to-video",
|
||||
),
|
||||
],
|
||||
indirect=["server_case"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"sampling_case",
|
||||
[
|
||||
pytest.param(SamplingCase(kind=SamplingKind.VIDEO_NONE, sampling_params=None), id="no-sampling-params"),
|
||||
pytest.param(
|
||||
SamplingCase(
|
||||
kind=SamplingKind.VIDEO_DIFFUSION_SINGLE,
|
||||
sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS,
|
||||
lora=LORA_PARAMS,
|
||||
),
|
||||
id="single-diffusion-sampling-params",
|
||||
),
|
||||
],
|
||||
indirect=["sampling_case"],
|
||||
)
|
||||
async def test_video_generation_node(api_server: str, model: str, image_input: bool, sampling_case: SamplingCase):
|
||||
node = VLLMOmniGenerateVideo()
|
||||
|
||||
kwargs = {
|
||||
"url": api_server,
|
||||
"model": model,
|
||||
"prompt": "A beautiful sunset timelapse",
|
||||
"negative_prompt": "",
|
||||
"width": VIDEO_WIDTH,
|
||||
"height": VIDEO_HEIGHT,
|
||||
"fps": VIDEO_FPS,
|
||||
"num_frames": VIDEO_NUM_FRAMES,
|
||||
"model_params": VIDEO_MODEL_PARAMS,
|
||||
}
|
||||
if image_input:
|
||||
kwargs["image"] = torch.zeros((1, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
|
||||
if sampling_case.sampling_params is not None:
|
||||
kwargs["sampling_params"] = sampling_case.sampling_params
|
||||
if sampling_case.lora:
|
||||
kwargs["lora"] = sampling_case.lora
|
||||
|
||||
result = await node.generate(**kwargs)
|
||||
|
||||
assert isinstance(result, tuple)
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0], VideoInput)
|
||||
+244
-19
@@ -18,16 +18,21 @@ import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import imageio.v3 as iio
|
||||
import numpy as np
|
||||
import psutil
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import yaml
|
||||
from openai import OpenAI
|
||||
from openai import OpenAI, omit
|
||||
from PIL import Image
|
||||
from vllm import TextPrompt
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
from vllm.logger import init_logger
|
||||
@@ -44,6 +49,65 @@ PromptImageInput = list[Any] | Any | None
|
||||
PromptVideoInput = list[Any] | Any | None
|
||||
|
||||
|
||||
class OmniServerParams(NamedTuple):
|
||||
model: str
|
||||
port: int | None = None
|
||||
stage_config_path: str | None = None
|
||||
server_args: list[str] | None = None
|
||||
|
||||
|
||||
def assert_image_valid(image: Path | Image.Image, *, width: int | None = None, height: int | None = None):
|
||||
"""Assert the file is a loadable image with optional exact dimensions."""
|
||||
if isinstance(image, Path):
|
||||
assert image.exists(), f"Image not found: {image}"
|
||||
image = Image.open(image)
|
||||
image.load()
|
||||
assert image.width > 0 and image.height > 0
|
||||
if width is not None:
|
||||
assert image.width == width, f"Expected width={width}, got {image.width} in {image.name}"
|
||||
if height is not None:
|
||||
assert image.height == height, f"Expected height={height}, got {image.height} in {image.name}"
|
||||
return image
|
||||
|
||||
|
||||
def assert_video_valid(frames: Path | np.ndarray, *, width: int, height: int, num_frames: int) -> None:
|
||||
"""Assert the MP4 has the expected resolution and exact frame count."""
|
||||
if isinstance(frames, Path):
|
||||
assert frames.exists(), f"Video not found: {frames}"
|
||||
frames = iio.imread(str(frames), plugin="pyav", index=None)
|
||||
assert frames.shape[0] == num_frames, f"Expected {num_frames} frames, got {frames.shape[0]}"
|
||||
assert frames.shape[1] == height, f"Expected height={height}, got {frames.shape[1]}"
|
||||
assert frames.shape[2] == width, f"Expected width={width}, got {frames.shape[2]}"
|
||||
|
||||
|
||||
def assert_audio_valid(path: Path, *, sample_rate: int, channels: int, duration_s: float) -> None:
|
||||
"""Assert the WAV has the expected sample rate, channel count, and duration."""
|
||||
|
||||
assert path.exists(), f"Audio not found: {path}"
|
||||
info = sf.info(str(path))
|
||||
assert info.samplerate == sample_rate, f"Expected sample_rate={sample_rate}, got {info.samplerate}"
|
||||
assert info.channels == channels, f"Expected {channels} channel(s), got {info.channels}"
|
||||
expected_frames = int(duration_s * sample_rate)
|
||||
assert info.frames == expected_frames, (
|
||||
f"Expected {expected_frames} frames ({duration_s}s @ {sample_rate} Hz), got {info.frames}"
|
||||
)
|
||||
|
||||
|
||||
def decode_b64_image(b64: str):
|
||||
img = Image.open(BytesIO(base64.b64decode(b64)))
|
||||
img.load()
|
||||
return img
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def model_prefix() -> str:
|
||||
"""Optional model-path prefix from MODEL_PREFIX env var.
|
||||
Useful if models are downloaded to non-default local directories.
|
||||
"""
|
||||
prefix = os.environ.get("MODEL_PREFIX", "")
|
||||
return f"{prefix.rstrip('/')}/" if prefix else ""
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def default_vllm_config():
|
||||
"""Set a default VllmConfig for all tests.
|
||||
@@ -1151,7 +1215,9 @@ def pytest_addoption(parser):
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def run_level(request):
|
||||
def run_level(request) -> str:
|
||||
"""A command-line argument that specifies the level of tests to run in this session.
|
||||
See https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/CI_5levels/"""
|
||||
return request.config.getoption("--run-level")
|
||||
|
||||
|
||||
@@ -1159,15 +1225,17 @@ _omni_server_lock = threading.Lock()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server(request, run_level):
|
||||
def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: str) -> Generator[OmniServer, Any, None]:
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
||||
Uses session scope so the server starts only once for the entire test session.
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
*port, model, stage_config_path = request.param
|
||||
port = port[0] if port else None
|
||||
if run_level == "advanced_model":
|
||||
params: OmniServerParams = request.param
|
||||
model = model_prefix + params.model
|
||||
port = params.port
|
||||
stage_config_path = params.stage_config_path
|
||||
if run_level == "advanced_model" and stage_config_path is not None:
|
||||
stage_config_path = modify_stage_config(
|
||||
stage_config_path,
|
||||
deletes={
|
||||
@@ -1179,7 +1247,11 @@ def omni_server(request, run_level):
|
||||
},
|
||||
)
|
||||
|
||||
server_args = ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]
|
||||
server_args = params.server_args or []
|
||||
server_args = ["--stage-init-timeout", "120", *server_args]
|
||||
if stage_config_path is not None:
|
||||
server_args += ["--stage-configs-path", stage_config_path]
|
||||
|
||||
with OmniServer(model, server_args, port=port) if port else OmniServer(model, server_args) as server:
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
@@ -1199,6 +1271,17 @@ class OmniResponse:
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiffusionResponse:
|
||||
text_content: str | None = None
|
||||
images: list[Image.Image] | None = None
|
||||
audios: list[Any] | None = None
|
||||
videos: list[Any] | None = None
|
||||
e2e_latency: float | None = None
|
||||
success: bool = False
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
def assert_omni_response(response: OmniResponse, request_config: dict[str, Any], run_level):
|
||||
"""
|
||||
Validate response results.
|
||||
@@ -1247,6 +1330,51 @@ def assert_omni_response(response: OmniResponse, request_config: dict[str, Any],
|
||||
print(f"similarity is: {response.similarity}")
|
||||
|
||||
|
||||
def assert_diffusion_response(response: DiffusionResponse, request_config: dict[str, Any], run_level: str = None):
|
||||
"""
|
||||
Validate diffusion response results.
|
||||
|
||||
Args:
|
||||
response: DiffusionResponse object. Any not-None content will be validated based on the request_config.
|
||||
request_config: Request configuration dictionary containing parameters like model, messages, extra_body.
|
||||
When validating a certain modality, the corresponding params in request_config['extra_body'] must present.
|
||||
It will be used to check against the multimedia file in the response.
|
||||
run_level: Test run level (e.g., "core_model", "advanced_model")
|
||||
|
||||
Raises:
|
||||
AssertionError: When the response does not meet validation criteria
|
||||
KeyError: When the request_config does not contain necessary parameters for validation
|
||||
"""
|
||||
assert response.success, "The request failed."
|
||||
|
||||
e2e_latency = response.e2e_latency
|
||||
if e2e_latency is not None:
|
||||
print(f"the avg e2e is: {e2e_latency}")
|
||||
|
||||
extra_body = request_config.get("extra_body", {})
|
||||
|
||||
num_outputs_per_prompt = extra_body.get("num_outputs_per_prompt", 1)
|
||||
|
||||
if response.images is not None:
|
||||
assert len(response.images) > 0, "No images in response"
|
||||
assert len(response.images) == num_outputs_per_prompt, (
|
||||
f"Expected {num_outputs_per_prompt} images, got {len(response.images)}"
|
||||
)
|
||||
if run_level == "advanced_model":
|
||||
expected_width = extra_body["width"] # intend to raise KeyError
|
||||
expected_height = extra_body["height"] # intend to raise KeyError
|
||||
for img in response.images:
|
||||
assert_image_valid(img, width=expected_width, height=expected_height)
|
||||
if response.videos is not None:
|
||||
raise NotImplementedError(
|
||||
"Video validation is not implemented yet"
|
||||
) # consider using assert_video_valid defined above
|
||||
if response.audios is not None:
|
||||
raise NotImplementedError(
|
||||
"Audio validation is not implemented yet"
|
||||
) # consider using assert_audio_valid defined above
|
||||
|
||||
|
||||
class OpenAIClientHandler:
|
||||
"""
|
||||
OpenAI client handler class, encapsulating both streaming and non-streaming response processing logic.
|
||||
@@ -1269,7 +1397,7 @@ class OpenAIClientHandler:
|
||||
self.client = OpenAI(base_url=f"http://{host}:{port}/v1", api_key=api_key)
|
||||
self.run_level = run_level
|
||||
|
||||
def _process_stream_response(self, chat_completion) -> OmniResponse:
|
||||
def _process_stream_omni_response(self, chat_completion) -> OmniResponse:
|
||||
"""
|
||||
Process streaming responses.
|
||||
|
||||
@@ -1330,7 +1458,7 @@ class OpenAIClientHandler:
|
||||
|
||||
return result
|
||||
|
||||
def _process_non_stream_response(self, chat_completion) -> OmniResponse:
|
||||
def _process_non_stream_omni_response(self, chat_completion) -> OmniResponse:
|
||||
"""
|
||||
Process non-streaming responses.
|
||||
|
||||
@@ -1384,7 +1512,46 @@ class OpenAIClientHandler:
|
||||
|
||||
return result
|
||||
|
||||
def send_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]:
|
||||
def _process_diffusion_response(self, chat_completion) -> DiffusionResponse:
|
||||
"""
|
||||
Process diffusion responses (image generation/editing).
|
||||
|
||||
Args:
|
||||
chat_completion: OpenAI response object
|
||||
|
||||
Returns:
|
||||
DiffusionResponse: Processed response object
|
||||
"""
|
||||
result = DiffusionResponse()
|
||||
start_time = time.perf_counter()
|
||||
|
||||
try:
|
||||
images = []
|
||||
# [TODO] reading video and audio output from API response for later validation
|
||||
|
||||
for choice in chat_completion.choices:
|
||||
if hasattr(choice.message, "content") and choice.message.content is not None:
|
||||
content = choice.message.content
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
if hasattr(item, "image_url") and item.image_url is not None:
|
||||
image_url = item.image_url.url
|
||||
if image_url.startswith("data:image"):
|
||||
b64_data = image_url.split(",", 1)[1]
|
||||
img = decode_b64_image(b64_data)
|
||||
images.append(img)
|
||||
|
||||
result.e2e_latency = time.perf_counter() - start_time
|
||||
result.images = images if images else None
|
||||
result.success = True
|
||||
|
||||
except Exception as e:
|
||||
result.error_message = f"Diffusion response processing error: {str(e)}"
|
||||
print(f"Error: {result.error_message}")
|
||||
|
||||
return result
|
||||
|
||||
def send_omni_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]:
|
||||
"""
|
||||
Send OpenAI requests.
|
||||
|
||||
@@ -1410,9 +1577,9 @@ class OpenAIClientHandler:
|
||||
)
|
||||
|
||||
if stream:
|
||||
response = self._process_stream_response(chat_completion)
|
||||
response = self._process_stream_omni_response(chat_completion)
|
||||
else:
|
||||
response = self._process_non_stream_response(chat_completion)
|
||||
response = self._process_non_stream_omni_response(chat_completion)
|
||||
|
||||
assert_omni_response(response, request_config, run_level=self.run_level)
|
||||
responses.append(response)
|
||||
@@ -1438,25 +1605,82 @@ class OpenAIClientHandler:
|
||||
chat_completion = future.result()
|
||||
|
||||
if stream:
|
||||
response = self._process_stream_response(chat_completion)
|
||||
response = self._process_stream_omni_response(chat_completion)
|
||||
else:
|
||||
response = self._process_non_stream_response(chat_completion)
|
||||
response = self._process_non_stream_omni_response(chat_completion)
|
||||
|
||||
assert_omni_response(response, request_config, run_level=self.run_level)
|
||||
responses.append(response)
|
||||
|
||||
return responses
|
||||
|
||||
def send_diffusion_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]:
|
||||
"""
|
||||
Send OpenAI requests for diffusion models.
|
||||
|
||||
Args:
|
||||
request_config: Request configuration dictionary containing parameters like model, messages
|
||||
request_num: Number of requests to send concurrently, defaults to 1 (single request)
|
||||
Returns:
|
||||
List[OmniResponse]: List of response objects
|
||||
"""
|
||||
responses = []
|
||||
stream = request_config.get("stream", False)
|
||||
modalities = request_config.get("modalities", omit) # Most diffusion models don't require modalities param
|
||||
extra_body = request_config.get("extra_body", None)
|
||||
|
||||
if stream:
|
||||
raise NotImplementedError("Streaming is not currently implemented for diffusion model e2e test")
|
||||
|
||||
if request_num == 1:
|
||||
# Send single request
|
||||
chat_completion = self.client.chat.completions.create(
|
||||
model=request_config.get("model"),
|
||||
messages=request_config.get("messages"),
|
||||
extra_body=extra_body,
|
||||
modalities=modalities,
|
||||
)
|
||||
|
||||
response = self._process_diffusion_response(chat_completion)
|
||||
assert_diffusion_response(response, request_config, run_level=self.run_level)
|
||||
responses.append(response)
|
||||
|
||||
else:
|
||||
# Send concurrent requests
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=request_num) as executor:
|
||||
futures = []
|
||||
|
||||
# Submit all request tasks
|
||||
for _ in range(request_num):
|
||||
future = executor.submit(
|
||||
self.client.chat.completions.create,
|
||||
model=request_config.get("model"),
|
||||
messages=request_config.get("messages"),
|
||||
modalities=modalities,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
futures.append(future)
|
||||
|
||||
# Process completed tasks
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
chat_completion = future.result()
|
||||
response = self._process_diffusion_response(chat_completion)
|
||||
assert_diffusion_response(response, request_config, run_level=self.run_level)
|
||||
responses.append(response)
|
||||
|
||||
return responses
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_client(omni_server, run_level):
|
||||
"""Create OpenAIClientHandler fixture"""
|
||||
def openai_client(omni_server: OmniServer, run_level: str):
|
||||
"""Create OpenAIClientHandler fixture to facilitate communication with OmniServer
|
||||
with encapsulated request sending, concurrent requests, response handling, and validation."""
|
||||
return OpenAIClientHandler(host=omni_server.host, port=omni_server.port, api_key="EMPTY", run_level=run_level)
|
||||
|
||||
|
||||
class OmniRunner:
|
||||
"""
|
||||
Test runner for Omni models.
|
||||
Offline test runner for Omni models.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1727,9 +1951,10 @@ class OmniRunner:
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_runner(request):
|
||||
def omni_runner(request, model_prefix):
|
||||
with _omni_server_lock:
|
||||
model, stage_config_path = request.param
|
||||
model = model_prefix + model
|
||||
with OmniRunner(model, seed=42, stage_configs_path=stage_config_path, stage_init_timeout=300) as runner:
|
||||
print("OmniRunner started successfully")
|
||||
yield runner
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.flux2.flux2_transformer import (
|
||||
Flux2PosEmbed,
|
||||
Flux2Transformer2DModel,
|
||||
)
|
||||
|
||||
|
||||
# Initialize TP group before tests
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def setup_tp_group():
|
||||
"""Set up TP group for each test function"""
|
||||
with patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=2):
|
||||
with patch("vllm.distributed.parallel_state.get_tp_group") as mock_get_tp_group:
|
||||
mock_tp_group = MagicMock()
|
||||
mock_tp_group.world_size = 2
|
||||
mock_get_tp_group.return_value = mock_tp_group
|
||||
yield
|
||||
|
||||
|
||||
class TestFlux2TransformerWeightLoading:
|
||||
"""Test Flux2Transformer weight loading functionality"""
|
||||
|
||||
def test_weight_loading_tp2(self, setup_tp_group):
|
||||
"""Verify weights load correctly with TP=2"""
|
||||
# Prepare test data
|
||||
model = Flux2Transformer2DModel(
|
||||
num_layers=2,
|
||||
num_single_layers=1,
|
||||
num_attention_heads=48,
|
||||
attention_head_dim=128,
|
||||
joint_attention_dim=15360,
|
||||
)
|
||||
|
||||
# Mock TP=2 weight loading
|
||||
mock_weights = []
|
||||
|
||||
# 1. Test regular weight loading
|
||||
mock_weights.append(("x_embedder.weight", torch.randn(6144, 128)))
|
||||
mock_weights.append(("context_embedder.weight", torch.randn(6144, 15360)))
|
||||
mock_weights.append(("proj_out.weight", torch.randn(128, 6144)))
|
||||
|
||||
# 2. Test stacked_params_mapping weight loading
|
||||
# Full weights - load_weights handles sharding internally
|
||||
to_qkv_weight = torch.randn(18432, 6144) # Full weights
|
||||
mock_weights.append(("transformer_blocks.0.attn.to_qkv.weight", to_qkv_weight))
|
||||
|
||||
# Add_kv_proj weights
|
||||
add_kv_proj_weight = torch.randn(18432, 6144) # Full weights
|
||||
mock_weights.append(("transformer_blocks.0.attn.add_kv_proj.weight", add_kv_proj_weight))
|
||||
|
||||
# 3. Test single block weight loading
|
||||
single_block_qkv_mlp_weight = torch.randn(18432 + 18432 * 3, 6144) # Full weights
|
||||
mock_weights.append(("single_transformer_blocks.0.attn.to_qkv_mlp_proj.weight", single_block_qkv_mlp_weight))
|
||||
|
||||
# Execute weight loading
|
||||
loaded_params = model.load_weights(mock_weights)
|
||||
|
||||
# Verify
|
||||
assert len(loaded_params) > 0, "Parameters should be loaded"
|
||||
|
||||
# Verify stacked_params_mapping is correctly set
|
||||
assert model.stacked_params_mapping is not None
|
||||
# Should have 6 mappings: 3 for to_qkv + 3 for add_kv_proj
|
||||
assert len(model.stacked_params_mapping) == 6, "Should have 6 mappings"
|
||||
|
||||
# Verify weights are correctly loaded to corresponding layers
|
||||
assert hasattr(model.transformer_blocks[0].attn.to_qkv, "weight")
|
||||
# With TP=2, weight dimension on each GPU should be 18432/2 = 9216
|
||||
assert model.transformer_blocks[0].attn.to_qkv.weight.shape == (9216, 6144), (
|
||||
f"With TP=2, to_qkv weight dimension should be (9216, 6144), got {model.transformer_blocks[0].attn.to_qkv.weight.shape}"
|
||||
)
|
||||
|
||||
|
||||
class TestFlux2RopePositionEmbedding:
|
||||
"""Test Flux2 RoPE position embedding functionality"""
|
||||
|
||||
def test_rope_position_embedding(self):
|
||||
"""Verify RoPE produces correct embeddings for 4D coordinates"""
|
||||
# Prepare test data - use model default configuration
|
||||
# axes_dims_rope default is (32, 32, 32, 32)
|
||||
# get_1d_rotary_pos_embed outputs half the dimension for real/imag parts
|
||||
# So actual output dimension should be (16+16+16+16) = 64
|
||||
axes_dims = (32, 32, 32, 32) # Use model default
|
||||
rope_theta = 2000 # Model default is 2000
|
||||
pos_embed = Flux2PosEmbed(theta=rope_theta, axes_dim=axes_dims)
|
||||
|
||||
# Create test IDs
|
||||
seq_len = 10
|
||||
ids = torch.randint(0, 100, (seq_len, 4)) # [S, 4]
|
||||
|
||||
# Forward pass
|
||||
freqs_cos, freqs_sin = pos_embed(ids)
|
||||
|
||||
# Verify output shape - based on model config, expected dimension is 64
|
||||
# Each axes_dim=32 outputs 16-dim real part, sum of 4 dimensions = 64
|
||||
expected_dim = sum(axes_dims) // 2 # 128/2 = 64
|
||||
assert freqs_cos.shape == (seq_len, expected_dim), (
|
||||
f"Expected shape {(seq_len, expected_dim)}, got {freqs_cos.shape}"
|
||||
)
|
||||
assert freqs_sin.shape == (seq_len, expected_dim), (
|
||||
f"Expected shape {(seq_len, expected_dim)}, got {freqs_sin.shape}"
|
||||
)
|
||||
|
||||
# Verify output type - NPU may use float32, other devices use float64
|
||||
assert freqs_cos.dtype in [torch.float32, torch.float64], "Should be float type"
|
||||
assert freqs_sin.dtype in [torch.float32, torch.float64], "Should be float type"
|
||||
|
||||
# Verify value range
|
||||
assert torch.all(freqs_cos >= -1) and torch.all(freqs_cos <= 1), "cos values should be in [-1, 1]"
|
||||
assert torch.all(freqs_sin >= -1) and torch.all(freqs_sin <= 1), "sin values should be in [-1, 1]"
|
||||
|
||||
# Verify trigonometric relationship: cos^2 + sin^2 ≈ 1
|
||||
cos_sq_sin_sq = freqs_cos**2 + freqs_sin**2
|
||||
assert torch.allclose(cos_sq_sin_sq, torch.ones_like(cos_sq_sin_sq), atol=1e-6), "cos^2 + sin^2 should ≈ 1"
|
||||
|
||||
# Verify different positions produce different embeddings
|
||||
ids_diff = torch.randint(0, 100, (seq_len, 4))
|
||||
freqs_cos_diff, freqs_sin_diff = pos_embed(ids_diff)
|
||||
assert not torch.allclose(freqs_cos, freqs_cos_diff), "Different positions should produce different embeddings"
|
||||
|
||||
# Verify same positions produce same embeddings
|
||||
ids_same = ids.clone()
|
||||
freqs_cos_same, freqs_sin_same = pos_embed(ids_same)
|
||||
assert torch.allclose(freqs_cos, freqs_cos_same), "Same positions should produce same embeddings"
|
||||
assert torch.allclose(freqs_sin, freqs_sin_same), "Same positions should produce same embeddings"
|
||||
|
||||
|
||||
class TestFlux2PackedModuleMapping:
|
||||
"""Test Flux2 packed module mapping functionality"""
|
||||
|
||||
def test_packed_module_mapping(self, setup_tp_group):
|
||||
"""Verify to_qkv packing matches HF checkpoint"""
|
||||
model = Flux2Transformer2DModel(
|
||||
num_layers=1,
|
||||
num_single_layers=0,
|
||||
num_attention_heads=48,
|
||||
attention_head_dim=128,
|
||||
joint_attention_dim=15360,
|
||||
)
|
||||
|
||||
# Verify stacked_params_mapping is correctly initialized
|
||||
model.load_weights([]) # Trigger stacked_params_mapping initialization
|
||||
assert model.stacked_params_mapping is not None
|
||||
|
||||
# Verify mapping configuration
|
||||
expected_mappings = [
|
||||
(".to_qkv.", ".to_q.", "q"),
|
||||
(".to_qkv.", ".to_k.", "k"),
|
||||
(".to_qkv.", ".to_v.", "v"),
|
||||
(".add_kv_proj", ".add_q_proj", "q"),
|
||||
(".add_kv_proj", ".add_k_proj", "k"),
|
||||
(".add_kv_proj", ".add_v_proj", "v"),
|
||||
]
|
||||
assert model.stacked_params_mapping == expected_mappings
|
||||
|
||||
# Create mock HF checkpoint weights
|
||||
hf_weights = []
|
||||
|
||||
# Mock HF separated Q/K/V weights
|
||||
attn_block = model.transformer_blocks[0].attn
|
||||
head_dim = 128
|
||||
num_heads = 48
|
||||
hidden_size = 6144
|
||||
# Full weight dimension
|
||||
full_shard_size = num_heads * head_dim # 6144
|
||||
|
||||
# Q projection weights (full weights)
|
||||
q_weight = torch.randn(full_shard_size, hidden_size)
|
||||
hf_weights.append(("transformer_blocks.0.attn.to_q.weight", q_weight))
|
||||
|
||||
# K projection weights (full weights)
|
||||
k_weight = torch.randn(full_shard_size, hidden_size)
|
||||
hf_weights.append(("transformer_blocks.0.attn.to_k.weight", k_weight))
|
||||
|
||||
# V projection weights (full weights)
|
||||
v_weight = torch.randn(full_shard_size, hidden_size)
|
||||
hf_weights.append(("transformer_blocks.0.attn.to_v.weight", v_weight))
|
||||
|
||||
# Mock HF separated add_q/k/v projection weights
|
||||
add_q_weight = torch.randn(full_shard_size, hidden_size)
|
||||
hf_weights.append(("transformer_blocks.0.attn.add_q_proj.weight", add_q_weight))
|
||||
|
||||
add_k_weight = torch.randn(full_shard_size, hidden_size)
|
||||
hf_weights.append(("transformer_blocks.0.attn.add_k_proj.weight", add_k_weight))
|
||||
|
||||
add_v_weight = torch.randn(full_shard_size, hidden_size)
|
||||
hf_weights.append(("transformer_blocks.0.attn.add_v_proj.weight", add_v_weight))
|
||||
|
||||
# Execute weight loading
|
||||
loaded_params = model.load_weights(hf_weights)
|
||||
|
||||
# Verify weights are loaded
|
||||
assert len(loaded_params) > 0
|
||||
|
||||
# Verify final QKV weights are correctly combined (considering TP sharding)
|
||||
# With TP=2, dimension on each GPU should be half of full dimension
|
||||
expected_qkv_shape = (full_shard_size * 3 // 2, hidden_size) # 18432/2 = 9216
|
||||
assert attn_block.to_qkv.weight.shape == expected_qkv_shape, (
|
||||
f"to_qkv weight dimension should be {expected_qkv_shape}, got {attn_block.to_qkv.weight.shape}"
|
||||
)
|
||||
|
||||
expected_add_kv_shape = (full_shard_size * 3 // 2, hidden_size)
|
||||
assert attn_block.add_kv_proj.weight.shape == expected_add_kv_shape, (
|
||||
f"add_kv_proj weight dimension should be {expected_add_kv_shape}, got {attn_block.add_kv_proj.weight.shape}"
|
||||
)
|
||||
|
||||
def test_packed_mapping_edge_cases(self, setup_tp_group):
|
||||
"""Test edge cases for packed mapping"""
|
||||
model = Flux2Transformer2DModel(
|
||||
num_layers=1,
|
||||
num_single_layers=1,
|
||||
num_attention_heads=48,
|
||||
attention_head_dim=128,
|
||||
joint_attention_dim=15360,
|
||||
)
|
||||
model.load_weights([])
|
||||
|
||||
# Test invalid weight names
|
||||
invalid_weights = [("invalid.weight", torch.randn(10, 10))]
|
||||
loaded_params = model.load_weights(invalid_weights)
|
||||
assert len(loaded_params) == 0, "Should not load invalid weights"
|
||||
|
||||
# Test to_out weight renaming
|
||||
to_out_weight = torch.randn(6144, 6144)
|
||||
weights = [("transformer_blocks.0.attn.to_out.0.weight", to_out_weight)]
|
||||
loaded_params = model.load_weights(weights)
|
||||
|
||||
# Check if to_out related weights are loaded
|
||||
to_out_loaded = any("to_out" in p for p in loaded_params)
|
||||
assert to_out_loaded, "to_out weights should be correctly renamed and loaded"
|
||||
|
||||
# Test partial weight loading
|
||||
partial_weights = [
|
||||
("x_embedder.weight", torch.randn(6144, 128)),
|
||||
("transformer_blocks.0.attn.to_q.weight", torch.randn(6144, 6144)), # Full weights
|
||||
]
|
||||
loaded_params = model.load_weights(partial_weights)
|
||||
assert len(loaded_params) == 2, "Should load two weights"
|
||||
assert "x_embedder.weight" in loaded_params
|
||||
assert any("to_qkv" in p for p in loaded_params), "to_q should be mapped to to_qkv"
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for HunyuanFusedMoE (Support HunyuanImage3 Diffusion Model, 5a779b4)."""
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class TestHunyuanFusedMoEPlatformDispatch:
|
||||
"""Test platform dispatch via platform qualname hooks."""
|
||||
|
||||
def test_default_platform_uses_default_impl_qualname(self, mocker):
|
||||
"""HunyuanFusedMoE should resolve the impl class from the platform hook."""
|
||||
import vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe as hunyuan_moe
|
||||
|
||||
mock_platform = mocker.MagicMock()
|
||||
mock_platform.get_diffusion_model_impl_qualname.return_value = (
|
||||
"vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault"
|
||||
)
|
||||
|
||||
mocker.patch.object(
|
||||
hunyuan_moe,
|
||||
"current_omni_platform",
|
||||
mock_platform,
|
||||
)
|
||||
mock_resolve = mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname")
|
||||
mock_impl = mocker.MagicMock()
|
||||
mock_resolve.return_value = mock_impl
|
||||
|
||||
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe import (
|
||||
HunyuanFusedMoE,
|
||||
)
|
||||
|
||||
HunyuanFusedMoE(prefix="")
|
||||
|
||||
mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("hunyuan_fused_moe")
|
||||
mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe")
|
||||
mock_resolve.assert_called_once_with(
|
||||
"vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault"
|
||||
)
|
||||
mock_impl.assert_called_once_with(prefix="")
|
||||
|
||||
|
||||
class TestHunyuanFusedMoEFactory:
|
||||
"""Test HunyuanFusedMoE factory __new__ and make_expert_params_mapping delegation."""
|
||||
|
||||
def test_new_delegates_to_impl_class(self, mocker):
|
||||
"""HunyuanFusedMoE(prefix=..., **kwargs) should instantiate and return impl instance."""
|
||||
import vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe as hunyuan_moe
|
||||
|
||||
class MockImpl:
|
||||
def __init__(self, *, prefix: str = "", **kwargs):
|
||||
self.prefix = prefix
|
||||
self.kwargs = kwargs
|
||||
|
||||
mock_platform = mocker.MagicMock()
|
||||
mock_platform.get_diffusion_model_impl_qualname.return_value = "mock.impl.Qualname"
|
||||
mocker.patch.object(hunyuan_moe, "current_omni_platform", mock_platform)
|
||||
|
||||
mock_impl_class = mocker.MagicMock(return_value=MockImpl(prefix="test", a=1))
|
||||
mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname", return_value=mock_impl_class)
|
||||
|
||||
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe import (
|
||||
HunyuanFusedMoE,
|
||||
)
|
||||
|
||||
result = HunyuanFusedMoE(prefix="test", a=1)
|
||||
|
||||
assert isinstance(result, MockImpl)
|
||||
assert result.prefix == "test"
|
||||
assert result.kwargs == {"a": 1}
|
||||
mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("hunyuan_fused_moe")
|
||||
mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe")
|
||||
mock_impl_class.assert_called_once_with(prefix="test", a=1)
|
||||
|
||||
def test_make_expert_params_mapping_delegates_to_impl(self, mocker):
|
||||
"""make_expert_params_mapping should delegate to impl class method."""
|
||||
import vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe as hunyuan_moe
|
||||
|
||||
expected_mapping = [("a", "b", 0, "c")]
|
||||
mock_platform = mocker.MagicMock()
|
||||
mock_platform.get_diffusion_model_impl_qualname.return_value = "mock.impl.Qualname"
|
||||
mocker.patch.object(hunyuan_moe, "current_omni_platform", mock_platform)
|
||||
|
||||
mock_impl_class = mocker.MagicMock()
|
||||
mock_impl_class.make_expert_params_mapping = mocker.MagicMock(return_value=expected_mapping)
|
||||
mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname", return_value=mock_impl_class)
|
||||
|
||||
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe import (
|
||||
HunyuanFusedMoE,
|
||||
)
|
||||
|
||||
result = HunyuanFusedMoE.make_expert_params_mapping(
|
||||
model=None,
|
||||
ckpt_gate_proj_name="gate",
|
||||
ckpt_down_proj_name="down",
|
||||
ckpt_up_proj_name="up",
|
||||
num_experts=4,
|
||||
num_redundant_experts=0,
|
||||
)
|
||||
|
||||
assert result == expected_mapping
|
||||
mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe")
|
||||
mock_impl_class.make_expert_params_mapping.assert_called_once_with(
|
||||
None,
|
||||
ckpt_gate_proj_name="gate",
|
||||
ckpt_down_proj_name="down",
|
||||
ckpt_up_proj_name="up",
|
||||
num_experts=4,
|
||||
num_redundant_experts=0,
|
||||
)
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for OmniDiffusionConfig.is_moe (fix is_moe type and threshold, 6663c0b)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class TestOmniDiffusionConfigIsMoE:
|
||||
"""Tests for OmniDiffusionConfig.is_moe property.
|
||||
|
||||
Covers commit 6663c0b: fix is_moe type and threshold
|
||||
- num_experts must be (list, tuple, int); otherwise return False.
|
||||
- Threshold: is_moe is True when num_experts > 0 (not > 1).
|
||||
"""
|
||||
|
||||
def test_is_moe_missing_num_experts_returns_false(self):
|
||||
"""When num_experts is absent, is_moe should be False."""
|
||||
tf_config = TransformerConfig.from_dict({})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is False
|
||||
|
||||
def test_is_moe_none_num_experts_returns_false(self):
|
||||
"""When num_experts is explicitly None (e.g. in params), is_moe should be False."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": None})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is False
|
||||
|
||||
def test_is_moe_non_allowed_type_returns_false(self):
|
||||
"""When num_experts is not int/list/tuple (e.g. str), is_moe should be False."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": "2"})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is False
|
||||
|
||||
def test_is_moe_int_zero_returns_false(self):
|
||||
"""num_experts int 0 should yield is_moe False (threshold > 0)."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": 0})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is False
|
||||
|
||||
def test_is_moe_int_one_returns_true(self):
|
||||
"""num_experts int 1 should yield is_moe True (threshold > 0, not > 1)."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": 1})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is True
|
||||
|
||||
def test_is_moe_int_gt_one_returns_true(self):
|
||||
"""num_experts int > 1 should yield is_moe True."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": 2})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is True
|
||||
|
||||
def test_is_moe_list_all_zero_returns_false(self):
|
||||
"""num_experts list with all <= 0 should yield is_moe False."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": [0]})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is False
|
||||
|
||||
def test_is_moe_list_has_positive_returns_true(self):
|
||||
"""num_experts list with any int > 0 should yield is_moe True."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": [0, 1]})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is True
|
||||
|
||||
def test_is_moe_tuple_has_positive_returns_true(self):
|
||||
"""num_experts tuple with any int > 0 should yield is_moe True."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": (0, 2)})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is True
|
||||
|
||||
def test_is_moe_list_non_int_ignored(self):
|
||||
"""num_experts list with only non-int entries should yield is_moe False."""
|
||||
tf_config = TransformerConfig.from_dict({"num_experts": ["a", 0.0]})
|
||||
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
|
||||
assert config.is_moe is False
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from time import time
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
InstanceInfo,
|
||||
RandomBalancer,
|
||||
StageStatus,
|
||||
)
|
||||
|
||||
|
||||
def test_load_balancer_select_returns_valid_index():
|
||||
"""Verify RandomBalancer.select() returns a valid index for instances."""
|
||||
# Task structure mirrors async_omni; RandomBalancer ignores task contents.
|
||||
task: dict = {
|
||||
"request_id": "test",
|
||||
"engine_inputs": None,
|
||||
"sampling_params": None,
|
||||
}
|
||||
|
||||
now = time()
|
||||
instances = [
|
||||
InstanceInfo(
|
||||
input_addr="tcp://host:10001",
|
||||
output_addr="tcp://host:10001-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
queue_length=0,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
input_addr="tcp://host:10002",
|
||||
output_addr="tcp://host:10002-out",
|
||||
stage_id=0,
|
||||
status=StageStatus.UP,
|
||||
queue_length=1,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
InstanceInfo(
|
||||
input_addr="tcp://host:10003",
|
||||
output_addr="tcp://host:10003-out",
|
||||
stage_id=1,
|
||||
status=StageStatus.UP,
|
||||
queue_length=2,
|
||||
last_heartbeat=now,
|
||||
registered_at=now,
|
||||
),
|
||||
]
|
||||
|
||||
balancer = RandomBalancer()
|
||||
|
||||
index = balancer.select(task, instances)
|
||||
|
||||
assert isinstance(index, int)
|
||||
assert 0 <= index < len(instances)
|
||||
@@ -0,0 +1,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
InstanceList,
|
||||
OmniCoordClientForHub,
|
||||
)
|
||||
|
||||
|
||||
def _bind_pub() -> tuple[zmq.Context, zmq.Socket, str]:
|
||||
ctx = zmq.Context.instance()
|
||||
pub = ctx.socket(zmq.PUB)
|
||||
pub.bind("tcp://127.0.0.1:*")
|
||||
endpoint = pub.getsockopt(zmq.LAST_ENDPOINT).decode("ascii")
|
||||
return ctx, pub, endpoint
|
||||
|
||||
|
||||
def _wait_for_condition(cond, timeout: float = 2.0, interval: float = 0.01) -> bool:
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(interval)
|
||||
return False
|
||||
|
||||
|
||||
def test_hub_client_caches_instance_list_from_pub():
|
||||
"""Verify OmniCoordClientForHub receives instance list updates from OmniCoordinator and caches for get_instance_list()."""
|
||||
ctx, pub, endpoint = _bind_pub()
|
||||
|
||||
client = OmniCoordClientForHub(endpoint)
|
||||
# ZMQ PUB/SUB slow-joiner: allow SUB to finish connecting before first send
|
||||
time.sleep(0.2)
|
||||
|
||||
now = time.time()
|
||||
instances_payload = [
|
||||
{
|
||||
"input_addr": "tcp://stage:10001",
|
||||
"output_addr": "tcp://stage:10001-out",
|
||||
"stage_id": 0,
|
||||
"status": "up",
|
||||
"queue_length": 0,
|
||||
"last_heartbeat": now,
|
||||
"registered_at": now,
|
||||
},
|
||||
{
|
||||
"input_addr": "tcp://stage:10002",
|
||||
"output_addr": "tcp://stage:10002-out",
|
||||
"stage_id": 0,
|
||||
"status": "up",
|
||||
"queue_length": 1,
|
||||
"last_heartbeat": now,
|
||||
"registered_at": now,
|
||||
},
|
||||
{
|
||||
"input_addr": "tcp://stage:10003",
|
||||
"output_addr": "tcp://stage:10003-out",
|
||||
"stage_id": 1,
|
||||
"status": "error",
|
||||
"queue_length": 5,
|
||||
"last_heartbeat": now,
|
||||
"registered_at": now,
|
||||
},
|
||||
]
|
||||
|
||||
payload = {"instances": instances_payload, "timestamp": now}
|
||||
pub.send(json.dumps(payload).encode("utf-8"))
|
||||
|
||||
assert _wait_for_condition(lambda: len(client.get_instance_list().instances) == 3)
|
||||
|
||||
inst_list = client.get_instance_list()
|
||||
assert isinstance(inst_list, InstanceList)
|
||||
assert len(inst_list.instances) == 3
|
||||
|
||||
for src, inst in zip(instances_payload, inst_list.instances, strict=True):
|
||||
assert inst.input_addr == src["input_addr"]
|
||||
assert inst.output_addr == src["output_addr"]
|
||||
assert inst.stage_id == src["stage_id"]
|
||||
assert inst.status.value == src["status"]
|
||||
|
||||
stage0 = client.get_instances_for_stage(0)
|
||||
stage1 = client.get_instances_for_stage(1)
|
||||
|
||||
assert all(inst.stage_id == 0 for inst in stage0.instances)
|
||||
assert all(inst.stage_id == 1 for inst in stage1.instances)
|
||||
|
||||
# Send an updated list with fewer instances and verify cache refresh.
|
||||
updated_payload = {
|
||||
"instances": instances_payload[:2],
|
||||
"timestamp": now + 1.0,
|
||||
}
|
||||
pub.send(json.dumps(updated_payload).encode("utf-8"))
|
||||
|
||||
assert _wait_for_condition(lambda: len(client.get_instance_list().instances) == 2)
|
||||
updated_list = client.get_instance_list()
|
||||
assert len(updated_list.instances) == 2
|
||||
|
||||
client.close()
|
||||
pub.close(0)
|
||||
ctx.term()
|
||||
|
||||
|
||||
def test_hub_client_close_closes_sub_socket():
|
||||
"""Verify OmniCoordClientForHub.close() marks client as closed; second close raises."""
|
||||
ctx, pub, endpoint = _bind_pub()
|
||||
client = OmniCoordClientForHub(endpoint)
|
||||
client.close()
|
||||
|
||||
with pytest.raises(RuntimeError, match="already closed"):
|
||||
client.close()
|
||||
|
||||
pub.close(0)
|
||||
ctx.term()
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import zmq
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
OmniCoordClientForStage,
|
||||
StageStatus,
|
||||
)
|
||||
|
||||
|
||||
def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]:
|
||||
ctx = zmq.Context.instance()
|
||||
router = ctx.socket(zmq.ROUTER)
|
||||
router.bind("tcp://127.0.0.1:*")
|
||||
endpoint = router.getsockopt(zmq.LAST_ENDPOINT).decode("ascii")
|
||||
return ctx, router, endpoint
|
||||
|
||||
|
||||
def _recv_event(router: zmq.Socket) -> dict:
|
||||
frames = router.recv_multipart()
|
||||
# ROUTER adds identity frame; the last frame is the payload.
|
||||
payload = frames[-1]
|
||||
return json.loads(payload.decode("utf-8"))
|
||||
|
||||
|
||||
def test_stage_client_auto_register_on_init():
|
||||
"""Verify OmniCoordClientForStage automatically sends initial registration/status-up event when created."""
|
||||
ctx, router, endpoint = _bind_router()
|
||||
|
||||
input_addr = "tcp://stage:10001"
|
||||
output_addr = "tcp://stage:10001-out"
|
||||
stage_id = 0
|
||||
|
||||
client = OmniCoordClientForStage(endpoint, input_addr, output_addr, stage_id)
|
||||
|
||||
event = _recv_event(router)
|
||||
|
||||
assert event["event_type"] == "update"
|
||||
assert event["status"] == StageStatus.UP.value
|
||||
assert event["stage_id"] == stage_id
|
||||
assert event["input_addr"] == input_addr
|
||||
assert event["output_addr"] == output_addr
|
||||
|
||||
client.close()
|
||||
router.close(0)
|
||||
ctx.term()
|
||||
|
||||
|
||||
def test_stage_client_update_info_sends_correct_event():
|
||||
"""Verify OmniCoordClientForStage.update_info() sends status/load update events with expected fields."""
|
||||
ctx, router, endpoint = _bind_router()
|
||||
|
||||
input_addr = "tcp://stage:10002"
|
||||
output_addr = "tcp://stage:10002-out"
|
||||
stage_id = 1
|
||||
|
||||
client = OmniCoordClientForStage(endpoint, input_addr, output_addr, stage_id)
|
||||
|
||||
# Discard initial registration event.
|
||||
_recv_event(router)
|
||||
|
||||
client.update_info(status=StageStatus.ERROR)
|
||||
client.update_info(queue_length=10)
|
||||
|
||||
first = _recv_event(router)
|
||||
second = _recv_event(router)
|
||||
|
||||
assert first["status"] == StageStatus.ERROR.value
|
||||
assert first["stage_id"] == stage_id
|
||||
assert first["input_addr"] == input_addr
|
||||
assert first["output_addr"] == output_addr
|
||||
|
||||
assert second["queue_length"] == 10
|
||||
assert second["stage_id"] == stage_id
|
||||
assert second["input_addr"] == input_addr
|
||||
assert second["output_addr"] == output_addr
|
||||
|
||||
client.close()
|
||||
router.close(0)
|
||||
ctx.term()
|
||||
|
||||
|
||||
def test_stage_client_close_sends_down_status():
|
||||
"""Verify close() sends final status-down event before closing underlying socket."""
|
||||
ctx, router, endpoint = _bind_router()
|
||||
|
||||
input_addr = "tcp://stage:10003"
|
||||
output_addr = "tcp://stage:10003-out"
|
||||
stage_id = 2
|
||||
|
||||
client = OmniCoordClientForStage(endpoint, input_addr, output_addr, stage_id)
|
||||
|
||||
# Discard initial registration event.
|
||||
_recv_event(router)
|
||||
|
||||
client.close()
|
||||
|
||||
event = _recv_event(router)
|
||||
assert event["status"] == StageStatus.DOWN.value
|
||||
assert event["stage_id"] == stage_id
|
||||
assert event["input_addr"] == input_addr
|
||||
assert event["output_addr"] == output_addr
|
||||
|
||||
assert client._socket.closed # DEALER socket no longer usable after close
|
||||
|
||||
router.close(0)
|
||||
ctx.term()
|
||||
@@ -0,0 +1,209 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
import zmq
|
||||
from vllm.v1.utils import get_engine_client_zmq_addr
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
OmniCoordClientForStage,
|
||||
OmniCoordinator,
|
||||
StageStatus,
|
||||
)
|
||||
|
||||
|
||||
def _recv_instance_list(sub: zmq.Socket, timeout_ms: int = 2000) -> dict | None:
|
||||
"""Receive InstanceList JSON from SUB socket. Returns None on timeout."""
|
||||
sub.setsockopt(zmq.RCVTIMEO, timeout_ms)
|
||||
try:
|
||||
data = sub.recv()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
except zmq.Again:
|
||||
return None
|
||||
|
||||
|
||||
def _wait_for_instance_list(
|
||||
sub: zmq.Socket,
|
||||
expected_count: int,
|
||||
timeout: float = 3.0,
|
||||
) -> dict | None:
|
||||
"""Wait until received InstanceList with expected_count active instances."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
msg = _recv_instance_list(sub, timeout_ms=500)
|
||||
if msg is not None and len(msg.get("instances", [])) == expected_count:
|
||||
return msg
|
||||
return None
|
||||
|
||||
|
||||
def test_omni_coordinator_registration_broadcast():
|
||||
"""Verify that after multiple OmniCoordClientForStage instances register,
|
||||
OmniCoordinator publishes an InstanceList containing all registered instances.
|
||||
"""
|
||||
router_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
)
|
||||
pub_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
)
|
||||
coordinator = OmniCoordinator(
|
||||
router_zmq_addr=router_addr,
|
||||
pub_zmq_addr=pub_addr,
|
||||
heartbeat_timeout=1000.0,
|
||||
)
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
# ZMQ PUB/SUB slow-joiner: allow SUB to connect before clients register.
|
||||
time.sleep(0.3)
|
||||
|
||||
# Create 3 stage clients; each auto-registers on init.
|
||||
clients = [
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10001", "tcp://stage:10001-out", 0),
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10002", "tcp://stage:10002-out", 0),
|
||||
OmniCoordClientForStage(router_addr, "tcp://stage:10003", "tcp://stage:10003-out", 1),
|
||||
]
|
||||
|
||||
msg = _wait_for_instance_list(sub, expected_count=3)
|
||||
assert msg is not None, "Expected InstanceList with 3 instances"
|
||||
assert len(msg["instances"]) == 3
|
||||
assert isinstance(msg["timestamp"], (int, float))
|
||||
|
||||
input_addrs = {inst["input_addr"] for inst in msg["instances"]}
|
||||
assert "tcp://stage:10001" in input_addrs
|
||||
assert "tcp://stage:10002" in input_addrs
|
||||
assert "tcp://stage:10003" in input_addrs
|
||||
|
||||
for c in clients:
|
||||
c.close()
|
||||
coordinator.close()
|
||||
sub.close(0)
|
||||
sub_ctx.term()
|
||||
|
||||
|
||||
def test_omni_coordinator_heartbeat_timeout_handling():
|
||||
"""Verify that when a stage instance stops sending heartbeats,
|
||||
OmniCoordinator marks it as unhealthy and excludes it from the active list.
|
||||
"""
|
||||
router_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
)
|
||||
pub_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
)
|
||||
coordinator = OmniCoordinator(
|
||||
router_zmq_addr=router_addr,
|
||||
pub_zmq_addr=pub_addr,
|
||||
heartbeat_timeout=5.0,
|
||||
)
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
time.sleep(0.3)
|
||||
|
||||
# A and B: real clients that send heartbeats every 5s.
|
||||
client_a = OmniCoordClientForStage(router_addr, "tcp://stage:a", "tcp://stage:a-out", 0)
|
||||
client_b = OmniCoordClientForStage(router_addr, "tcp://stage:b", "tcp://stage:b-out", 0)
|
||||
|
||||
# C: raw DEALER that sends only registration, no heartbeat.
|
||||
dealer_ctx = zmq.Context.instance()
|
||||
dealer_c = dealer_ctx.socket(zmq.DEALER)
|
||||
dealer_c.connect(router_addr)
|
||||
reg_event = {
|
||||
"input_addr": "tcp://stage:c",
|
||||
"output_addr": "tcp://stage:c-out",
|
||||
"stage_id": 0,
|
||||
"event_type": "update",
|
||||
"status": StageStatus.UP.value,
|
||||
"queue_length": 0,
|
||||
}
|
||||
dealer_c.send(json.dumps(reg_event).encode("utf-8"))
|
||||
|
||||
msg = _wait_for_instance_list(sub, expected_count=3)
|
||||
assert msg is not None, "Expected initial 3 instances"
|
||||
assert len(msg["instances"]) == 3
|
||||
|
||||
# Wait for heartbeat timeout (timeout=5s, check interval ~2.5s).
|
||||
time.sleep(8.0)
|
||||
|
||||
# Receive the update (C should be ERROR and excluded from active list).
|
||||
msg_after_timeout = _wait_for_instance_list(sub, expected_count=2, timeout=5.0)
|
||||
assert msg_after_timeout is not None, "Expected InstanceList with 2 instances after timeout"
|
||||
instances = msg_after_timeout.get("instances", [])
|
||||
input_addrs = {inst["input_addr"] for inst in instances}
|
||||
|
||||
assert "tcp://stage:a" in input_addrs
|
||||
assert "tcp://stage:b" in input_addrs
|
||||
assert "tcp://stage:c" not in input_addrs
|
||||
|
||||
client_a.close()
|
||||
client_b.close()
|
||||
dealer_c.close(0)
|
||||
coordinator.close()
|
||||
sub.close(0)
|
||||
dealer_ctx.term()
|
||||
sub_ctx.term()
|
||||
|
||||
|
||||
def test_omni_coordinator_instance_shutdown_handling():
|
||||
"""Verify that when a stage instance sends status='down',
|
||||
OmniCoordinator removes it from the active list and broadcasts an updated list.
|
||||
"""
|
||||
router_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
)
|
||||
pub_addr = get_engine_client_zmq_addr(
|
||||
local_only=False,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
)
|
||||
coordinator = OmniCoordinator(
|
||||
router_zmq_addr=router_addr,
|
||||
pub_zmq_addr=pub_addr,
|
||||
heartbeat_timeout=1000.0,
|
||||
)
|
||||
|
||||
sub_ctx = zmq.Context.instance()
|
||||
sub = sub_ctx.socket(zmq.SUB)
|
||||
sub.connect(pub_addr)
|
||||
sub.setsockopt(zmq.SUBSCRIBE, b"")
|
||||
|
||||
time.sleep(0.3) # PUB/SUB slow-joiner
|
||||
|
||||
client = OmniCoordClientForStage(router_addr, "tcp://stage:shutdown", "tcp://stage:shutdown-out", 0)
|
||||
|
||||
msg = _wait_for_instance_list(sub, expected_count=1)
|
||||
assert msg is not None
|
||||
assert len(msg["instances"]) == 1
|
||||
assert msg["instances"][0]["input_addr"] == "tcp://stage:shutdown"
|
||||
|
||||
# Send down status (simulating graceful shutdown).
|
||||
client.update_info(status=StageStatus.DOWN)
|
||||
|
||||
# Receive updated list (should have 0 active instances).
|
||||
msg = _wait_for_instance_list(sub, expected_count=0)
|
||||
assert msg is not None
|
||||
assert len(msg["instances"]) == 0
|
||||
|
||||
client.close()
|
||||
coordinator.close()
|
||||
sub.close(0)
|
||||
sub_ctx.term()
|
||||
@@ -6,7 +6,7 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor, hardware_test
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -23,9 +23,9 @@ models = ["riverclouds/qwen_image_random"]
|
||||
def inference(model_name: str, offload: bool = True):
|
||||
gc.collect()
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = torch.cuda.current_device()
|
||||
torch.cuda.reset_peak_memory_stats(device=device_index)
|
||||
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
device_index = current_omni_platform.current_device()
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
m = Omni(
|
||||
model=model_name,
|
||||
@@ -33,7 +33,7 @@ def inference(model_name: str, offload: bool = True):
|
||||
# cache_backend="cache_dit",
|
||||
enable_cpu_offload=offload,
|
||||
)
|
||||
torch.cuda.reset_peak_memory_stats(device=device_index)
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
height = 256
|
||||
width = 256
|
||||
|
||||
@@ -44,7 +44,7 @@ def inference(model_name: str, offload: bool = True):
|
||||
width=width,
|
||||
num_inference_steps=9,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
),
|
||||
)
|
||||
peak = monitor.peak_used_mb
|
||||
@@ -70,6 +70,11 @@ def test_cpu_offload_diffusion_model(model_name: str):
|
||||
pytest.fail("Inference failed")
|
||||
print(f"Offload peak memory: {offload_peak_memory} MB")
|
||||
print(f"No offload peak memory: {no_offload_peak_memory} MB")
|
||||
assert offload_peak_memory + 2500 < no_offload_peak_memory, (
|
||||
f"Offload peak memory {offload_peak_memory} MB should be less than no offload peak memory {no_offload_peak_memory} MB"
|
||||
# Set platform-specific VRAM saving thresholds to account
|
||||
# for varying runtime memory overhead and fragmentation between CUDA and ROCm.
|
||||
is_rocm = torch.version.hip is not None
|
||||
threshold = 2500 if not is_rocm else 2100
|
||||
assert offload_peak_memory + threshold < no_offload_peak_memory, (
|
||||
f"Offload peak memory {offload_peak_memory} MB should be less than "
|
||||
f"no offload peak memory {no_offload_peak_memory} MB by {threshold} MB"
|
||||
)
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor
|
||||
from tests.utils import DeviceMemoryMonitor
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -28,11 +28,9 @@ def run_inference(
|
||||
layerwise_offload: bool = False,
|
||||
num_inference_steps: int = 3,
|
||||
) -> float:
|
||||
# For now, only support on GPU, so apply torch.cuda operations here
|
||||
# NPU / ROCm platforms are expected to be detected and skipped this test function
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = torch.cuda.current_device()
|
||||
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
device_index = current_omni_platform.current_device()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
|
||||
m = Omni(
|
||||
@@ -44,7 +42,7 @@ def run_inference(
|
||||
flow_shift=5.0,
|
||||
)
|
||||
|
||||
torch.cuda.reset_peak_memory_stats(device=device_index)
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
|
||||
# Refer to tests/e2e/offline_inference/test_t2v_model.py
|
||||
# Use minimal settings for testing
|
||||
@@ -57,7 +55,7 @@ def run_inference(
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
guidance_scale=1.0,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
|
||||
@@ -146,9 +146,6 @@ def _run_inference(
|
||||
# - warmup: whether to run warmup for this SP config
|
||||
# - is_perf_test: whether this is a performance test (show speedup metrics)
|
||||
SP_CONFIGS_L2 = [
|
||||
# Ulysses-2 - performance test
|
||||
(2, 1, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True),
|
||||
(1, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True), # Ring-2 - performance test
|
||||
# Hybrid - correctness only
|
||||
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
|
||||
]
|
||||
|
||||
@@ -22,7 +22,7 @@ import torch
|
||||
from PIL import Image
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor, hardware_test
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
@@ -54,12 +54,13 @@ def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor:
|
||||
|
||||
|
||||
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
|
||||
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
|
||||
"""Return (mean_abs_diff, p99_abs_diff) over RGB pixels in [0, 1]."""
|
||||
ta = _pil_to_float_rgb_tensor(a)
|
||||
tb = _pil_to_float_rgb_tensor(b)
|
||||
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
|
||||
abs_diff = torch.abs(ta - tb)
|
||||
return abs_diff.mean().item(), abs_diff.max().item()
|
||||
p99_abs_diff = torch.quantile(abs_diff.flatten(), 0.99).item()
|
||||
return abs_diff.mean().item(), p99_abs_diff
|
||||
|
||||
|
||||
def _extract_single_image(outputs) -> Image.Image:
|
||||
@@ -93,9 +94,9 @@ def _run_zimage_generate(
|
||||
if num_requests < 2:
|
||||
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
device_index = torch.cuda.current_device()
|
||||
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = current_omni_platform.current_device()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
m = Omni(
|
||||
model=_get_zimage_model(),
|
||||
@@ -161,8 +162,8 @@ def _run_zimage_generate(
|
||||
def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
pytest.skip("Z-Image TP e2e test is only supported on CUDA for now.")
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
|
||||
pytest.skip("Z-Image TP=2 requires >= 2 CUDA devices.")
|
||||
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
|
||||
pytest.skip("Z-Image TP=2 requires >= 2 devices.")
|
||||
|
||||
enforce_eager = False
|
||||
|
||||
@@ -196,18 +197,18 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
assert tp1_img.width == width and tp1_img.height == height
|
||||
assert tp2_img.width == width and tp2_img.height == height
|
||||
|
||||
mean_abs_diff, max_abs_diff = _diff_metrics(tp1_img, tp2_img)
|
||||
mean_abs_diff, p99_abs_diff = _diff_metrics(tp1_img, tp2_img)
|
||||
mean_threshold = 3e-2
|
||||
max_threshold = 5e-1
|
||||
p99_threshold = 2.5e-1
|
||||
print(
|
||||
"Z-Image TP image diff stats (TP=1 vs TP=2): "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
|
||||
f"tp1_img={tp1_path}, tp2_img={tp2_path}"
|
||||
)
|
||||
assert mean_abs_diff <= mean_threshold and max_abs_diff <= max_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e})"
|
||||
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
|
||||
)
|
||||
|
||||
print(f"Z-Image TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}")
|
||||
@@ -219,59 +220,59 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA for now.")
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
|
||||
pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 CUDA devices.")
|
||||
# @pytest.mark.integration
|
||||
# def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
|
||||
# if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
# pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA for now.")
|
||||
# if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
|
||||
# pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 devices.")
|
||||
|
||||
enforce_eager = False
|
||||
# enforce_eager = False
|
||||
|
||||
# Use a larger image to ensure there are multiple VAE tiles.
|
||||
height = 1152
|
||||
width = 1152
|
||||
num_inference_steps = 2
|
||||
seed = 42
|
||||
# # Use a larger image to ensure there are multiple VAE tiles.
|
||||
# height = 1152
|
||||
# width = 1152
|
||||
# num_inference_steps = 2
|
||||
# seed = 42
|
||||
|
||||
baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=True,
|
||||
vae_patch_parallel_size=1,
|
||||
num_requests=2,
|
||||
)
|
||||
pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=True,
|
||||
vae_patch_parallel_size=2,
|
||||
num_requests=2,
|
||||
)
|
||||
# baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
|
||||
# tp_size=2,
|
||||
# height=height,
|
||||
# width=width,
|
||||
# num_inference_steps=num_inference_steps,
|
||||
# seed=seed,
|
||||
# enforce_eager=enforce_eager,
|
||||
# vae_use_tiling=True,
|
||||
# vae_patch_parallel_size=1,
|
||||
# num_requests=2,
|
||||
# )
|
||||
# pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
|
||||
# tp_size=2,
|
||||
# height=height,
|
||||
# width=width,
|
||||
# num_inference_steps=num_inference_steps,
|
||||
# seed=seed,
|
||||
# enforce_eager=enforce_eager,
|
||||
# vae_use_tiling=True,
|
||||
# vae_patch_parallel_size=2,
|
||||
# num_requests=2,
|
||||
# )
|
||||
|
||||
baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
|
||||
pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
|
||||
baseline_img.save(baseline_path)
|
||||
pp2_img.save(pp2_path)
|
||||
# baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
|
||||
# pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
|
||||
# baseline_img.save(baseline_path)
|
||||
# pp2_img.save(pp2_path)
|
||||
|
||||
mean_abs_diff, max_abs_diff = _diff_metrics(baseline_img, pp2_img)
|
||||
mean_threshold = 5e-3
|
||||
max_threshold = 1e-1
|
||||
print(
|
||||
"Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
|
||||
f"pp1_img={baseline_path}, pp2_img={pp2_path}"
|
||||
)
|
||||
assert mean_abs_diff <= mean_threshold and max_abs_diff <= max_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e})"
|
||||
)
|
||||
# mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img)
|
||||
# mean_threshold = 5e-3
|
||||
# p99_threshold = 1e-1
|
||||
# print(
|
||||
# "Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
|
||||
# f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
|
||||
# f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
|
||||
# f"pp1_img={baseline_path}, pp2_img={pp2_path}"
|
||||
# )
|
||||
# assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
|
||||
# f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
|
||||
# f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
|
||||
# )
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServerParams,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
@@ -51,7 +52,9 @@ else:
|
||||
stage_configs = [get_chunk_config()]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
@@ -115,12 +118,11 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"stream": True,
|
||||
"key_words": {
|
||||
"audio": ["test"],
|
||||
"image": ["square", "quadrate"],
|
||||
},
|
||||
}
|
||||
|
||||
# Test single completion
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -146,4 +148,4 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -12,6 +12,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServerParams,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
@@ -50,7 +51,9 @@ def get_chunk_config(default_path):
|
||||
default_path = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
stage_configs = [default_path, get_chunk_config(default_path)]
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
@@ -106,7 +109,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -128,7 +131,7 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -153,7 +156,7 @@ def test_image_to_text_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -177,7 +180,7 @@ def test_image_to_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -201,7 +204,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -225,7 +228,7 @@ def test_video_to_text_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -249,7 +252,7 @@ def test_video_to_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -273,7 +276,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -298,7 +301,7 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"audio": AUDIO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -324,7 +327,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -351,7 +354,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
|
||||
@@ -383,4 +386,4 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"stream": True,
|
||||
"key_words": {"audio": AUDIO_KEY, "image": IMAGE_KEY, "video": VIDEO_KEY},
|
||||
}
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online tests for Qwen3-TTS WebSocket streaming speech.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.conftest import OmniServer
|
||||
from tests.utils import hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
||||
|
||||
|
||||
def get_stage_config() -> str:
|
||||
return str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server():
|
||||
stage_config_path = get_stage_config()
|
||||
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--stage-configs-path",
|
||||
stage_config_path,
|
||||
"--stage-init-timeout",
|
||||
"120",
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
env_dict={"VLLM_DISABLE_COMPILE_CACHE": "1"},
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
async def _run_ws_session(host: str, port: int) -> dict:
|
||||
uri = f"ws://{host}:{port}/v1/audio/speech/stream"
|
||||
starts: list[dict] = []
|
||||
dones: list[dict] = []
|
||||
chunk_lengths: dict[int, list[int]] = {}
|
||||
session_done: dict | None = None
|
||||
|
||||
async with websockets.connect(uri, max_size=None) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": MODEL,
|
||||
"voice": "vivian",
|
||||
"language": "English",
|
||||
"response_format": "pcm",
|
||||
"stream_audio": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": (
|
||||
"Hello, this is a websocket streaming test for Qwen three TTS, "
|
||||
"and this sentence is intentionally long enough to produce audio chunks. "
|
||||
"This is the second sentence."
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
|
||||
while True:
|
||||
message = await asyncio.wait_for(ws.recv(), timeout=180)
|
||||
if isinstance(message, bytes):
|
||||
if not starts:
|
||||
raise AssertionError("Received audio bytes before audio.start")
|
||||
sentence_index = starts[-1]["sentence_index"]
|
||||
chunk_lengths.setdefault(sentence_index, []).append(len(message))
|
||||
continue
|
||||
|
||||
payload = json.loads(message)
|
||||
msg_type = payload.get("type")
|
||||
if msg_type == "audio.start":
|
||||
starts.append(payload)
|
||||
chunk_lengths.setdefault(payload["sentence_index"], [])
|
||||
elif msg_type == "audio.done":
|
||||
dones.append(payload)
|
||||
elif msg_type == "session.done":
|
||||
session_done = payload
|
||||
break
|
||||
elif msg_type == "error":
|
||||
raise AssertionError(f"WebSocket error: {payload['message']}")
|
||||
else:
|
||||
raise AssertionError(f"Unexpected WebSocket message: {payload}")
|
||||
|
||||
return {
|
||||
"starts": starts,
|
||||
"dones": dones,
|
||||
"chunk_lengths": chunk_lengths,
|
||||
"session_done": session_done,
|
||||
}
|
||||
|
||||
|
||||
class TestQwen3TTSWebSocket:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
def test_streaming_pcm_output(self, omni_server) -> None:
|
||||
result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port))
|
||||
|
||||
starts = result["starts"]
|
||||
dones = result["dones"]
|
||||
chunk_lengths = result["chunk_lengths"]
|
||||
session_done = result["session_done"]
|
||||
|
||||
assert session_done is not None
|
||||
assert session_done["total_sentences"] == 2
|
||||
assert len(starts) == 2
|
||||
assert len(dones) == 2
|
||||
|
||||
for idx, start in enumerate(starts):
|
||||
assert start["type"] == "audio.start"
|
||||
assert start["sentence_index"] == idx
|
||||
assert start["format"] == "pcm"
|
||||
assert start["sample_rate"] == 24000
|
||||
assert start["sentence_text"]
|
||||
|
||||
for done in dones:
|
||||
sentence_index = done["sentence_index"]
|
||||
total_bytes = done["total_bytes"]
|
||||
assert done["error"] is False
|
||||
assert total_bytes > 0
|
||||
assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}"
|
||||
assert sum(chunk_lengths[sentence_index]) == total_bytes
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Comprehensive tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following models:
|
||||
- Qwen-Image-Edit: single image input
|
||||
- Qwen-Image-Edit-2509: two image inputs
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_image,
|
||||
)
|
||||
from tests.utils import hardware_marks
|
||||
|
||||
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
"Transform the second image into a Vincent van Gogh style painting. "
|
||||
"Then juxtapose the two transformed images into a single artwork for visual contrast."
|
||||
)
|
||||
NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
||||
|
||||
|
||||
# This test file targets two models, so I write a helper function.
|
||||
# If a similar test only involves one model, one can just define a global list variable.
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together.
|
||||
"--enable-layerwise-offload",
|
||||
],
|
||||
),
|
||||
id="single_card_001",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"--ulysses-degree",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_001",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"--ring",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_002",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"tea_cache",
|
||||
"--cfg-parallel-size",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_003",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--vae-patch-parallel-size",
|
||||
"2",
|
||||
"--vae-use-tiling",
|
||||
],
|
||||
),
|
||||
id="parallel_004",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit_2509(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test all diffusion features with Qwen-Image-Edit-2509 in regular end-user scenarios."""
|
||||
image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(
|
||||
image_data_url=[image_data_url_1, image_data_url_2], content_text=MULTI_EDIT_PROMPT
|
||||
)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -136,7 +136,6 @@ def mock_async_diffusion(mocker: MockerFixture):
|
||||
|
||||
async def generate(**kwargs):
|
||||
# Return n PIL images wrapped in result object
|
||||
print("!!!!!!!!!!!!!!!!!!!!! kwargs", kwargs)
|
||||
n = kwargs["sampling_params_list"][0].num_outputs_per_prompt
|
||||
mock.captured_sampling_params_list = kwargs["sampling_params_list"]
|
||||
mock.captured_prompt = kwargs["prompt"]
|
||||
|
||||
@@ -1,23 +1,29 @@
|
||||
# tests/entrypoints/openai/test_serving_speech.py
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
from inspect import Signature, signature
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from fastapi import FastAPI, HTTPException, UploadFile
|
||||
from fastapi import FastAPI, HTTPException, Request, UploadFile
|
||||
from fastapi.params import File, Form
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import ValidationError
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
|
||||
from vllm_omni.entrypoints.openai import api_server as api_server_module
|
||||
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio, OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||
OmniOpenAIServingSpeech,
|
||||
_create_wav_header,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -916,17 +922,20 @@ class TestStreamingProtocolValidation:
|
||||
"""Unit tests for the stream field validators in OpenAICreateSpeechRequest."""
|
||||
|
||||
def test_stream_validation_errors(self):
|
||||
"""stream=True requires response_format='pcm' and speed=1.0."""
|
||||
with pytest.raises(ValidationError, match="response_format='pcm'"):
|
||||
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="wav")
|
||||
"""stream=True requires response_format not in ('pcm', 'wav') and speed=1.0."""
|
||||
with pytest.raises(ValidationError, match="requires response_format not in \\('pcm', 'wav'\\)"):
|
||||
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="mp3")
|
||||
with pytest.raises(ValidationError, match="Speed adjustment is not supported"):
|
||||
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm", speed=2.0)
|
||||
|
||||
def test_stream_valid(self):
|
||||
"""stream=True + response_format='pcm' + speed=1.0 is accepted."""
|
||||
"""stream=True + response_format in ('pcm', 'wav') + speed=1.0 is accepted."""
|
||||
req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm")
|
||||
assert req.stream is True
|
||||
|
||||
req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="wav")
|
||||
assert req.stream is True
|
||||
|
||||
def test_sse_stream_format_is_blocked(self):
|
||||
"""stream_format='sse' is blocked."""
|
||||
with pytest.raises(ValidationError, match="sse"):
|
||||
@@ -1017,3 +1026,256 @@ class TestStreamingResponse:
|
||||
response = client.post("/v1/audio/speech", json={"input": "Hello", "response_format": "wav"})
|
||||
assert response.status_code == 200
|
||||
assert "audio/wav" in response.headers["content-type"]
|
||||
|
||||
|
||||
class TestAsyncOmniSupportedTasks:
|
||||
"""Test that AsyncOmni reports correct supported tasks based on output modalities."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tts_only_no_generate_task(self):
|
||||
"""TTS-only models (audio output, no text) should not include 'generate'."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
omni = AsyncOmni.__new__(AsyncOmni)
|
||||
omni.output_modalities = [None, "audio"]
|
||||
stage = MagicMock()
|
||||
stage.is_comprehension = False
|
||||
omni.stage_list = [stage]
|
||||
tasks = await omni.get_supported_tasks()
|
||||
assert "generate" not in tasks
|
||||
assert "speech" in tasks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_omni_model_includes_generate(self):
|
||||
"""Models with text output (e.g. Qwen3-Omni) should include 'generate'."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
omni = AsyncOmni.__new__(AsyncOmni)
|
||||
omni.output_modalities = ["text", None, "audio"]
|
||||
stage = MagicMock()
|
||||
stage.is_comprehension = True
|
||||
omni.stage_list = [stage]
|
||||
tasks = await omni.get_supported_tasks()
|
||||
assert "generate" in tasks
|
||||
|
||||
|
||||
def test_api_server_create_speech_wraps_error_response_status():
|
||||
handler = MagicMock()
|
||||
handler.create_speech = AsyncMock(
|
||||
return_value=ErrorResponse(
|
||||
error=ErrorInfo(message="bad request", type="BadRequestError", param=None, code=400),
|
||||
)
|
||||
)
|
||||
|
||||
app = FastAPI()
|
||||
app.state.openai_serving_speech = handler
|
||||
scope = {
|
||||
"type": "http",
|
||||
"app": app,
|
||||
"method": "POST",
|
||||
"path": "/v1/audio/speech",
|
||||
"headers": [],
|
||||
"query_string": b"",
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"server": ("testserver", 80),
|
||||
"scheme": "http",
|
||||
}
|
||||
raw_request = Request(scope)
|
||||
request = OpenAICreateSpeechRequest(input="Hello")
|
||||
|
||||
response = asyncio.run(api_server_module.create_speech(request, raw_request))
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
class TestWAVHeaderGeneration:
|
||||
"""Unit tests for WAV header generation with placeholder values."""
|
||||
|
||||
def test_wav_header_basic_structure(self):
|
||||
"""Test basic WAV header structure with default parameters."""
|
||||
header = _create_wav_header(sample_rate=24000, num_channels=1, bits_per_sample=16)
|
||||
|
||||
# Verify header length (should be 44 bytes)
|
||||
assert len(header) == 44, f"Expected 44 bytes, got {len(header)}"
|
||||
|
||||
# Parse and verify header structure
|
||||
(
|
||||
chunk_id,
|
||||
chunk_size,
|
||||
format_type,
|
||||
subchunk1_id,
|
||||
subchunk1_size,
|
||||
audio_format,
|
||||
num_channels,
|
||||
sample_rate,
|
||||
byte_rate,
|
||||
block_align,
|
||||
bits_per_sample,
|
||||
subchunk2_id,
|
||||
subchunk2_size,
|
||||
) = struct.unpack("<4sI4s4sIHHIIHH4sI", header)
|
||||
|
||||
# Verify RIFF header
|
||||
assert chunk_id == b"RIFF", f"Expected RIFF, got {chunk_id}"
|
||||
assert chunk_size == 0xFFFFFFFF, f"Expected placeholder 0xFFFFFFFF, got {chunk_size:#x}"
|
||||
assert format_type == b"WAVE", f"Expected WAVE, got {format_type}"
|
||||
|
||||
# Verify fmt chunk
|
||||
assert subchunk1_id == b"fmt ", f"Expected 'fmt ', got {subchunk1_id}"
|
||||
assert subchunk1_size == 16, f"Expected 16, got {subchunk1_size}"
|
||||
assert audio_format == 1, f"Expected PCM (1), got {audio_format}"
|
||||
assert num_channels == 1, f"Expected 1 channel, got {num_channels}"
|
||||
assert sample_rate == 24000, f"Expected 24000 Hz, got {sample_rate}"
|
||||
assert byte_rate == 48000, f"Expected 48000 byte/s, got {byte_rate}"
|
||||
assert block_align == 2, f"Expected 2 bytes block align, got {block_align}"
|
||||
assert bits_per_sample == 16, f"Expected 16 bits, got {bits_per_sample}"
|
||||
|
||||
# Verify data chunk
|
||||
assert subchunk2_id == b"data", f"Expected 'data', got {subchunk2_id}"
|
||||
assert subchunk2_size == 0xFFFFFFFF, f"Expected placeholder 0xFFFFFFFF, got {subchunk2_size:#x}"
|
||||
|
||||
def test_wav_header_different_sample_rates(self):
|
||||
"""Test WAV header with different sample rates."""
|
||||
test_cases = [
|
||||
(16000, 1, 16),
|
||||
(22050, 1, 16),
|
||||
(24000, 1, 16),
|
||||
(44100, 1, 16),
|
||||
(48000, 1, 16),
|
||||
]
|
||||
|
||||
for sample_rate, num_channels, bits_per_sample in test_cases:
|
||||
header = _create_wav_header(sample_rate, num_channels, bits_per_sample)
|
||||
assert len(header) == 44, f"Header length mismatch for {sample_rate} Hz"
|
||||
|
||||
# Parse sample rate from header
|
||||
parsed_sample_rate = struct.unpack("<I", header[24:28])[0]
|
||||
assert parsed_sample_rate == sample_rate, (
|
||||
f"Sample rate mismatch: expected {sample_rate}, got {parsed_sample_rate}"
|
||||
)
|
||||
|
||||
def test_wav_header_stereo(self):
|
||||
"""Test WAV header with stereo audio."""
|
||||
header = _create_wav_header(sample_rate=44100, num_channels=2, bits_per_sample=16)
|
||||
|
||||
# Parse header
|
||||
parsed = struct.unpack("<4sI4s4sIHHIIHH4sI", header)
|
||||
num_channels = parsed[6]
|
||||
byte_rate = parsed[8]
|
||||
block_align = parsed[9]
|
||||
|
||||
assert num_channels == 2, f"Expected 2 channels, got {num_channels}"
|
||||
assert byte_rate == 44100 * 2 * 16 // 8, "Byte rate mismatch"
|
||||
assert block_align == 2 * 16 // 8, "Block align mismatch"
|
||||
|
||||
def test_wav_header_placeholder_values(self):
|
||||
"""Test that placeholder values are correctly set to 0xFFFFFFFF."""
|
||||
header = _create_wav_header(sample_rate=24000)
|
||||
|
||||
# Extract size fields
|
||||
chunk_size = struct.unpack("<I", header[4:8])[0]
|
||||
subchunk2_size = struct.unpack("<I", header[40:44])[0]
|
||||
|
||||
assert chunk_size == 0xFFFFFFFF, "ChunkSize should be 0xFFFFFFFF for streaming"
|
||||
assert subchunk2_size == 0xFFFFFFFF, "Subchunk2Size should be 0xFFFFFFFF for streaming"
|
||||
|
||||
|
||||
class TestWAVStreaming:
|
||||
"""Integration tests for WAV format streaming."""
|
||||
|
||||
@pytest.fixture
|
||||
def wav_streaming_app(self, mocker: MockerFixture):
|
||||
"""Test app configured for WAV streaming."""
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
|
||||
def _make_output(finished: bool) -> OmniRequestOutput:
|
||||
chunk = torch.sin(torch.linspace(0, 440 * 2 * torch.pi, 24000))
|
||||
|
||||
class MockCompletionOutput:
|
||||
def __init__(self, index: int = 0):
|
||||
self.index = index
|
||||
self.text = ""
|
||||
self.token_ids = []
|
||||
self.finish_reason = "stop"
|
||||
self.stop_reason = None
|
||||
self.logprobs = None
|
||||
|
||||
class MockRequestOutput:
|
||||
def __init__(self, audio_tensor: torch.Tensor):
|
||||
self.request_id = "speech-wav-stream-test"
|
||||
self.outputs = [MockCompletionOutput(index=0)]
|
||||
self.multimodal_output = {"audio": audio_tensor, "sr": 24000}
|
||||
self.finished = finished
|
||||
self.prompt_token_ids = None
|
||||
self.encoder_prompt_token_ids = None
|
||||
self.num_cached_tokens = None
|
||||
self.prompt_logprobs = None
|
||||
self.kv_transfer_params = None
|
||||
|
||||
return OmniRequestOutput(
|
||||
stage_id=0,
|
||||
final_output_type="audio",
|
||||
request_output=MockRequestOutput(audio_tensor=chunk),
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
async def mock_generate_streaming(*args, **kwargs):
|
||||
yield _make_output(finished=False)
|
||||
yield _make_output(finished=True)
|
||||
|
||||
mock_engine_client.generate = mocker.MagicMock(side_effect=mock_generate_streaming)
|
||||
mock_engine_client.default_sampling_params_list = [{}]
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
speech_server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
original_create_speech = speech_server.create_speech
|
||||
sig = signature(original_create_speech)
|
||||
new_parameters = [p for name, p in sig.parameters.items() if name != "raw_request"]
|
||||
new_sig = Signature(parameters=new_parameters, return_annotation=sig.return_annotation)
|
||||
|
||||
async def awaitable_create_speech(*args, **kwargs):
|
||||
return await original_create_speech(*args, **kwargs)
|
||||
|
||||
awaitable_create_speech.__signature__ = new_sig
|
||||
speech_server.create_speech = awaitable_create_speech
|
||||
|
||||
app = FastAPI()
|
||||
app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None)
|
||||
return app
|
||||
|
||||
def test_wav_streaming_success(self, wav_streaming_app):
|
||||
"""Test WAV format streaming returns correct content type and includes WAV header."""
|
||||
client = TestClient(wav_streaming_app)
|
||||
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": "wav"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "audio/wav" in response.headers["content-type"]
|
||||
assert len(response.content) > 44 # Should have WAV header + audio data
|
||||
|
||||
# Verify WAV header is present
|
||||
header = response.content[:44]
|
||||
chunk_id = header[0:4]
|
||||
format_type = header[8:12]
|
||||
assert chunk_id == b"RIFF", "Should start with RIFF"
|
||||
assert format_type == b"WAVE", "Should contain WAVE format"
|
||||
|
||||
def test_streaming_unsupported_format_rejected(self, wav_streaming_app):
|
||||
"""Test that unsupported formats are rejected for streaming."""
|
||||
client = TestClient(wav_streaming_app)
|
||||
|
||||
unsupported_formats = ["mp3"]
|
||||
for fmt in unsupported_formats:
|
||||
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": fmt})
|
||||
assert response.status_code == 422
|
||||
@@ -0,0 +1,387 @@
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, WebSocket
|
||||
from starlette.testclient import TestClient
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from vllm_omni.entrypoints.openai import serving_speech_stream as streaming_speech_module
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _build_test_app(speech_service=None, *, idle_timeout=30.0, config_timeout=10.0):
|
||||
if speech_service is None:
|
||||
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
|
||||
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"RIFF" + b"\x00" * 32, "audio/wav"))
|
||||
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-1", object(), {}))
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
||||
for chunk in (b"\x01\x02", b"\x03\x04\x05"):
|
||||
yield chunk
|
||||
|
||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||
speech_service.engine_client = MagicMock()
|
||||
speech_service.engine_client.abort = AsyncMock()
|
||||
|
||||
handler = OmniStreamingSpeechHandler(
|
||||
speech_service=speech_service,
|
||||
idle_timeout=idle_timeout,
|
||||
config_timeout=config_timeout,
|
||||
)
|
||||
app = FastAPI()
|
||||
|
||||
@app.websocket("/v1/audio/speech/stream")
|
||||
async def ws_endpoint(websocket: WebSocket):
|
||||
await handler.handle_session(websocket)
|
||||
|
||||
return app, speech_service
|
||||
|
||||
|
||||
class TestStreamingSpeechWebSocket:
|
||||
def test_non_streaming_single_frame(self):
|
||||
app, speech_service = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||
|
||||
start = ws.receive_json()
|
||||
assert start["type"] == "audio.start"
|
||||
assert start["sentence_index"] == 0
|
||||
assert start["sentence_text"] == "Hello world."
|
||||
assert start["format"] == "wav"
|
||||
|
||||
audio = ws.receive_bytes()
|
||||
assert audio.startswith(b"RIFF")
|
||||
|
||||
done = ws.receive_json()
|
||||
assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": len(audio), "error": False}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
session_done = ws.receive_json()
|
||||
assert session_done == {"type": "session.done", "total_sentences": 1}
|
||||
|
||||
assert speech_service._generate_audio_bytes.await_count == 1
|
||||
|
||||
def test_streaming_multiple_binary_frames(self):
|
||||
captured_requests = []
|
||||
|
||||
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
|
||||
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"", "audio/wav"))
|
||||
speech_service.engine_client = MagicMock()
|
||||
speech_service.engine_client.abort = AsyncMock()
|
||||
|
||||
async def mock_prepare_speech_generation(request):
|
||||
captured_requests.append(request)
|
||||
return "req-stream", object(), {}
|
||||
|
||||
speech_service._prepare_speech_generation = mock_prepare_speech_generation
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
||||
for chunk in (b"\x01\x02", b"\x03\x04\x05", b"\x06"):
|
||||
yield chunk
|
||||
|
||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||
app, _ = _build_test_app(speech_service)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"stream_audio": True,
|
||||
"response_format": "pcm",
|
||||
"initial_codec_chunk_frames": 12,
|
||||
}
|
||||
)
|
||||
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||
|
||||
start = ws.receive_json()
|
||||
assert start["type"] == "audio.start"
|
||||
assert start["format"] == "pcm"
|
||||
assert start["sample_rate"] == 24000
|
||||
|
||||
assert ws.receive_bytes() == b"\x01\x02"
|
||||
assert ws.receive_bytes() == b"\x03\x04\x05"
|
||||
assert ws.receive_bytes() == b"\x06"
|
||||
|
||||
done = ws.receive_json()
|
||||
assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": 6, "error": False}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
|
||||
|
||||
assert len(captured_requests) == 1
|
||||
assert captured_requests[0].stream is True
|
||||
assert captured_requests[0].response_format == "pcm"
|
||||
assert captured_requests[0].initial_codec_chunk_frames == 12
|
||||
assert speech_service._generate_audio_bytes.await_count == 0
|
||||
|
||||
def test_flush_on_input_done(self):
|
||||
app, _ = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": "Hello world without punctuation"})
|
||||
ws.send_json({"type": "input.done"})
|
||||
|
||||
assert ws.receive_json()["type"] == "audio.start"
|
||||
assert ws.receive_bytes()
|
||||
assert ws.receive_json() == {
|
||||
"type": "audio.done",
|
||||
"sentence_index": 0,
|
||||
"total_bytes": 36,
|
||||
"error": False,
|
||||
}
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
|
||||
|
||||
def test_invalid_streaming_config(self):
|
||||
app, _ = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"stream_audio": True,
|
||||
"response_format": "wav",
|
||||
}
|
||||
)
|
||||
error = ws.receive_json()
|
||||
assert error["type"] == "error"
|
||||
assert "response_format='pcm'" in error["message"]
|
||||
|
||||
def test_empty_input_text_emits_no_audio(self):
|
||||
app, speech_service = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": ""})
|
||||
ws.send_json({"type": "input.done"})
|
||||
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 0}
|
||||
|
||||
assert speech_service._generate_audio_bytes.await_count == 0
|
||||
|
||||
def test_multiple_sentences_increment_indices(self):
|
||||
app, _ = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": "First sentence. Second sentence. "})
|
||||
|
||||
first_start = ws.receive_json()
|
||||
assert first_start["sentence_index"] == 0
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json() == {
|
||||
"type": "audio.done",
|
||||
"sentence_index": 0,
|
||||
"total_bytes": 36,
|
||||
"error": False,
|
||||
}
|
||||
|
||||
second_start = ws.receive_json()
|
||||
assert second_start["sentence_index"] == 1
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json() == {
|
||||
"type": "audio.done",
|
||||
"sentence_index": 1,
|
||||
"total_bytes": 36,
|
||||
"error": False,
|
||||
}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 2}
|
||||
|
||||
def test_unknown_message_type_keeps_session_open(self):
|
||||
app, _ = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "unknown"})
|
||||
|
||||
error = ws.receive_json()
|
||||
assert error == {"type": "error", "message": "Unknown message type: unknown"}
|
||||
|
||||
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||
assert ws.receive_json()["type"] == "audio.start"
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json() == {
|
||||
"type": "audio.done",
|
||||
"sentence_index": 0,
|
||||
"total_bytes": 36,
|
||||
"error": False,
|
||||
}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
|
||||
|
||||
def test_config_timeout_closes_session(self):
|
||||
app, _ = _build_test_app(config_timeout=0.01)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
error = ws.receive_json()
|
||||
assert error == {"type": "error", "message": "Timeout waiting for session.config"}
|
||||
|
||||
def test_generation_error_marks_audio_done(self):
|
||||
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
|
||||
speech_service._generate_audio_bytes = AsyncMock(side_effect=RuntimeError("boom"))
|
||||
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-err", object(), {}))
|
||||
speech_service._generate_pcm_chunks = AsyncMock()
|
||||
speech_service.engine_client = MagicMock()
|
||||
speech_service.engine_client.abort = AsyncMock()
|
||||
app, _ = _build_test_app(speech_service)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||
|
||||
assert ws.receive_json()["type"] == "audio.start"
|
||||
assert ws.receive_json() == {"type": "error", "message": "Generation failed for sentence 0: boom"}
|
||||
assert ws.receive_json() == {"type": "audio.done", "sentence_index": 0, "total_bytes": 0, "error": True}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
|
||||
|
||||
def test_streaming_generation_error_marks_audio_done(self):
|
||||
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
|
||||
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"", "audio/wav"))
|
||||
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-stream-err", object(), {}))
|
||||
speech_service.engine_client = MagicMock()
|
||||
speech_service.engine_client.abort = AsyncMock()
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
||||
yield b"\x01\x02"
|
||||
raise RuntimeError("stream boom")
|
||||
|
||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||
app, _ = _build_test_app(speech_service)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"stream_audio": True,
|
||||
"response_format": "pcm",
|
||||
}
|
||||
)
|
||||
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||
|
||||
assert ws.receive_json()["type"] == "audio.start"
|
||||
assert ws.receive_bytes() == b"\x01\x02"
|
||||
assert ws.receive_json() == {
|
||||
"type": "error",
|
||||
"message": "Generation failed for sentence 0: stream boom",
|
||||
}
|
||||
assert ws.receive_json() == {
|
||||
"type": "audio.done",
|
||||
"sentence_index": 0,
|
||||
"total_bytes": 2,
|
||||
"error": True,
|
||||
}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
|
||||
|
||||
def test_invalid_input_text_type_returns_validation_error(self):
|
||||
app, speech_service = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": 123})
|
||||
|
||||
assert ws.receive_json() == {
|
||||
"type": "error",
|
||||
"message": "input.text requires a string value",
|
||||
}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 0}
|
||||
|
||||
assert speech_service._generate_audio_bytes.await_count == 0
|
||||
|
||||
def test_input_text_message_too_large(self, monkeypatch):
|
||||
monkeypatch.setattr(streaming_speech_module, "_MAX_INPUT_TEXT_MESSAGE_SIZE", 32)
|
||||
app, speech_service = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian"})
|
||||
ws.send_json({"type": "input.text", "text": "x" * 128})
|
||||
|
||||
assert ws.receive_json() == {
|
||||
"type": "error",
|
||||
"message": "input.text message too large",
|
||||
}
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {"type": "session.done", "total_sentences": 0}
|
||||
|
||||
assert speech_service._generate_audio_bytes.await_count == 0
|
||||
|
||||
def test_session_config_message_too_large(self, monkeypatch):
|
||||
monkeypatch.setattr(streaming_speech_module, "_MAX_CONFIG_MESSAGE_SIZE", 64)
|
||||
app, _ = _build_test_app()
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian", "ref_audio": "x" * 512})
|
||||
|
||||
assert ws.receive_json() == {
|
||||
"type": "error",
|
||||
"message": "session.config message too large",
|
||||
}
|
||||
|
||||
def test_disconnect_aborts_streaming_request(self):
|
||||
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
|
||||
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"", "audio/wav"))
|
||||
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-abort", object(), {}))
|
||||
speech_service.engine_client = MagicMock()
|
||||
speech_service.engine_client.abort = AsyncMock()
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
||||
yield b"\x01\x02"
|
||||
|
||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||
handler = OmniStreamingSpeechHandler(speech_service=speech_service)
|
||||
|
||||
websocket = MagicMock()
|
||||
websocket.send_json = AsyncMock(side_effect=[None, WebSocketDisconnect()])
|
||||
websocket.send_bytes = AsyncMock(side_effect=WebSocketDisconnect())
|
||||
|
||||
config = MagicMock()
|
||||
config.model = None
|
||||
config.voice = "Vivian"
|
||||
config.task_type = None
|
||||
config.language = None
|
||||
config.instructions = None
|
||||
config.response_format = "pcm"
|
||||
config.speed = 1.0
|
||||
config.max_new_tokens = None
|
||||
config.initial_codec_chunk_frames = None
|
||||
config.ref_audio = None
|
||||
config.ref_text = None
|
||||
config.x_vector_only_mode = None
|
||||
config.stream_audio = True
|
||||
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
asyncio.run(handler._generate_and_send(websocket, config, "Hello world.", 0))
|
||||
|
||||
speech_service.engine_client.abort.assert_awaited_once_with("req-abort")
|
||||
assert websocket.send_json.await_count == 2
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Tests for SentenceSplitter used in streaming TTS input."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.openai.text_splitter import SentenceSplitter
|
||||
|
||||
pytestmark = [pytest.mark.openai, pytest.mark.speech]
|
||||
|
||||
|
||||
class TestSentenceSplitterEnglish:
|
||||
"""Tests for English sentence splitting."""
|
||||
|
||||
def test_single_sentence_no_boundary(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Hello world")
|
||||
assert result == []
|
||||
assert splitter.buffer == "Hello world"
|
||||
|
||||
def test_single_sentence_with_boundary(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Hello world. How are you?")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "Hello world."
|
||||
|
||||
def test_multiple_sentences(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Hello. How are you? I am fine! ")
|
||||
assert len(result) == 3
|
||||
assert result[0] == "Hello."
|
||||
assert result[1] == "How are you?"
|
||||
assert result[2] == "I am fine!"
|
||||
|
||||
def test_exclamation_mark(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Wow, that is great! Tell me more.")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "Wow, that is great!"
|
||||
|
||||
def test_question_mark(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Can you hear me? I hope so.")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "Can you hear me?"
|
||||
|
||||
|
||||
class TestSentenceSplitterChinese:
|
||||
"""Tests for CJK sentence splitting."""
|
||||
|
||||
def test_chinese_period(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("你好世界。你好吗")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "你好世界。"
|
||||
|
||||
def test_chinese_exclamation(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("太好了!谢谢你")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "太好了!"
|
||||
|
||||
def test_chinese_question(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("你是谁?我是小明")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "你是谁?"
|
||||
|
||||
def test_chinese_comma_no_split(self):
|
||||
"""Chinese commas are clause-level and should not trigger a split."""
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("你好,世界")
|
||||
assert result == []
|
||||
assert splitter.buffer == "你好,世界"
|
||||
|
||||
def test_chinese_semicolon_no_split(self):
|
||||
"""Chinese semicolons are clause-level and should not trigger a split."""
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("第一点;第二点")
|
||||
assert result == []
|
||||
assert splitter.buffer == "第一点;第二点"
|
||||
|
||||
def test_chinese_multiple(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("你好!你好吗?我很好。")
|
||||
assert len(result) == 3
|
||||
assert result[0] == "你好!"
|
||||
assert result[1] == "你好吗?"
|
||||
assert result[2] == "我很好。"
|
||||
|
||||
|
||||
class TestSentenceSplitterMixed:
|
||||
"""Tests for mixed-language sentence splitting."""
|
||||
|
||||
def test_mixed_english_chinese(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Hello世界。How are you? ")
|
||||
assert len(result) == 2
|
||||
assert result[0] == "Hello世界。"
|
||||
assert result[1] == "How are you?"
|
||||
|
||||
|
||||
class TestSentenceSplitterIncremental:
|
||||
"""Tests for incremental (multi-chunk) text input."""
|
||||
|
||||
def test_accumulation_across_chunks(self):
|
||||
splitter = SentenceSplitter()
|
||||
# First chunk: no boundary
|
||||
result1 = splitter.add_text("Hello ")
|
||||
assert result1 == []
|
||||
|
||||
# Second chunk: completes a sentence
|
||||
result2 = splitter.add_text("world. How")
|
||||
assert len(result2) == 1
|
||||
assert result2[0] == "Hello world."
|
||||
assert splitter.buffer == "How"
|
||||
|
||||
def test_word_by_word(self):
|
||||
splitter = SentenceSplitter()
|
||||
words = ["Hello, ", "how ", "are ", "you? ", "I ", "am ", "fine."]
|
||||
all_sentences = []
|
||||
for word in words:
|
||||
all_sentences.extend(splitter.add_text(word))
|
||||
|
||||
assert len(all_sentences) == 1
|
||||
assert all_sentences[0] == "Hello, how are you?"
|
||||
# "I am fine." stays in buffer (no trailing whitespace after period)
|
||||
|
||||
def test_three_chunks(self):
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("The quick brown ")
|
||||
splitter.add_text("fox jumps. ")
|
||||
result = splitter.add_text("Over the lazy dog. ")
|
||||
# "The quick brown fox jumps." should have been returned on second chunk
|
||||
# "Over the lazy dog." on third chunk
|
||||
assert len(result) == 1
|
||||
assert result[0] == "Over the lazy dog."
|
||||
|
||||
|
||||
class TestSentenceSplitterFlush:
|
||||
"""Tests for flush behavior."""
|
||||
|
||||
def test_flush_returns_remaining(self):
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("Hello world")
|
||||
result = splitter.flush()
|
||||
assert result == "Hello world"
|
||||
assert splitter.buffer == ""
|
||||
|
||||
def test_flush_empty_buffer(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.flush()
|
||||
assert result is None
|
||||
|
||||
def test_flush_after_sentence(self):
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("Hello world. Remaining text")
|
||||
result = splitter.flush()
|
||||
assert result == "Remaining text"
|
||||
|
||||
def test_flush_whitespace_only(self):
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("Hello. ")
|
||||
# "Hello." extracted, buffer is " "
|
||||
result = splitter.flush()
|
||||
# Whitespace-only should return None
|
||||
assert result is None
|
||||
|
||||
def test_flush_clears_buffer(self):
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("some text")
|
||||
splitter.flush()
|
||||
assert splitter.buffer == ""
|
||||
# Second flush should return None
|
||||
assert splitter.flush() is None
|
||||
|
||||
|
||||
class TestSentenceSplitterEdgeCases:
|
||||
"""Edge case tests."""
|
||||
|
||||
def test_empty_input(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("")
|
||||
assert result == []
|
||||
assert splitter.buffer == ""
|
||||
|
||||
def test_none_like_empty(self):
|
||||
"""Empty string should not affect buffer."""
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("Hello")
|
||||
splitter.add_text("")
|
||||
assert splitter.buffer == "Hello"
|
||||
|
||||
def test_only_punctuation(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text(". ")
|
||||
# "." is 1 char, below default min_sentence_length of 2
|
||||
# It will be carried forward
|
||||
assert result == []
|
||||
|
||||
def test_min_sentence_length(self):
|
||||
splitter = SentenceSplitter(min_sentence_length=10)
|
||||
result = splitter.add_text("Hi. Hello world. ")
|
||||
# "Hi." is 3 chars (< 10), so it gets carried to "Hello world."
|
||||
assert len(result) == 1
|
||||
assert "Hi." in result[0]
|
||||
assert "Hello world." in result[0]
|
||||
|
||||
def test_short_segments_are_carried_until_long_enough(self):
|
||||
splitter = SentenceSplitter(min_sentence_length=10)
|
||||
result = splitter.add_text("Hi. Ok. Hello there. ")
|
||||
assert result == ["Hi.Ok.Hello there."]
|
||||
assert splitter.buffer == ""
|
||||
|
||||
def test_min_sentence_length_zero(self):
|
||||
splitter = SentenceSplitter(min_sentence_length=0)
|
||||
result = splitter.add_text("A. B. ")
|
||||
assert len(result) == 2
|
||||
|
||||
def test_no_boundary_then_flush(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Hello world how are you")
|
||||
assert result == []
|
||||
flushed = splitter.flush()
|
||||
assert flushed == "Hello world how are you"
|
||||
|
||||
def test_consecutive_punctuation(self):
|
||||
splitter = SentenceSplitter()
|
||||
result = splitter.add_text("Really?! Yes, really. ")
|
||||
assert len(result) >= 1
|
||||
|
||||
def test_reuse_after_flush(self):
|
||||
"""Splitter can be reused after flush."""
|
||||
splitter = SentenceSplitter()
|
||||
splitter.add_text("First session.")
|
||||
splitter.flush()
|
||||
|
||||
result = splitter.add_text("Second session. More text")
|
||||
assert len(result) == 1
|
||||
assert result[0] == "Second session."
|
||||
assert splitter.buffer == "More text"
|
||||
|
||||
|
||||
class TestSentenceSplitterBufferLimit:
|
||||
"""Tests for buffer overflow protection."""
|
||||
|
||||
def test_buffer_overflow_raises(self):
|
||||
from vllm_omni.entrypoints.openai.text_splitter import _MAX_BUFFER_SIZE
|
||||
|
||||
splitter = SentenceSplitter()
|
||||
# Fill buffer just under the limit
|
||||
splitter.add_text("x" * (_MAX_BUFFER_SIZE - 1))
|
||||
# One more char should exceed the limit
|
||||
with pytest.raises(ValueError, match="exceeded maximum size"):
|
||||
splitter.add_text("xx")
|
||||
@@ -345,7 +345,7 @@ def _setup_multiprocessing_mocks(monkeypatch: pytest.MonkeyPatch, mocker: Mocker
|
||||
fake_process_instance = mocker.MagicMock()
|
||||
fake_process_instance.start = mocker.MagicMock()
|
||||
fake_process_instance.join = mocker.MagicMock()
|
||||
fake_process_instance.is_alive = mocker.MagicMock(return_value=False)
|
||||
fake_process_instance.is_alive = mocker.MagicMock(return_value=True)
|
||||
fake_process_instance.terminate = mocker.MagicMock()
|
||||
fake_process_class.return_value = fake_process_instance
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import convert_audio_file_to_text, cosine_similarity_text
|
||||
from tests.conftest import OmniServerParams, convert_audio_file_to_text, cosine_similarity_text
|
||||
from tests.utils import hardware_test
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-7B"]
|
||||
@@ -22,7 +22,11 @@ stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_config
|
||||
|
||||
example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "qwen2_5_omni")
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(8091, model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, port=8091, stage_config_path=stage_config)
|
||||
for model in models
|
||||
for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
def run_cmd(command):
|
||||
|
||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import convert_audio_file_to_text, cosine_similarity_text
|
||||
from tests.conftest import OmniServerParams, convert_audio_file_to_text, cosine_similarity_text
|
||||
from tests.utils import hardware_test
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
@@ -22,7 +22,11 @@ stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_config
|
||||
|
||||
example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "qwen3_omni")
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(8091, model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, port=8091, stage_config_path=stage_config)
|
||||
for model in models
|
||||
for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
def run_cmd(command):
|
||||
|
||||
@@ -107,8 +107,14 @@ def omni_server(request):
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_prompt) -> Any:
|
||||
"""Generate synthetic image with random values."""
|
||||
def run_benchmark(
|
||||
args: list,
|
||||
test_name: str,
|
||||
flow,
|
||||
dataset_name: str,
|
||||
num_prompt,
|
||||
) -> Any:
|
||||
"""Run a single benchmark iteration and return the parsed result JSON."""
|
||||
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_{current_dt}.json"
|
||||
if "--result-filename" in args:
|
||||
@@ -117,10 +123,6 @@ def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_promp
|
||||
["vllm", "bench", "serve", "--omni"]
|
||||
+ args
|
||||
+ [
|
||||
"--backend",
|
||||
"openai-chat-omni",
|
||||
"--endpoint",
|
||||
"/v1/chat/completions",
|
||||
"--save-result",
|
||||
"--result-dir",
|
||||
os.environ.get("BENCHMARK_DIR", "tests"),
|
||||
@@ -196,7 +198,10 @@ def benchmark_params(request, omni_server):
|
||||
total = len(all_params)
|
||||
print(f"\n Running benchmark {current}/{total} for {test_name}")
|
||||
|
||||
return {"test_name": test_name, "params": all_params[param_index]}
|
||||
return {
|
||||
"test_name": test_name,
|
||||
"params": all_params[param_index],
|
||||
}
|
||||
|
||||
|
||||
def assert_result(result, params, num_prompt):
|
||||
@@ -266,7 +271,11 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
for qps, num_prompt in zip(qps_list, num_prompt_list):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
result = run_benchmark(
|
||||
args=args, test_name=test_name, flow=qps, dataset_name=dataset_name, num_prompt=num_prompt
|
||||
args=args,
|
||||
test_name=test_name,
|
||||
flow=qps,
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
)
|
||||
assert_result(result, params, num_prompt=num_prompt)
|
||||
|
||||
@@ -274,6 +283,10 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
|
||||
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
|
||||
result = run_benchmark(
|
||||
args=args, test_name=test_name, flow=concurrency, dataset_name=dataset_name, num_prompt=num_prompt
|
||||
args=args,
|
||||
test_name=test_name,
|
||||
flow=concurrency,
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
)
|
||||
assert_result(result, params, num_prompt=num_prompt)
|
||||
@@ -0,0 +1,99 @@
|
||||
# Stage config for running Qwen3-TTS with 2-stage architecture
|
||||
# Stage 0: Talker (text -> 8-layer RVQ codec codes)
|
||||
# Stage 1: Code2Wav (codec codes -> audio waveform)
|
||||
#
|
||||
# The following config has been verified on 1x H100-80G GPU.
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 4
|
||||
engine_args:
|
||||
model_stage: qwen3_tts
|
||||
model_arch: Qwen3TTSTalkerForConditionalGeneration
|
||||
hf_overrides:
|
||||
architectures: [Qwen3TTSTalkerForConditionalGeneration]
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.3
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 512
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 4
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3TTSCode2Wav
|
||||
hf_overrides:
|
||||
architectures: [Qwen3TTSCode2Wav]
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.2
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 32768
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
tts_args:
|
||||
max_instructions_length: 500
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1
|
||||
max_inflight: 4
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
window_size: -1
|
||||
@@ -8,6 +8,8 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -28,6 +30,8 @@
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -88,6 +92,8 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -108,6 +114,8 @@
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -140,5 +148,38 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen3_tts",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"stage_config_name": "qwen3_tts.yaml"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"extra_body": {
|
||||
"voice": "Vivian",
|
||||
"language": "English"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,570 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for StageConfigFactory and related classes.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
ModelPipeline,
|
||||
StageConfig,
|
||||
StageConfigFactory,
|
||||
StageType,
|
||||
)
|
||||
|
||||
|
||||
class TestStageType:
|
||||
"""Tests for StageType enum."""
|
||||
|
||||
def test_stage_type_values(self):
|
||||
"""Test StageType enum values."""
|
||||
assert StageType.LLM.value == "llm"
|
||||
assert StageType.DIFFUSION.value == "diffusion"
|
||||
|
||||
def test_stage_type_from_string(self):
|
||||
"""Test creating StageType from string."""
|
||||
assert StageType("llm") == StageType.LLM
|
||||
assert StageType("diffusion") == StageType.DIFFUSION
|
||||
|
||||
|
||||
class TestStageConfig:
|
||||
"""Tests for StageConfig dataclass."""
|
||||
|
||||
def test_minimal_config(self):
|
||||
"""Test creating StageConfig with minimal required fields."""
|
||||
config = StageConfig(stage_id=0, model_stage="thinker")
|
||||
assert config.stage_id == 0
|
||||
assert config.model_stage == "thinker"
|
||||
assert config.stage_type == StageType.LLM
|
||||
assert config.input_sources == []
|
||||
assert config.final_output is False
|
||||
assert config.worker_type is None
|
||||
|
||||
def test_full_config(self):
|
||||
"""Test creating StageConfig with all fields."""
|
||||
config = StageConfig(
|
||||
stage_id=1,
|
||||
model_stage="talker",
|
||||
stage_type=StageType.LLM,
|
||||
input_sources=[0],
|
||||
custom_process_input_func="module.path.func",
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
worker_type="ar",
|
||||
scheduler_cls="path.to.Scheduler",
|
||||
hf_config_name="talker_config",
|
||||
is_comprehension=False,
|
||||
)
|
||||
assert config.stage_id == 1
|
||||
assert config.model_stage == "talker"
|
||||
assert config.input_sources == [0]
|
||||
assert config.final_output_type == "audio"
|
||||
assert config.worker_type == "ar"
|
||||
|
||||
def test_to_omegaconf_basic(self):
|
||||
"""Test converting StageConfig to OmegaConf format."""
|
||||
config = StageConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
stage_type=StageType.LLM,
|
||||
worker_type="ar",
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
)
|
||||
omega_config = config.to_omegaconf()
|
||||
|
||||
assert omega_config.stage_id == 0
|
||||
assert omega_config.stage_type == "llm"
|
||||
assert omega_config.engine_args.model_stage == "thinker"
|
||||
assert omega_config.engine_args.worker_type == "ar"
|
||||
assert omega_config.final_output is True
|
||||
assert omega_config.final_output_type == "text"
|
||||
# Legacy field name for backward compatibility
|
||||
assert omega_config.engine_input_source == []
|
||||
|
||||
def test_to_omegaconf_with_runtime_overrides(self):
|
||||
"""Test that runtime overrides are applied to OmegaConf output."""
|
||||
config = StageConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
runtime_overrides={
|
||||
"gpu_memory_utilization": 0.9,
|
||||
"tensor_parallel_size": 2,
|
||||
"devices": "0,1",
|
||||
"max_batch_size": 64,
|
||||
},
|
||||
)
|
||||
omega_config = config.to_omegaconf()
|
||||
|
||||
assert omega_config.engine_args.gpu_memory_utilization == 0.9
|
||||
assert omega_config.engine_args.tensor_parallel_size == 2
|
||||
assert omega_config.runtime.devices == "0,1"
|
||||
assert omega_config.runtime.max_batch_size == 64
|
||||
|
||||
|
||||
class TestModelPipeline:
|
||||
"""Tests for ModelPipeline class."""
|
||||
|
||||
def test_valid_linear_dag(self):
|
||||
"""Test validation of a valid linear DAG."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="thinker", input_sources=[]),
|
||||
StageConfig(stage_id=1, model_stage="talker", input_sources=[0]),
|
||||
StageConfig(stage_id=2, model_stage="code2wav", input_sources=[1]),
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert errors == [], f"Unexpected errors: {errors}"
|
||||
|
||||
def test_valid_branching_dag(self):
|
||||
"""Test validation of a valid branching DAG."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="input", input_sources=[]),
|
||||
StageConfig(stage_id=1, model_stage="branch_a", input_sources=[0]),
|
||||
StageConfig(stage_id=2, model_stage="branch_b", input_sources=[0]),
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert errors == [], f"Unexpected errors: {errors}"
|
||||
|
||||
def test_missing_entry_point(self):
|
||||
"""Test that missing entry point is detected."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="stage_a", input_sources=[1]),
|
||||
StageConfig(stage_id=1, model_stage="stage_b", input_sources=[0]),
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert any("entry point" in e.lower() for e in errors)
|
||||
|
||||
def test_missing_dependency(self):
|
||||
"""Test that missing stage reference is detected."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="input", input_sources=[]),
|
||||
StageConfig(stage_id=1, model_stage="output", input_sources=[99]), # Invalid
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert any("non-existent" in e.lower() for e in errors)
|
||||
|
||||
def test_duplicate_stage_ids(self):
|
||||
"""Test that duplicate stage IDs are detected."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="stage_a", input_sources=[]),
|
||||
StageConfig(stage_id=0, model_stage="stage_b", input_sources=[]), # Duplicate
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert any("duplicate" in e.lower() for e in errors)
|
||||
|
||||
def test_self_reference(self):
|
||||
"""Test that self-references are detected."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="entry", input_sources=[]),
|
||||
StageConfig(stage_id=1, model_stage="self_ref", input_sources=[1]), # Self
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert any("itself" in e.lower() for e in errors)
|
||||
|
||||
def test_get_stage_by_id(self):
|
||||
"""Test getting stage by ID."""
|
||||
stages = [
|
||||
StageConfig(stage_id=0, model_stage="thinker", input_sources=[]),
|
||||
StageConfig(stage_id=1, model_stage="talker", input_sources=[0]),
|
||||
]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
|
||||
stage = pipeline.get_stage(1)
|
||||
assert stage is not None
|
||||
assert stage.model_stage == "talker"
|
||||
|
||||
missing = pipeline.get_stage(99)
|
||||
assert missing is None
|
||||
|
||||
def test_empty_pipeline(self):
|
||||
"""Test validation of empty pipeline."""
|
||||
pipeline = ModelPipeline(model_type="test", stages=[])
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert any("no stages" in e.lower() for e in errors)
|
||||
|
||||
|
||||
class TestStageConfigFactory:
|
||||
"""Tests for StageConfigFactory class."""
|
||||
|
||||
def test_default_diffusion_no_yaml(self):
|
||||
"""Test single-stage diffusion works without YAML config (@ZJY0516)."""
|
||||
kwargs = {
|
||||
"cache_backend": "none",
|
||||
"cache_config": None,
|
||||
"dtype": "bfloat16",
|
||||
}
|
||||
configs = StageConfigFactory.create_default_diffusion(kwargs)
|
||||
|
||||
assert len(configs) == 1
|
||||
cfg = configs[0]
|
||||
assert cfg["stage_id"] == 0
|
||||
assert cfg["stage_type"] == "diffusion"
|
||||
assert cfg["final_output"] is True
|
||||
assert cfg["final_output_type"] == "image"
|
||||
|
||||
def test_default_diffusion_with_parallel_config(self):
|
||||
"""Test diffusion config calculates devices from parallel_config."""
|
||||
|
||||
class MockParallelConfig:
|
||||
world_size = 4
|
||||
|
||||
kwargs = {
|
||||
"parallel_config": MockParallelConfig(),
|
||||
"cache_backend": "tea_cache",
|
||||
}
|
||||
configs = StageConfigFactory.create_default_diffusion(kwargs)
|
||||
|
||||
assert configs[0]["runtime"]["devices"] == "0,1,2,3"
|
||||
|
||||
def test_per_stage_override_precedence(self):
|
||||
"""Test that --stage-0-gpu-memory-utilization overrides global."""
|
||||
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
|
||||
cli_overrides = {
|
||||
"gpu_memory_utilization": 0.5, # Global
|
||||
"stage_0_gpu_memory_utilization": 0.9, # Per-stage override
|
||||
}
|
||||
|
||||
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
|
||||
|
||||
# Per-stage should override global
|
||||
assert overrides["gpu_memory_utilization"] == 0.9
|
||||
|
||||
def test_cli_override_forwards_engine_registered_args(self):
|
||||
"""Test that any engine-registered CLI arg is forwarded (@wuhang2014)."""
|
||||
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
|
||||
cli_overrides = {
|
||||
"gpu_memory_utilization": 0.9, # Well-known param
|
||||
"custom_engine_flag": True, # Not in _INTERNAL_KEYS, so forwarded
|
||||
}
|
||||
|
||||
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
|
||||
|
||||
assert overrides["gpu_memory_utilization"] == 0.9
|
||||
assert overrides["custom_engine_flag"] is True
|
||||
|
||||
def test_cli_override_excludes_internal_keys(self):
|
||||
"""Test that internal/orchestrator keys are not forwarded."""
|
||||
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
|
||||
cli_overrides = {
|
||||
"gpu_memory_utilization": 0.9,
|
||||
"model": "some_model", # Internal
|
||||
"stage_configs_path": "/path", # Internal
|
||||
"batch_timeout": 10, # Internal
|
||||
}
|
||||
|
||||
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
|
||||
|
||||
assert overrides["gpu_memory_utilization"] == 0.9
|
||||
assert "model" not in overrides
|
||||
assert "stage_configs_path" not in overrides
|
||||
assert "batch_timeout" not in overrides
|
||||
|
||||
def test_per_stage_override_excludes_internal_keys(self):
|
||||
"""Test that per-stage overrides also skip internal keys."""
|
||||
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
|
||||
cli_overrides = {
|
||||
"stage_0_gpu_memory_utilization": 0.9,
|
||||
"stage_0_model": "override_model", # Internal, should be skipped
|
||||
"stage_0_batch_timeout": 5, # Internal, should be skipped
|
||||
}
|
||||
|
||||
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
|
||||
|
||||
assert overrides["gpu_memory_utilization"] == 0.9
|
||||
assert "model" not in overrides
|
||||
assert "batch_timeout" not in overrides
|
||||
|
||||
|
||||
class TestPipelineYamlParsing:
|
||||
"""Tests for pipeline YAML file parsing (@ZJY0516)."""
|
||||
|
||||
def test_parse_qwen3_omni_moe_yaml(self, tmp_path):
|
||||
"""Test parsing the qwen3_omni_moe pipeline YAML."""
|
||||
yaml_content = """\
|
||||
model_type: qwen3_omni_moe
|
||||
async_chunk: false
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: thinker
|
||||
stage_type: llm
|
||||
input_sources: []
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
hf_config_name: thinker_config
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
|
||||
- stage_id: 1
|
||||
model_stage: talker
|
||||
stage_type: llm
|
||||
input_sources: [0]
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
hf_config_name: talker_config
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
|
||||
- stage_id: 2
|
||||
model_stage: code2wav
|
||||
stage_type: llm
|
||||
input_sources: [1]
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
hf_config_name: thinker_config
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
"""
|
||||
yaml_file = tmp_path / "qwen3_omni_moe.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "qwen3_omni_moe")
|
||||
|
||||
assert pipeline.model_type == "qwen3_omni_moe"
|
||||
assert len(pipeline.stages) == 3
|
||||
|
||||
# Stage 0: thinker
|
||||
s0 = pipeline.stages[0]
|
||||
assert s0.stage_id == 0
|
||||
assert s0.model_stage == "thinker"
|
||||
assert s0.stage_type == StageType.LLM
|
||||
assert s0.input_sources == []
|
||||
assert s0.worker_type == "ar"
|
||||
assert s0.final_output is True
|
||||
assert s0.final_output_type == "text"
|
||||
assert s0.is_comprehension is True
|
||||
|
||||
# Stage 1: talker
|
||||
s1 = pipeline.stages[1]
|
||||
assert s1.stage_id == 1
|
||||
assert s1.input_sources == [0]
|
||||
assert s1.custom_process_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker"
|
||||
)
|
||||
assert s1.final_output is False
|
||||
|
||||
# Stage 2: code2wav
|
||||
s2 = pipeline.stages[2]
|
||||
assert s2.stage_id == 2
|
||||
assert s2.input_sources == [1]
|
||||
assert s2.worker_type == "generation"
|
||||
assert s2.final_output_type == "audio"
|
||||
|
||||
def test_parse_yaml_with_legacy_engine_input_source(self, tmp_path):
|
||||
"""Test backward compatibility with engine_input_source field."""
|
||||
yaml_content = """\
|
||||
model_type: legacy_model
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: entry
|
||||
stage_type: llm
|
||||
- stage_id: 1
|
||||
model_stage: downstream
|
||||
stage_type: llm
|
||||
engine_input_source: [0]
|
||||
"""
|
||||
yaml_file = tmp_path / "legacy.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "legacy_model")
|
||||
assert pipeline.stages[1].input_sources == [0]
|
||||
|
||||
def test_parse_yaml_with_connectors_and_edges(self, tmp_path):
|
||||
"""Test parsing pipeline with optional connectors and edges."""
|
||||
yaml_content = """\
|
||||
model_type: test_model
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: entry
|
||||
stage_type: llm
|
||||
input_sources: []
|
||||
|
||||
connectors:
|
||||
type: ray
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
"""
|
||||
yaml_file = tmp_path / "with_connectors.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "test_model")
|
||||
assert pipeline.connectors == {"type": "ray"}
|
||||
assert pipeline.edges == [{"from": 0, "to": 1}]
|
||||
|
||||
def test_parsed_pipeline_passes_validation(self, tmp_path):
|
||||
"""Test that a well-formed YAML produces a valid pipeline."""
|
||||
yaml_content = """\
|
||||
model_type: valid_model
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: entry
|
||||
stage_type: llm
|
||||
input_sources: []
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
- stage_id: 1
|
||||
model_stage: next
|
||||
stage_type: llm
|
||||
input_sources: [0]
|
||||
"""
|
||||
yaml_file = tmp_path / "valid.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "valid_model")
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert errors == [], f"Unexpected validation errors: {errors}"
|
||||
|
||||
def test_parse_diffusion_stage_type(self, tmp_path):
|
||||
"""Test parsing a diffusion stage type from YAML."""
|
||||
yaml_content = """\
|
||||
model_type: diff_model
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: dit
|
||||
stage_type: diffusion
|
||||
input_sources: []
|
||||
final_output: true
|
||||
final_output_type: image
|
||||
"""
|
||||
yaml_file = tmp_path / "diffusion.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "diff_model")
|
||||
assert pipeline.stages[0].stage_type == StageType.DIFFUSION
|
||||
|
||||
def test_parse_mimo_audio_yaml(self, tmp_path):
|
||||
"""Test parsing the MiMo Audio pipeline YAML."""
|
||||
yaml_content = """\
|
||||
model_type: mimo_audio
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: fused_thinker_talker
|
||||
stage_type: llm
|
||||
input_sources: []
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
|
||||
- stage_id: 1
|
||||
model_stage: code2wav
|
||||
stage_type: llm
|
||||
input_sources: [0]
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
"""
|
||||
yaml_file = tmp_path / "mimo_audio.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "mimo_audio")
|
||||
|
||||
assert pipeline.model_type == "mimo_audio"
|
||||
assert len(pipeline.stages) == 2
|
||||
|
||||
# Stage 0: fused_thinker_talker
|
||||
s0 = pipeline.stages[0]
|
||||
assert s0.stage_id == 0
|
||||
assert s0.model_stage == "fused_thinker_talker"
|
||||
assert s0.input_sources == []
|
||||
assert s0.worker_type == "ar"
|
||||
assert s0.is_comprehension is True
|
||||
assert s0.final_output is True
|
||||
assert s0.final_output_type == "text"
|
||||
|
||||
# Stage 1: code2wav
|
||||
s1 = pipeline.stages[1]
|
||||
assert s1.stage_id == 1
|
||||
assert s1.input_sources == [0]
|
||||
assert s1.worker_type == "generation"
|
||||
assert s1.custom_process_input_func == (
|
||||
"vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav"
|
||||
)
|
||||
assert s1.final_output is True
|
||||
assert s1.final_output_type == "audio"
|
||||
|
||||
# Pipeline validation
|
||||
errors = pipeline.validate_pipeline()
|
||||
assert errors == [], f"Unexpected errors: {errors}"
|
||||
|
||||
|
||||
class TestAsyncChunk:
|
||||
"""Tests for async_chunk pipeline flag."""
|
||||
|
||||
def test_model_pipeline_async_chunk_default(self):
|
||||
"""Test ModelPipeline defaults async_chunk to False."""
|
||||
stages = [StageConfig(stage_id=0, model_stage="entry", input_sources=[])]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages)
|
||||
assert pipeline.async_chunk is False
|
||||
|
||||
def test_model_pipeline_async_chunk_set(self):
|
||||
"""Test ModelPipeline with async_chunk=True."""
|
||||
stages = [StageConfig(stage_id=0, model_stage="entry", input_sources=[])]
|
||||
pipeline = ModelPipeline(model_type="test", stages=stages, async_chunk=True)
|
||||
assert pipeline.async_chunk is True
|
||||
|
||||
def test_parse_async_chunk_from_yaml(self, tmp_path):
|
||||
"""Test that async_chunk is parsed from pipeline YAML."""
|
||||
yaml_content = """\
|
||||
model_type: test_async
|
||||
async_chunk: true
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: entry
|
||||
stage_type: llm
|
||||
input_sources: []
|
||||
"""
|
||||
yaml_file = tmp_path / "async.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "test_async")
|
||||
assert pipeline.async_chunk is True
|
||||
|
||||
def test_parse_missing_async_chunk_defaults_false(self, tmp_path):
|
||||
"""Test that missing async_chunk defaults to False."""
|
||||
yaml_content = """\
|
||||
model_type: test_no_async
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
model_stage: entry
|
||||
stage_type: llm
|
||||
input_sources: []
|
||||
"""
|
||||
yaml_file = tmp_path / "no_async.yaml"
|
||||
yaml_file.write_text(yaml_content)
|
||||
|
||||
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "test_no_async")
|
||||
assert pipeline.async_chunk is False
|
||||
|
||||
|
||||
class TestArchitectureFallback:
|
||||
"""Tests for architecture-based model detection fallback."""
|
||||
|
||||
def test_architecture_models_mapping_exists(self):
|
||||
"""Test that _ARCHITECTURE_MODELS contains expected entries."""
|
||||
assert "MiMoAudioForConditionalGeneration" in StageConfigFactory._ARCHITECTURE_MODELS
|
||||
assert StageConfigFactory._ARCHITECTURE_MODELS["MiMoAudioForConditionalGeneration"] == "mimo_audio"
|
||||
assert "HunyuanImage3ForCausalMM" in StageConfigFactory._ARCHITECTURE_MODELS
|
||||
assert StageConfigFactory._ARCHITECTURE_MODELS["HunyuanImage3ForCausalMM"] == "hunyuan_image3"
|
||||
|
||||
def test_mimo_audio_in_pipeline_models(self):
|
||||
"""Test that mimo_audio is registered in PIPELINE_MODELS."""
|
||||
assert "mimo_audio" in StageConfigFactory.PIPELINE_MODELS
|
||||
+39
-27
@@ -20,6 +20,8 @@ from typing_extensions import ParamSpec
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import cuda_device_count_stateless
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
if current_platform.is_rocm():
|
||||
@@ -441,27 +443,11 @@ def npu_marks(*, res: str, num_cards: int):
|
||||
return [mark for mark in [test_platform, test_resource, test_distributed] if mark is not None]
|
||||
|
||||
|
||||
def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
"""
|
||||
Decorate a test for multiple hardware platforms with a single call.
|
||||
Automatically wraps the test with @create_new_process_for_each_test() for distributed tests.
|
||||
|
||||
Args:
|
||||
res: Mapping from platform to resource type. Supported platforms/resources:
|
||||
- cuda: L4, H100
|
||||
- rocm: MI325
|
||||
- npu: A2, A3
|
||||
num_cards: Number of cards required. Can be:
|
||||
- int: same card count for all platforms (default: 1)
|
||||
- dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2}
|
||||
|
||||
Example:
|
||||
@hardware_test(
|
||||
res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
|
||||
num_cards={"cuda": 2, "rocm": 2, "npu": 2},
|
||||
)
|
||||
def test_multi_platform():
|
||||
...
|
||||
Get a collection of pytest marks to apply for `@hardware_test`,
|
||||
including CUDA, ROCm, and NPU,
|
||||
based on the specified platforms and resources.
|
||||
"""
|
||||
# Validate platforms
|
||||
# Don't validate platform details in this decorator
|
||||
@@ -484,7 +470,7 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
num_cards_dict[platform] = 1
|
||||
|
||||
# Collect marks from all platforms
|
||||
all_marks: list[Callable[[Callable[_P, None]], Callable[_P, None]]] = []
|
||||
all_marks: list[pytest.MarkDecorator] = []
|
||||
for platform, resource in res.items():
|
||||
cards = num_cards_dict[platform]
|
||||
if platform == "cuda" or platform == "rocm":
|
||||
@@ -494,6 +480,32 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
else:
|
||||
raise ValueError(f"Unsupported platform: {platform}")
|
||||
all_marks.extend(marks)
|
||||
return all_marks
|
||||
|
||||
|
||||
def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
"""
|
||||
Decorate a test for multiple hardware platforms with a single call.
|
||||
Automatically wraps the test with @create_new_process_for_each_test() for distributed tests.
|
||||
|
||||
Args:
|
||||
res: Mapping from platform to resource type. Supported platforms/resources:
|
||||
- cuda: L4, H100
|
||||
- rocm: MI325
|
||||
- npu: A2, A3
|
||||
num_cards: Number of cards required. Can be:
|
||||
- int: same card count for all platforms (default: 1)
|
||||
- dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2}
|
||||
|
||||
Example:
|
||||
@hardware_test(
|
||||
res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
|
||||
num_cards={"cuda": 2, "rocm": 2, "npu": 2},
|
||||
)
|
||||
def test_multi_platform():
|
||||
...
|
||||
"""
|
||||
all_marks = hardware_marks(res=res, num_cards=num_cards)
|
||||
|
||||
def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
|
||||
func = f
|
||||
@@ -504,8 +516,8 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
return wrapper
|
||||
|
||||
|
||||
class GPUMemoryMonitor:
|
||||
"""Poll global device memory usage via CUDA APIs."""
|
||||
class DeviceMemoryMonitor:
|
||||
"""Poll global device memory usage."""
|
||||
|
||||
def __init__(self, device_index: int, interval: float = 0.05):
|
||||
self.device_index = device_index
|
||||
@@ -518,8 +530,8 @@ class GPUMemoryMonitor:
|
||||
def monitor_loop() -> None:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
with torch.cuda.device(self.device_index):
|
||||
free_bytes, total_bytes = torch.cuda.mem_get_info()
|
||||
with current_omni_platform.device(self.device_index):
|
||||
free_bytes, total_bytes = current_omni_platform.mem_get_info()
|
||||
used_mb = (total_bytes - free_bytes) / (1024**2)
|
||||
self._peak_used_mb = max(self._peak_used_mb, used_mb)
|
||||
except Exception:
|
||||
@@ -537,8 +549,8 @@ class GPUMemoryMonitor:
|
||||
|
||||
@property
|
||||
def peak_used_mb(self) -> float:
|
||||
fallback_alloc = torch.cuda.max_memory_allocated(device=self.device_index) / (1024**2)
|
||||
fallback_reserved = torch.cuda.max_memory_reserved(device=self.device_index) / (1024**2)
|
||||
fallback_alloc = current_omni_platform.max_memory_allocated(device=self.device_index) / (1024**2)
|
||||
fallback_reserved = current_omni_platform.max_memory_reserved(device=self.device_index) / (1024**2)
|
||||
return max(self._peak_used_mb, fallback_alloc, fallback_reserved)
|
||||
|
||||
def __del__(self):
|
||||
|
||||
Reference in new issue
Block a user