Merge branch 'main' into dev/vllm-align

This commit is contained in:
tzhouam committed 2026-03-13 06:51:22 +00:00
commit d3db476f2d
144 files changed
+17247 -1024

No files matched your search

+4 -1
View File
@@ -3,13 +3,16 @@ from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
models = ["Qwen/Qwen2.5-Omni-7B"]
stage_configs = [str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen2_5_omni_ci.yaml")]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
test_params = [
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
]
@pytest.mark.core_model
+7
View File
@@ -63,6 +63,12 @@ def _setup_comfyui_test_environment():
mock_comfy_api_input.AudioInput = AudioInput
mock_comfy_api_input.VideoInput = VideoInput
mock_comfy_api.input = mock_comfy_api_input
mock_comfy_api_latest = MagicMock()
mock_comfy_api_latest.Types.VideoComponents = MagicMock(side_effect=lambda **kwargs: kwargs)
mock_comfy_api_latest.InputImpl.VideoFromComponents = MagicMock(
side_effect=lambda _: VideoInput(b"mock_video_from_components")
)
mock_comfy_api.latest = mock_comfy_api_latest
def mock_load(_: str | BinaryIO):
"""Mock nodes_audio.load that returns a waveform tensor (channels, samples) and sample rate."""
@@ -78,5 +84,6 @@ def _setup_comfyui_test_environment():
# Install mock modules BEFORE importing any comfyui_vllm_omni code
sys.modules["comfy_api"] = mock_comfy_api
sys.modules["comfy_api.input"] = mock_comfy_api_input
sys.modules["comfy_api.latest"] = mock_comfy_api_latest
sys.modules["comfy_extras"] = mock_comfy_extras
sys.modules["comfy_extras.nodes_audio"] = mock_nodes_audio
+165 -6
View File
@@ -20,11 +20,12 @@ import torch
from comfy_api.input import AudioInput, VideoInput
from comfyui_vllm_omni.nodes import (
VLLMOmniGenerateImage,
VLLMOmniGenerateVideo,
VLLMOmniTTS,
VLLMOmniUnderstanding,
VLLMOmniVoiceClone,
)
from comfyui_vllm_omni.utils.types import AutoregressionSamplingParams, DiffusionSamplingParams
from comfyui_vllm_omni.utils.types import AutoregressionSamplingParams, DiffusionSamplingParams, WanModelSpecificParams
from PIL import Image
from vllm import SamplingParams
from vllm.outputs import CompletionOutput, RequestOutput
@@ -34,6 +35,8 @@ from vllm_omni.entrypoints.cli.serve import OmniServeCommand
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class ServerCase(NamedTuple):
"""Parametrizing the model to serve."""
@@ -49,6 +52,7 @@ class SamplingCase(NamedTuple):
kind: "SamplingKind"
sampling_params: dict | list[dict] | None
lora: dict | None = None
class SamplingKind(StrEnum):
@@ -58,11 +62,17 @@ class SamplingKind(StrEnum):
UNDERSTANDING_AR_LIST = auto()
TTS_NONE = auto()
TTS_DIFFUSION_SINGLE = auto()
VIDEO_NONE = auto()
VIDEO_DIFFUSION_SINGLE = auto()
# Pre-defined arguments to be used in function calls during the tests
IMAGE_WIDTH = 64
IMAGE_HEIGHT = 64
VIDEO_WIDTH = 32
VIDEO_HEIGHT = 32
VIDEO_FPS = 8
VIDEO_NUM_FRAMES = 5
DIFFUSION_SINGLE_SAMPLING_PARAMS = DiffusionSamplingParams(
{
"n": 2,
@@ -102,6 +112,10 @@ AR_LIST_SAMPLING_PARAMS = [
),
]
VIDEO_MODEL_PARAMS = WanModelSpecificParams({"guidance_scale_2": 5.0, "boundary_ratio": 0.98, "flow_shift": 12.0})
LORA_PARAMS = {"local_path": "test_lora_path", "name": "test_name", "scale": 0.7, "int_id": 10}
def _build_image_output(size: tuple[int, int] = (IMAGE_WIDTH, IMAGE_HEIGHT), color: str = "red") -> Image.Image:
return Image.new("RGB", size, color=color)
@@ -181,6 +195,16 @@ def _build_diffusion_image_output_for_images_endpoint() -> OmniRequestOutput:
)
def _build_diffusion_video_output() -> OmniRequestOutput:
# Small video: VIDEO_NUM_FRAMES frames of (VIDEO_HEIGHT x VIDEO_WIDTH) RGB, shape (F, H, W, C)
video_frames = torch.zeros((VIDEO_NUM_FRAMES, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
return OmniRequestOutput.from_diffusion(
request_id="test_req_video",
images=[video_frames],
final_output_type="video",
)
def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
request_output = MagicMock()
request_output.images = [_build_image_output(color="blue")]
@@ -193,12 +217,40 @@ def _build_diffusion_image_output_for_chat_endpoint() -> OmniRequestOutput:
)
def _assert_sampling_param_values(received: OmniSamplingParams, expected: dict[str, Any]):
def _assert_sampling_param_values(
received: OmniSamplingParams, expected: dict[str, Any], expected_lora: dict | None = None
):
for key, expected_value in expected.items():
actual_value = getattr(received, key, None)
actual_value = getattr(received, key)
assert actual_value == expected_value, (
f"Expected sampling param '{key}'={expected_value}, got {actual_value}. The received sampling params: {received}"
)
if expected_lora:
assert received.lora_request.lora_name == expected_lora["name"], (
f"Expected lora name={(expected_lora['name'])}, got {received.lora_request.lora_name}. The received sampling params: {received}"
)
assert received.lora_request.lora_int_id == expected_lora["int_id"], (
f"Expected lora int_id={expected_lora['int_id']}, got {received.lora_request.lora_int_id}. The received sampling params: {received}"
)
assert received.lora_request.lora_path == expected_lora["local_path"], (
f"Expected lora path={expected_lora['local_path']}, got {received.lora_request.lora_path}. The received sampling params: {received}"
)
assert received.lora_scale == expected_lora["scale"], (
f"Expected lora scale={expected_lora['scale']}, got {received.lora_scale}. The received sampling params: {received}"
)
def _assert_model_param_values(received: OmniSamplingParams, expected: dict):
for key, expected_value in expected.items():
try:
actual_value = getattr(received, key)
expected_param_name = key
except AttributeError:
actual_value = received.extra_args.get(key, None)
expected_param_name = f'extra_args["{key}"]'
assert actual_value == expected_value, (
f"Expected model param '{expected_param_name}'={expected_value}, got {actual_value}. The received sampling params: {received}"
)
def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: SamplingCase, server_case: ServerCase):
@@ -235,6 +287,7 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam
"height": IMAGE_HEIGHT,
**expected,
},
LORA_PARAMS,
)
elif sampling_case.kind is SamplingKind.UNDERSTANDING_NONE:
assert len(received_sampling_params_list) == 3
@@ -244,6 +297,33 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam
_assert_sampling_param_values(received_sampling_params_list[i], expected)
elif sampling_case.kind in {SamplingKind.TTS_NONE, SamplingKind.TTS_DIFFUSION_SINGLE}:
assert len(received_sampling_params_list) == 1
elif sampling_case.kind is SamplingKind.VIDEO_NONE:
assert len(received_sampling_params_list) == 1
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": VIDEO_WIDTH,
"height": VIDEO_HEIGHT,
"num_frames": VIDEO_NUM_FRAMES,
"fps": VIDEO_FPS,
},
)
elif sampling_case.kind is SamplingKind.VIDEO_DIFFUSION_SINGLE:
assert len(received_sampling_params_list) == 1
expected = DIFFUSION_SINGLE_SAMPLING_PARAMS.copy()
expected["num_outputs_per_prompt"] = expected.pop("n") # convert from n to num_outputs_per_prompt
_assert_sampling_param_values(
received_sampling_params_list[0],
{
"width": VIDEO_WIDTH,
"height": VIDEO_HEIGHT,
"num_frames": VIDEO_NUM_FRAMES,
"fps": VIDEO_FPS,
**expected,
},
LORA_PARAMS,
)
_assert_model_param_values(received_sampling_params_list[0], VIDEO_MODEL_PARAMS)
else:
raise AssertionError(f"Unknown sampling case: {sampling_case.kind}")
@@ -401,9 +481,15 @@ def api_server(unused_tcp_port_factory, server_case: ServerCase, mock_async_omni
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.IMAGE_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(kind=SamplingKind.IMAGE_DIFFUSION_SINGLE, sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS),
SamplingCase(kind=SamplingKind.IMAGE_NONE, sampling_params=None, lora=None), id="no-sampling-params"
),
pytest.param(
SamplingCase(
kind=SamplingKind.IMAGE_DIFFUSION_SINGLE,
sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS,
lora=LORA_PARAMS,
),
id="single-diffusion-sampling-params",
),
],
@@ -423,7 +509,8 @@ async def test_image_generation_node(api_server: str, model: str, image_input: b
kwargs["image"] = torch.zeros((1, IMAGE_WIDTH, IMAGE_HEIGHT, 3), dtype=torch.float32)
if sampling_case.sampling_params is not None:
kwargs["sampling_params"] = sampling_case.sampling_params
print(f"!!!!!! Calling {model} node.generate with kwargs: {sampling_case.sampling_params}")
if sampling_case.lora:
kwargs["lora"] = sampling_case.lora
result = await node.generate(**kwargs)
@@ -565,3 +652,75 @@ async def test_tts_nodes(api_server: str, node_cls, call_kwargs: dict, sampling_
assert result[0]["sample_rate"] == 24000
assert isinstance(result[0]["waveform"], torch.Tensor)
assert result[0]["waveform"].shape == (1, 1, 24000)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server_case,model,image_input",
[
pytest.param(
ServerCase(
served_model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_video_output()],
),
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
False,
id="text-to-video",
),
pytest.param(
ServerCase(
served_model="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
stage_list=["diffusion"],
stage_configs=[{"stage_type": "diffusion"}],
outputs=[_build_diffusion_video_output()],
),
"Wan-AI/Wan2.2-I2V-A14B-Diffusers",
True,
id="image-to-video",
),
],
indirect=["server_case"],
)
@pytest.mark.parametrize(
"sampling_case",
[
pytest.param(SamplingCase(kind=SamplingKind.VIDEO_NONE, sampling_params=None), id="no-sampling-params"),
pytest.param(
SamplingCase(
kind=SamplingKind.VIDEO_DIFFUSION_SINGLE,
sampling_params=DIFFUSION_SINGLE_SAMPLING_PARAMS,
lora=LORA_PARAMS,
),
id="single-diffusion-sampling-params",
),
],
indirect=["sampling_case"],
)
async def test_video_generation_node(api_server: str, model: str, image_input: bool, sampling_case: SamplingCase):
node = VLLMOmniGenerateVideo()
kwargs = {
"url": api_server,
"model": model,
"prompt": "A beautiful sunset timelapse",
"negative_prompt": "",
"width": VIDEO_WIDTH,
"height": VIDEO_HEIGHT,
"fps": VIDEO_FPS,
"num_frames": VIDEO_NUM_FRAMES,
"model_params": VIDEO_MODEL_PARAMS,
}
if image_input:
kwargs["image"] = torch.zeros((1, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
if sampling_case.sampling_params is not None:
kwargs["sampling_params"] = sampling_case.sampling_params
if sampling_case.lora:
kwargs["lora"] = sampling_case.lora
result = await node.generate(**kwargs)
assert isinstance(result, tuple)
assert len(result) == 1
assert isinstance(result[0], VideoInput)
+244 -19
View File
@@ -18,16 +18,21 @@ import subprocess
import sys
import threading
import time
from collections.abc import Generator
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
from typing import Any
from typing import Any, NamedTuple
import imageio.v3 as iio
import numpy as np
import psutil
import pytest
import soundfile as sf
import torch
import yaml
from openai import OpenAI
from openai import OpenAI, omit
from PIL import Image
from vllm import TextPrompt
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm.logger import init_logger
@@ -44,6 +49,65 @@ PromptImageInput = list[Any] | Any | None
PromptVideoInput = list[Any] | Any | None
class OmniServerParams(NamedTuple):
model: str
port: int | None = None
stage_config_path: str | None = None
server_args: list[str] | None = None
def assert_image_valid(image: Path | Image.Image, *, width: int | None = None, height: int | None = None):
"""Assert the file is a loadable image with optional exact dimensions."""
if isinstance(image, Path):
assert image.exists(), f"Image not found: {image}"
image = Image.open(image)
image.load()
assert image.width > 0 and image.height > 0
if width is not None:
assert image.width == width, f"Expected width={width}, got {image.width} in {image.name}"
if height is not None:
assert image.height == height, f"Expected height={height}, got {image.height} in {image.name}"
return image
def assert_video_valid(frames: Path | np.ndarray, *, width: int, height: int, num_frames: int) -> None:
"""Assert the MP4 has the expected resolution and exact frame count."""
if isinstance(frames, Path):
assert frames.exists(), f"Video not found: {frames}"
frames = iio.imread(str(frames), plugin="pyav", index=None)
assert frames.shape[0] == num_frames, f"Expected {num_frames} frames, got {frames.shape[0]}"
assert frames.shape[1] == height, f"Expected height={height}, got {frames.shape[1]}"
assert frames.shape[2] == width, f"Expected width={width}, got {frames.shape[2]}"
def assert_audio_valid(path: Path, *, sample_rate: int, channels: int, duration_s: float) -> None:
"""Assert the WAV has the expected sample rate, channel count, and duration."""
assert path.exists(), f"Audio not found: {path}"
info = sf.info(str(path))
assert info.samplerate == sample_rate, f"Expected sample_rate={sample_rate}, got {info.samplerate}"
assert info.channels == channels, f"Expected {channels} channel(s), got {info.channels}"
expected_frames = int(duration_s * sample_rate)
assert info.frames == expected_frames, (
f"Expected {expected_frames} frames ({duration_s}s @ {sample_rate} Hz), got {info.frames}"
)
def decode_b64_image(b64: str):
img = Image.open(BytesIO(base64.b64decode(b64)))
img.load()
return img
@pytest.fixture(scope="session")
def model_prefix() -> str:
"""Optional model-path prefix from MODEL_PREFIX env var.
Useful if models are downloaded to non-default local directories.
"""
prefix = os.environ.get("MODEL_PREFIX", "")
return f"{prefix.rstrip('/')}/" if prefix else ""
@pytest.fixture(autouse=True)
def default_vllm_config():
"""Set a default VllmConfig for all tests.
@@ -1151,7 +1215,9 @@ def pytest_addoption(parser):
@pytest.fixture(scope="session")
def run_level(request):
def run_level(request) -> str:
"""A command-line argument that specifies the level of tests to run in this session.
See https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/CI_5levels/"""
return request.config.getoption("--run-level")
@@ -1159,15 +1225,17 @@ _omni_server_lock = threading.Lock()
@pytest.fixture(scope="module")
def omni_server(request, run_level):
def omni_server(request: pytest.FixtureRequest, run_level: str, model_prefix: str) -> Generator[OmniServer, Any, None]:
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
*port, model, stage_config_path = request.param
port = port[0] if port else None
if run_level == "advanced_model":
params: OmniServerParams = request.param
model = model_prefix + params.model
port = params.port
stage_config_path = params.stage_config_path
if run_level == "advanced_model" and stage_config_path is not None:
stage_config_path = modify_stage_config(
stage_config_path,
deletes={
@@ -1179,7 +1247,11 @@ def omni_server(request, run_level):
},
)
server_args = ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]
server_args = params.server_args or []
server_args = ["--stage-init-timeout", "120", *server_args]
if stage_config_path is not None:
server_args += ["--stage-configs-path", stage_config_path]
with OmniServer(model, server_args, port=port) if port else OmniServer(model, server_args) as server:
print("OmniServer started successfully")
yield server
@@ -1199,6 +1271,17 @@ class OmniResponse:
error_message: str | None = None
@dataclass
class DiffusionResponse:
text_content: str | None = None
images: list[Image.Image] | None = None
audios: list[Any] | None = None
videos: list[Any] | None = None
e2e_latency: float | None = None
success: bool = False
error_message: str | None = None
def assert_omni_response(response: OmniResponse, request_config: dict[str, Any], run_level):
"""
Validate response results.
@@ -1247,6 +1330,51 @@ def assert_omni_response(response: OmniResponse, request_config: dict[str, Any],
print(f"similarity is: {response.similarity}")
def assert_diffusion_response(response: DiffusionResponse, request_config: dict[str, Any], run_level: str = None):
"""
Validate diffusion response results.
Args:
response: DiffusionResponse object. Any not-None content will be validated based on the request_config.
request_config: Request configuration dictionary containing parameters like model, messages, extra_body.
When validating a certain modality, the corresponding params in request_config['extra_body'] must present.
It will be used to check against the multimedia file in the response.
run_level: Test run level (e.g., "core_model", "advanced_model")
Raises:
AssertionError: When the response does not meet validation criteria
KeyError: When the request_config does not contain necessary parameters for validation
"""
assert response.success, "The request failed."
e2e_latency = response.e2e_latency
if e2e_latency is not None:
print(f"the avg e2e is: {e2e_latency}")
extra_body = request_config.get("extra_body", {})
num_outputs_per_prompt = extra_body.get("num_outputs_per_prompt", 1)
if response.images is not None:
assert len(response.images) > 0, "No images in response"
assert len(response.images) == num_outputs_per_prompt, (
f"Expected {num_outputs_per_prompt} images, got {len(response.images)}"
)
if run_level == "advanced_model":
expected_width = extra_body["width"] # intend to raise KeyError
expected_height = extra_body["height"] # intend to raise KeyError
for img in response.images:
assert_image_valid(img, width=expected_width, height=expected_height)
if response.videos is not None:
raise NotImplementedError(
"Video validation is not implemented yet"
) # consider using assert_video_valid defined above
if response.audios is not None:
raise NotImplementedError(
"Audio validation is not implemented yet"
) # consider using assert_audio_valid defined above
class OpenAIClientHandler:
"""
OpenAI client handler class, encapsulating both streaming and non-streaming response processing logic.
@@ -1269,7 +1397,7 @@ class OpenAIClientHandler:
self.client = OpenAI(base_url=f"http://{host}:{port}/v1", api_key=api_key)
self.run_level = run_level
def _process_stream_response(self, chat_completion) -> OmniResponse:
def _process_stream_omni_response(self, chat_completion) -> OmniResponse:
"""
Process streaming responses.
@@ -1330,7 +1458,7 @@ class OpenAIClientHandler:
return result
def _process_non_stream_response(self, chat_completion) -> OmniResponse:
def _process_non_stream_omni_response(self, chat_completion) -> OmniResponse:
"""
Process non-streaming responses.
@@ -1384,7 +1512,46 @@ class OpenAIClientHandler:
return result
def send_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]:
def _process_diffusion_response(self, chat_completion) -> DiffusionResponse:
"""
Process diffusion responses (image generation/editing).
Args:
chat_completion: OpenAI response object
Returns:
DiffusionResponse: Processed response object
"""
result = DiffusionResponse()
start_time = time.perf_counter()
try:
images = []
# [TODO] reading video and audio output from API response for later validation
for choice in chat_completion.choices:
if hasattr(choice.message, "content") and choice.message.content is not None:
content = choice.message.content
if isinstance(content, list):
for item in content:
if hasattr(item, "image_url") and item.image_url is not None:
image_url = item.image_url.url
if image_url.startswith("data:image"):
b64_data = image_url.split(",", 1)[1]
img = decode_b64_image(b64_data)
images.append(img)
result.e2e_latency = time.perf_counter() - start_time
result.images = images if images else None
result.success = True
except Exception as e:
result.error_message = f"Diffusion response processing error: {str(e)}"
print(f"Error: {result.error_message}")
return result
def send_omni_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]:
"""
Send OpenAI requests.
@@ -1410,9 +1577,9 @@ class OpenAIClientHandler:
)
if stream:
response = self._process_stream_response(chat_completion)
response = self._process_stream_omni_response(chat_completion)
else:
response = self._process_non_stream_response(chat_completion)
response = self._process_non_stream_omni_response(chat_completion)
assert_omni_response(response, request_config, run_level=self.run_level)
responses.append(response)
@@ -1438,25 +1605,82 @@ class OpenAIClientHandler:
chat_completion = future.result()
if stream:
response = self._process_stream_response(chat_completion)
response = self._process_stream_omni_response(chat_completion)
else:
response = self._process_non_stream_response(chat_completion)
response = self._process_non_stream_omni_response(chat_completion)
assert_omni_response(response, request_config, run_level=self.run_level)
responses.append(response)
return responses
def send_diffusion_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]:
"""
Send OpenAI requests for diffusion models.
Args:
request_config: Request configuration dictionary containing parameters like model, messages
request_num: Number of requests to send concurrently, defaults to 1 (single request)
Returns:
List[OmniResponse]: List of response objects
"""
responses = []
stream = request_config.get("stream", False)
modalities = request_config.get("modalities", omit) # Most diffusion models don't require modalities param
extra_body = request_config.get("extra_body", None)
if stream:
raise NotImplementedError("Streaming is not currently implemented for diffusion model e2e test")
if request_num == 1:
# Send single request
chat_completion = self.client.chat.completions.create(
model=request_config.get("model"),
messages=request_config.get("messages"),
extra_body=extra_body,
modalities=modalities,
)
response = self._process_diffusion_response(chat_completion)
assert_diffusion_response(response, request_config, run_level=self.run_level)
responses.append(response)
else:
# Send concurrent requests
with concurrent.futures.ThreadPoolExecutor(max_workers=request_num) as executor:
futures = []
# Submit all request tasks
for _ in range(request_num):
future = executor.submit(
self.client.chat.completions.create,
model=request_config.get("model"),
messages=request_config.get("messages"),
modalities=modalities,
extra_body=extra_body,
)
futures.append(future)
# Process completed tasks
for future in concurrent.futures.as_completed(futures):
chat_completion = future.result()
response = self._process_diffusion_response(chat_completion)
assert_diffusion_response(response, request_config, run_level=self.run_level)
responses.append(response)
return responses
@pytest.fixture
def openai_client(omni_server, run_level):
"""Create OpenAIClientHandler fixture"""
def openai_client(omni_server: OmniServer, run_level: str):
"""Create OpenAIClientHandler fixture to facilitate communication with OmniServer
with encapsulated request sending, concurrent requests, response handling, and validation."""
return OpenAIClientHandler(host=omni_server.host, port=omni_server.port, api_key="EMPTY", run_level=run_level)
class OmniRunner:
"""
Test runner for Omni models.
Offline test runner for Omni models.
"""
def __init__(
@@ -1727,9 +1951,10 @@ class OmniRunner:
@pytest.fixture(scope="module")
def omni_runner(request):
def omni_runner(request, model_prefix):
with _omni_server_lock:
model, stage_config_path = request.param
model = model_prefix + model
with OmniRunner(model, seed=42, stage_configs_path=stage_config_path, stage_init_timeout=300) as runner:
print("OmniRunner started successfully")
yield runner
@@ -0,0 +1,244 @@
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm_omni.diffusion.models.flux2.flux2_transformer import (
Flux2PosEmbed,
Flux2Transformer2DModel,
)
# Initialize TP group before tests
@pytest.fixture(scope="function", autouse=True)
def setup_tp_group():
"""Set up TP group for each test function"""
with patch("vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size", return_value=2):
with patch("vllm.distributed.parallel_state.get_tp_group") as mock_get_tp_group:
mock_tp_group = MagicMock()
mock_tp_group.world_size = 2
mock_get_tp_group.return_value = mock_tp_group
yield
class TestFlux2TransformerWeightLoading:
"""Test Flux2Transformer weight loading functionality"""
def test_weight_loading_tp2(self, setup_tp_group):
"""Verify weights load correctly with TP=2"""
# Prepare test data
model = Flux2Transformer2DModel(
num_layers=2,
num_single_layers=1,
num_attention_heads=48,
attention_head_dim=128,
joint_attention_dim=15360,
)
# Mock TP=2 weight loading
mock_weights = []
# 1. Test regular weight loading
mock_weights.append(("x_embedder.weight", torch.randn(6144, 128)))
mock_weights.append(("context_embedder.weight", torch.randn(6144, 15360)))
mock_weights.append(("proj_out.weight", torch.randn(128, 6144)))
# 2. Test stacked_params_mapping weight loading
# Full weights - load_weights handles sharding internally
to_qkv_weight = torch.randn(18432, 6144) # Full weights
mock_weights.append(("transformer_blocks.0.attn.to_qkv.weight", to_qkv_weight))
# Add_kv_proj weights
add_kv_proj_weight = torch.randn(18432, 6144) # Full weights
mock_weights.append(("transformer_blocks.0.attn.add_kv_proj.weight", add_kv_proj_weight))
# 3. Test single block weight loading
single_block_qkv_mlp_weight = torch.randn(18432 + 18432 * 3, 6144) # Full weights
mock_weights.append(("single_transformer_blocks.0.attn.to_qkv_mlp_proj.weight", single_block_qkv_mlp_weight))
# Execute weight loading
loaded_params = model.load_weights(mock_weights)
# Verify
assert len(loaded_params) > 0, "Parameters should be loaded"
# Verify stacked_params_mapping is correctly set
assert model.stacked_params_mapping is not None
# Should have 6 mappings: 3 for to_qkv + 3 for add_kv_proj
assert len(model.stacked_params_mapping) == 6, "Should have 6 mappings"
# Verify weights are correctly loaded to corresponding layers
assert hasattr(model.transformer_blocks[0].attn.to_qkv, "weight")
# With TP=2, weight dimension on each GPU should be 18432/2 = 9216
assert model.transformer_blocks[0].attn.to_qkv.weight.shape == (9216, 6144), (
f"With TP=2, to_qkv weight dimension should be (9216, 6144), got {model.transformer_blocks[0].attn.to_qkv.weight.shape}"
)
class TestFlux2RopePositionEmbedding:
"""Test Flux2 RoPE position embedding functionality"""
def test_rope_position_embedding(self):
"""Verify RoPE produces correct embeddings for 4D coordinates"""
# Prepare test data - use model default configuration
# axes_dims_rope default is (32, 32, 32, 32)
# get_1d_rotary_pos_embed outputs half the dimension for real/imag parts
# So actual output dimension should be (16+16+16+16) = 64
axes_dims = (32, 32, 32, 32) # Use model default
rope_theta = 2000 # Model default is 2000
pos_embed = Flux2PosEmbed(theta=rope_theta, axes_dim=axes_dims)
# Create test IDs
seq_len = 10
ids = torch.randint(0, 100, (seq_len, 4)) # [S, 4]
# Forward pass
freqs_cos, freqs_sin = pos_embed(ids)
# Verify output shape - based on model config, expected dimension is 64
# Each axes_dim=32 outputs 16-dim real part, sum of 4 dimensions = 64
expected_dim = sum(axes_dims) // 2 # 128/2 = 64
assert freqs_cos.shape == (seq_len, expected_dim), (
f"Expected shape {(seq_len, expected_dim)}, got {freqs_cos.shape}"
)
assert freqs_sin.shape == (seq_len, expected_dim), (
f"Expected shape {(seq_len, expected_dim)}, got {freqs_sin.shape}"
)
# Verify output type - NPU may use float32, other devices use float64
assert freqs_cos.dtype in [torch.float32, torch.float64], "Should be float type"
assert freqs_sin.dtype in [torch.float32, torch.float64], "Should be float type"
# Verify value range
assert torch.all(freqs_cos >= -1) and torch.all(freqs_cos <= 1), "cos values should be in [-1, 1]"
assert torch.all(freqs_sin >= -1) and torch.all(freqs_sin <= 1), "sin values should be in [-1, 1]"
# Verify trigonometric relationship: cos^2 + sin^2 ≈ 1
cos_sq_sin_sq = freqs_cos**2 + freqs_sin**2
assert torch.allclose(cos_sq_sin_sq, torch.ones_like(cos_sq_sin_sq), atol=1e-6), "cos^2 + sin^2 should ≈ 1"
# Verify different positions produce different embeddings
ids_diff = torch.randint(0, 100, (seq_len, 4))
freqs_cos_diff, freqs_sin_diff = pos_embed(ids_diff)
assert not torch.allclose(freqs_cos, freqs_cos_diff), "Different positions should produce different embeddings"
# Verify same positions produce same embeddings
ids_same = ids.clone()
freqs_cos_same, freqs_sin_same = pos_embed(ids_same)
assert torch.allclose(freqs_cos, freqs_cos_same), "Same positions should produce same embeddings"
assert torch.allclose(freqs_sin, freqs_sin_same), "Same positions should produce same embeddings"
class TestFlux2PackedModuleMapping:
"""Test Flux2 packed module mapping functionality"""
def test_packed_module_mapping(self, setup_tp_group):
"""Verify to_qkv packing matches HF checkpoint"""
model = Flux2Transformer2DModel(
num_layers=1,
num_single_layers=0,
num_attention_heads=48,
attention_head_dim=128,
joint_attention_dim=15360,
)
# Verify stacked_params_mapping is correctly initialized
model.load_weights([]) # Trigger stacked_params_mapping initialization
assert model.stacked_params_mapping is not None
# Verify mapping configuration
expected_mappings = [
(".to_qkv.", ".to_q.", "q"),
(".to_qkv.", ".to_k.", "k"),
(".to_qkv.", ".to_v.", "v"),
(".add_kv_proj", ".add_q_proj", "q"),
(".add_kv_proj", ".add_k_proj", "k"),
(".add_kv_proj", ".add_v_proj", "v"),
]
assert model.stacked_params_mapping == expected_mappings
# Create mock HF checkpoint weights
hf_weights = []
# Mock HF separated Q/K/V weights
attn_block = model.transformer_blocks[0].attn
head_dim = 128
num_heads = 48
hidden_size = 6144
# Full weight dimension
full_shard_size = num_heads * head_dim # 6144
# Q projection weights (full weights)
q_weight = torch.randn(full_shard_size, hidden_size)
hf_weights.append(("transformer_blocks.0.attn.to_q.weight", q_weight))
# K projection weights (full weights)
k_weight = torch.randn(full_shard_size, hidden_size)
hf_weights.append(("transformer_blocks.0.attn.to_k.weight", k_weight))
# V projection weights (full weights)
v_weight = torch.randn(full_shard_size, hidden_size)
hf_weights.append(("transformer_blocks.0.attn.to_v.weight", v_weight))
# Mock HF separated add_q/k/v projection weights
add_q_weight = torch.randn(full_shard_size, hidden_size)
hf_weights.append(("transformer_blocks.0.attn.add_q_proj.weight", add_q_weight))
add_k_weight = torch.randn(full_shard_size, hidden_size)
hf_weights.append(("transformer_blocks.0.attn.add_k_proj.weight", add_k_weight))
add_v_weight = torch.randn(full_shard_size, hidden_size)
hf_weights.append(("transformer_blocks.0.attn.add_v_proj.weight", add_v_weight))
# Execute weight loading
loaded_params = model.load_weights(hf_weights)
# Verify weights are loaded
assert len(loaded_params) > 0
# Verify final QKV weights are correctly combined (considering TP sharding)
# With TP=2, dimension on each GPU should be half of full dimension
expected_qkv_shape = (full_shard_size * 3 // 2, hidden_size) # 18432/2 = 9216
assert attn_block.to_qkv.weight.shape == expected_qkv_shape, (
f"to_qkv weight dimension should be {expected_qkv_shape}, got {attn_block.to_qkv.weight.shape}"
)
expected_add_kv_shape = (full_shard_size * 3 // 2, hidden_size)
assert attn_block.add_kv_proj.weight.shape == expected_add_kv_shape, (
f"add_kv_proj weight dimension should be {expected_add_kv_shape}, got {attn_block.add_kv_proj.weight.shape}"
)
def test_packed_mapping_edge_cases(self, setup_tp_group):
"""Test edge cases for packed mapping"""
model = Flux2Transformer2DModel(
num_layers=1,
num_single_layers=1,
num_attention_heads=48,
attention_head_dim=128,
joint_attention_dim=15360,
)
model.load_weights([])
# Test invalid weight names
invalid_weights = [("invalid.weight", torch.randn(10, 10))]
loaded_params = model.load_weights(invalid_weights)
assert len(loaded_params) == 0, "Should not load invalid weights"
# Test to_out weight renaming
to_out_weight = torch.randn(6144, 6144)
weights = [("transformer_blocks.0.attn.to_out.0.weight", to_out_weight)]
loaded_params = model.load_weights(weights)
# Check if to_out related weights are loaded
to_out_loaded = any("to_out" in p for p in loaded_params)
assert to_out_loaded, "to_out weights should be correctly renamed and loaded"
# Test partial weight loading
partial_weights = [
("x_embedder.weight", torch.randn(6144, 128)),
("transformer_blocks.0.attn.to_q.weight", torch.randn(6144, 6144)), # Full weights
]
loaded_params = model.load_weights(partial_weights)
assert len(loaded_params) == 2, "Should load two weights"
assert "x_embedder.weight" in loaded_params
assert any("to_qkv" in p for p in loaded_params), "to_q should be mapped to to_qkv"
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for HunyuanFusedMoE (Support HunyuanImage3 Diffusion Model, 5a779b4)."""
import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestHunyuanFusedMoEPlatformDispatch:
"""Test platform dispatch via platform qualname hooks."""
def test_default_platform_uses_default_impl_qualname(self, mocker):
"""HunyuanFusedMoE should resolve the impl class from the platform hook."""
import vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe as hunyuan_moe
mock_platform = mocker.MagicMock()
mock_platform.get_diffusion_model_impl_qualname.return_value = (
"vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault"
)
mocker.patch.object(
hunyuan_moe,
"current_omni_platform",
mock_platform,
)
mock_resolve = mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname")
mock_impl = mocker.MagicMock()
mock_resolve.return_value = mock_impl
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe import (
HunyuanFusedMoE,
)
HunyuanFusedMoE(prefix="")
mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("hunyuan_fused_moe")
mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe")
mock_resolve.assert_called_once_with(
"vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault"
)
mock_impl.assert_called_once_with(prefix="")
class TestHunyuanFusedMoEFactory:
"""Test HunyuanFusedMoE factory __new__ and make_expert_params_mapping delegation."""
def test_new_delegates_to_impl_class(self, mocker):
"""HunyuanFusedMoE(prefix=..., **kwargs) should instantiate and return impl instance."""
import vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe as hunyuan_moe
class MockImpl:
def __init__(self, *, prefix: str = "", **kwargs):
self.prefix = prefix
self.kwargs = kwargs
mock_platform = mocker.MagicMock()
mock_platform.get_diffusion_model_impl_qualname.return_value = "mock.impl.Qualname"
mocker.patch.object(hunyuan_moe, "current_omni_platform", mock_platform)
mock_impl_class = mocker.MagicMock(return_value=MockImpl(prefix="test", a=1))
mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname", return_value=mock_impl_class)
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe import (
HunyuanFusedMoE,
)
result = HunyuanFusedMoE(prefix="test", a=1)
assert isinstance(result, MockImpl)
assert result.prefix == "test"
assert result.kwargs == {"a": 1}
mock_platform.prepare_diffusion_op_runtime.assert_called_once_with("hunyuan_fused_moe")
mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe")
mock_impl_class.assert_called_once_with(prefix="test", a=1)
def test_make_expert_params_mapping_delegates_to_impl(self, mocker):
"""make_expert_params_mapping should delegate to impl class method."""
import vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe as hunyuan_moe
expected_mapping = [("a", "b", 0, "c")]
mock_platform = mocker.MagicMock()
mock_platform.get_diffusion_model_impl_qualname.return_value = "mock.impl.Qualname"
mocker.patch.object(hunyuan_moe, "current_omni_platform", mock_platform)
mock_impl_class = mocker.MagicMock()
mock_impl_class.make_expert_params_mapping = mocker.MagicMock(return_value=expected_mapping)
mocker.patch.object(hunyuan_moe, "resolve_obj_by_qualname", return_value=mock_impl_class)
from vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe import (
HunyuanFusedMoE,
)
result = HunyuanFusedMoE.make_expert_params_mapping(
model=None,
ckpt_gate_proj_name="gate",
ckpt_down_proj_name="down",
ckpt_up_proj_name="up",
num_experts=4,
num_redundant_experts=0,
)
assert result == expected_mapping
mock_platform.get_diffusion_model_impl_qualname.assert_called_once_with("hunyuan_fused_moe")
mock_impl_class.make_expert_params_mapping.assert_called_once_with(
None,
ckpt_gate_proj_name="gate",
ckpt_down_proj_name="down",
ckpt_up_proj_name="up",
num_experts=4,
num_redundant_experts=0,
)
+78
View File
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for OmniDiffusionConfig.is_moe (fix is_moe type and threshold, 6663c0b)."""
import pytest
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestOmniDiffusionConfigIsMoE:
"""Tests for OmniDiffusionConfig.is_moe property.
Covers commit 6663c0b: fix is_moe type and threshold
- num_experts must be (list, tuple, int); otherwise return False.
- Threshold: is_moe is True when num_experts > 0 (not > 1).
"""
def test_is_moe_missing_num_experts_returns_false(self):
"""When num_experts is absent, is_moe should be False."""
tf_config = TransformerConfig.from_dict({})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is False
def test_is_moe_none_num_experts_returns_false(self):
"""When num_experts is explicitly None (e.g. in params), is_moe should be False."""
tf_config = TransformerConfig.from_dict({"num_experts": None})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is False
def test_is_moe_non_allowed_type_returns_false(self):
"""When num_experts is not int/list/tuple (e.g. str), is_moe should be False."""
tf_config = TransformerConfig.from_dict({"num_experts": "2"})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is False
def test_is_moe_int_zero_returns_false(self):
"""num_experts int 0 should yield is_moe False (threshold > 0)."""
tf_config = TransformerConfig.from_dict({"num_experts": 0})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is False
def test_is_moe_int_one_returns_true(self):
"""num_experts int 1 should yield is_moe True (threshold > 0, not > 1)."""
tf_config = TransformerConfig.from_dict({"num_experts": 1})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is True
def test_is_moe_int_gt_one_returns_true(self):
"""num_experts int > 1 should yield is_moe True."""
tf_config = TransformerConfig.from_dict({"num_experts": 2})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is True
def test_is_moe_list_all_zero_returns_false(self):
"""num_experts list with all <= 0 should yield is_moe False."""
tf_config = TransformerConfig.from_dict({"num_experts": [0]})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is False
def test_is_moe_list_has_positive_returns_true(self):
"""num_experts list with any int > 0 should yield is_moe True."""
tf_config = TransformerConfig.from_dict({"num_experts": [0, 1]})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is True
def test_is_moe_tuple_has_positive_returns_true(self):
"""num_experts tuple with any int > 0 should yield is_moe True."""
tf_config = TransformerConfig.from_dict({"num_experts": (0, 2)})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is True
def test_is_moe_list_non_int_ignored(self):
"""num_experts list with only non-int entries should yield is_moe False."""
tf_config = TransformerConfig.from_dict({"num_experts": ["a", 0.0]})
config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
assert config.is_moe is False
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from time import time
from vllm_omni.distributed.omni_coordinator import (
InstanceInfo,
RandomBalancer,
StageStatus,
)
def test_load_balancer_select_returns_valid_index():
"""Verify RandomBalancer.select() returns a valid index for instances."""
# Task structure mirrors async_omni; RandomBalancer ignores task contents.
task: dict = {
"request_id": "test",
"engine_inputs": None,
"sampling_params": None,
}
now = time()
instances = [
InstanceInfo(
input_addr="tcp://host:10001",
output_addr="tcp://host:10001-out",
stage_id=0,
status=StageStatus.UP,
queue_length=0,
last_heartbeat=now,
registered_at=now,
),
InstanceInfo(
input_addr="tcp://host:10002",
output_addr="tcp://host:10002-out",
stage_id=0,
status=StageStatus.UP,
queue_length=1,
last_heartbeat=now,
registered_at=now,
),
InstanceInfo(
input_addr="tcp://host:10003",
output_addr="tcp://host:10003-out",
stage_id=1,
status=StageStatus.UP,
queue_length=2,
last_heartbeat=now,
registered_at=now,
),
]
balancer = RandomBalancer()
index = balancer.select(task, instances)
assert isinstance(index, int)
assert 0 <= index < len(instances)
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import time
import pytest
import zmq
from vllm_omni.distributed.omni_coordinator import (
InstanceList,
OmniCoordClientForHub,
)
def _bind_pub() -> tuple[zmq.Context, zmq.Socket, str]:
ctx = zmq.Context.instance()
pub = ctx.socket(zmq.PUB)
pub.bind("tcp://127.0.0.1:*")
endpoint = pub.getsockopt(zmq.LAST_ENDPOINT).decode("ascii")
return ctx, pub, endpoint
def _wait_for_condition(cond, timeout: float = 2.0, interval: float = 0.01) -> bool:
start = time.time()
while time.time() - start < timeout:
if cond():
return True
time.sleep(interval)
return False
def test_hub_client_caches_instance_list_from_pub():
"""Verify OmniCoordClientForHub receives instance list updates from OmniCoordinator and caches for get_instance_list()."""
ctx, pub, endpoint = _bind_pub()
client = OmniCoordClientForHub(endpoint)
# ZMQ PUB/SUB slow-joiner: allow SUB to finish connecting before first send
time.sleep(0.2)
now = time.time()
instances_payload = [
{
"input_addr": "tcp://stage:10001",
"output_addr": "tcp://stage:10001-out",
"stage_id": 0,
"status": "up",
"queue_length": 0,
"last_heartbeat": now,
"registered_at": now,
},
{
"input_addr": "tcp://stage:10002",
"output_addr": "tcp://stage:10002-out",
"stage_id": 0,
"status": "up",
"queue_length": 1,
"last_heartbeat": now,
"registered_at": now,
},
{
"input_addr": "tcp://stage:10003",
"output_addr": "tcp://stage:10003-out",
"stage_id": 1,
"status": "error",
"queue_length": 5,
"last_heartbeat": now,
"registered_at": now,
},
]
payload = {"instances": instances_payload, "timestamp": now}
pub.send(json.dumps(payload).encode("utf-8"))
assert _wait_for_condition(lambda: len(client.get_instance_list().instances) == 3)
inst_list = client.get_instance_list()
assert isinstance(inst_list, InstanceList)
assert len(inst_list.instances) == 3
for src, inst in zip(instances_payload, inst_list.instances, strict=True):
assert inst.input_addr == src["input_addr"]
assert inst.output_addr == src["output_addr"]
assert inst.stage_id == src["stage_id"]
assert inst.status.value == src["status"]
stage0 = client.get_instances_for_stage(0)
stage1 = client.get_instances_for_stage(1)
assert all(inst.stage_id == 0 for inst in stage0.instances)
assert all(inst.stage_id == 1 for inst in stage1.instances)
# Send an updated list with fewer instances and verify cache refresh.
updated_payload = {
"instances": instances_payload[:2],
"timestamp": now + 1.0,
}
pub.send(json.dumps(updated_payload).encode("utf-8"))
assert _wait_for_condition(lambda: len(client.get_instance_list().instances) == 2)
updated_list = client.get_instance_list()
assert len(updated_list.instances) == 2
client.close()
pub.close(0)
ctx.term()
def test_hub_client_close_closes_sub_socket():
"""Verify OmniCoordClientForHub.close() marks client as closed; second close raises."""
ctx, pub, endpoint = _bind_pub()
client = OmniCoordClientForHub(endpoint)
client.close()
with pytest.raises(RuntimeError, match="already closed"):
client.close()
pub.close(0)
ctx.term()
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import zmq
from vllm_omni.distributed.omni_coordinator import (
OmniCoordClientForStage,
StageStatus,
)
def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]:
ctx = zmq.Context.instance()
router = ctx.socket(zmq.ROUTER)
router.bind("tcp://127.0.0.1:*")
endpoint = router.getsockopt(zmq.LAST_ENDPOINT).decode("ascii")
return ctx, router, endpoint
def _recv_event(router: zmq.Socket) -> dict:
frames = router.recv_multipart()
# ROUTER adds identity frame; the last frame is the payload.
payload = frames[-1]
return json.loads(payload.decode("utf-8"))
def test_stage_client_auto_register_on_init():
"""Verify OmniCoordClientForStage automatically sends initial registration/status-up event when created."""
ctx, router, endpoint = _bind_router()
input_addr = "tcp://stage:10001"
output_addr = "tcp://stage:10001-out"
stage_id = 0
client = OmniCoordClientForStage(endpoint, input_addr, output_addr, stage_id)
event = _recv_event(router)
assert event["event_type"] == "update"
assert event["status"] == StageStatus.UP.value
assert event["stage_id"] == stage_id
assert event["input_addr"] == input_addr
assert event["output_addr"] == output_addr
client.close()
router.close(0)
ctx.term()
def test_stage_client_update_info_sends_correct_event():
"""Verify OmniCoordClientForStage.update_info() sends status/load update events with expected fields."""
ctx, router, endpoint = _bind_router()
input_addr = "tcp://stage:10002"
output_addr = "tcp://stage:10002-out"
stage_id = 1
client = OmniCoordClientForStage(endpoint, input_addr, output_addr, stage_id)
# Discard initial registration event.
_recv_event(router)
client.update_info(status=StageStatus.ERROR)
client.update_info(queue_length=10)
first = _recv_event(router)
second = _recv_event(router)
assert first["status"] == StageStatus.ERROR.value
assert first["stage_id"] == stage_id
assert first["input_addr"] == input_addr
assert first["output_addr"] == output_addr
assert second["queue_length"] == 10
assert second["stage_id"] == stage_id
assert second["input_addr"] == input_addr
assert second["output_addr"] == output_addr
client.close()
router.close(0)
ctx.term()
def test_stage_client_close_sends_down_status():
"""Verify close() sends final status-down event before closing underlying socket."""
ctx, router, endpoint = _bind_router()
input_addr = "tcp://stage:10003"
output_addr = "tcp://stage:10003-out"
stage_id = 2
client = OmniCoordClientForStage(endpoint, input_addr, output_addr, stage_id)
# Discard initial registration event.
_recv_event(router)
client.close()
event = _recv_event(router)
assert event["status"] == StageStatus.DOWN.value
assert event["stage_id"] == stage_id
assert event["input_addr"] == input_addr
assert event["output_addr"] == output_addr
assert client._socket.closed # DEALER socket no longer usable after close
router.close(0)
ctx.term()
@@ -0,0 +1,209 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import time
import zmq
from vllm.v1.utils import get_engine_client_zmq_addr
from vllm_omni.distributed.omni_coordinator import (
OmniCoordClientForStage,
OmniCoordinator,
StageStatus,
)
def _recv_instance_list(sub: zmq.Socket, timeout_ms: int = 2000) -> dict | None:
"""Receive InstanceList JSON from SUB socket. Returns None on timeout."""
sub.setsockopt(zmq.RCVTIMEO, timeout_ms)
try:
data = sub.recv()
return json.loads(data.decode("utf-8"))
except zmq.Again:
return None
def _wait_for_instance_list(
sub: zmq.Socket,
expected_count: int,
timeout: float = 3.0,
) -> dict | None:
"""Wait until received InstanceList with expected_count active instances."""
start = time.time()
while time.time() - start < timeout:
msg = _recv_instance_list(sub, timeout_ms=500)
if msg is not None and len(msg.get("instances", [])) == expected_count:
return msg
return None
def test_omni_coordinator_registration_broadcast():
"""Verify that after multiple OmniCoordClientForStage instances register,
OmniCoordinator publishes an InstanceList containing all registered instances.
"""
router_addr = get_engine_client_zmq_addr(
local_only=False,
host="127.0.0.1",
port=0,
)
pub_addr = get_engine_client_zmq_addr(
local_only=False,
host="127.0.0.1",
port=0,
)
coordinator = OmniCoordinator(
router_zmq_addr=router_addr,
pub_zmq_addr=pub_addr,
heartbeat_timeout=1000.0,
)
sub_ctx = zmq.Context.instance()
sub = sub_ctx.socket(zmq.SUB)
sub.connect(pub_addr)
sub.setsockopt(zmq.SUBSCRIBE, b"")
# ZMQ PUB/SUB slow-joiner: allow SUB to connect before clients register.
time.sleep(0.3)
# Create 3 stage clients; each auto-registers on init.
clients = [
OmniCoordClientForStage(router_addr, "tcp://stage:10001", "tcp://stage:10001-out", 0),
OmniCoordClientForStage(router_addr, "tcp://stage:10002", "tcp://stage:10002-out", 0),
OmniCoordClientForStage(router_addr, "tcp://stage:10003", "tcp://stage:10003-out", 1),
]
msg = _wait_for_instance_list(sub, expected_count=3)
assert msg is not None, "Expected InstanceList with 3 instances"
assert len(msg["instances"]) == 3
assert isinstance(msg["timestamp"], (int, float))
input_addrs = {inst["input_addr"] for inst in msg["instances"]}
assert "tcp://stage:10001" in input_addrs
assert "tcp://stage:10002" in input_addrs
assert "tcp://stage:10003" in input_addrs
for c in clients:
c.close()
coordinator.close()
sub.close(0)
sub_ctx.term()
def test_omni_coordinator_heartbeat_timeout_handling():
"""Verify that when a stage instance stops sending heartbeats,
OmniCoordinator marks it as unhealthy and excludes it from the active list.
"""
router_addr = get_engine_client_zmq_addr(
local_only=False,
host="127.0.0.1",
port=0,
)
pub_addr = get_engine_client_zmq_addr(
local_only=False,
host="127.0.0.1",
port=0,
)
coordinator = OmniCoordinator(
router_zmq_addr=router_addr,
pub_zmq_addr=pub_addr,
heartbeat_timeout=5.0,
)
sub_ctx = zmq.Context.instance()
sub = sub_ctx.socket(zmq.SUB)
sub.connect(pub_addr)
sub.setsockopt(zmq.SUBSCRIBE, b"")
time.sleep(0.3)
# A and B: real clients that send heartbeats every 5s.
client_a = OmniCoordClientForStage(router_addr, "tcp://stage:a", "tcp://stage:a-out", 0)
client_b = OmniCoordClientForStage(router_addr, "tcp://stage:b", "tcp://stage:b-out", 0)
# C: raw DEALER that sends only registration, no heartbeat.
dealer_ctx = zmq.Context.instance()
dealer_c = dealer_ctx.socket(zmq.DEALER)
dealer_c.connect(router_addr)
reg_event = {
"input_addr": "tcp://stage:c",
"output_addr": "tcp://stage:c-out",
"stage_id": 0,
"event_type": "update",
"status": StageStatus.UP.value,
"queue_length": 0,
}
dealer_c.send(json.dumps(reg_event).encode("utf-8"))
msg = _wait_for_instance_list(sub, expected_count=3)
assert msg is not None, "Expected initial 3 instances"
assert len(msg["instances"]) == 3
# Wait for heartbeat timeout (timeout=5s, check interval ~2.5s).
time.sleep(8.0)
# Receive the update (C should be ERROR and excluded from active list).
msg_after_timeout = _wait_for_instance_list(sub, expected_count=2, timeout=5.0)
assert msg_after_timeout is not None, "Expected InstanceList with 2 instances after timeout"
instances = msg_after_timeout.get("instances", [])
input_addrs = {inst["input_addr"] for inst in instances}
assert "tcp://stage:a" in input_addrs
assert "tcp://stage:b" in input_addrs
assert "tcp://stage:c" not in input_addrs
client_a.close()
client_b.close()
dealer_c.close(0)
coordinator.close()
sub.close(0)
dealer_ctx.term()
sub_ctx.term()
def test_omni_coordinator_instance_shutdown_handling():
"""Verify that when a stage instance sends status='down',
OmniCoordinator removes it from the active list and broadcasts an updated list.
"""
router_addr = get_engine_client_zmq_addr(
local_only=False,
host="127.0.0.1",
port=0,
)
pub_addr = get_engine_client_zmq_addr(
local_only=False,
host="127.0.0.1",
port=0,
)
coordinator = OmniCoordinator(
router_zmq_addr=router_addr,
pub_zmq_addr=pub_addr,
heartbeat_timeout=1000.0,
)
sub_ctx = zmq.Context.instance()
sub = sub_ctx.socket(zmq.SUB)
sub.connect(pub_addr)
sub.setsockopt(zmq.SUBSCRIBE, b"")
time.sleep(0.3) # PUB/SUB slow-joiner
client = OmniCoordClientForStage(router_addr, "tcp://stage:shutdown", "tcp://stage:shutdown-out", 0)
msg = _wait_for_instance_list(sub, expected_count=1)
assert msg is not None
assert len(msg["instances"]) == 1
assert msg["instances"][0]["input_addr"] == "tcp://stage:shutdown"
# Send down status (simulating graceful shutdown).
client.update_info(status=StageStatus.DOWN)
# Receive updated list (should have 0 active instances).
msg = _wait_for_instance_list(sub, expected_count=0)
assert msg is not None
assert len(msg["instances"]) == 0
client.close()
coordinator.close()
sub.close(0)
sub_ctx.term()
@@ -6,7 +6,7 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.utils import GPUMemoryMonitor, hardware_test
from tests.utils import DeviceMemoryMonitor, hardware_test
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -23,9 +23,9 @@ models = ["riverclouds/qwen_image_random"]
def inference(model_name: str, offload: bool = True):
gc.collect()
current_omni_platform.empty_cache()
device_index = torch.cuda.current_device()
torch.cuda.reset_peak_memory_stats(device=device_index)
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
device_index = current_omni_platform.current_device()
current_omni_platform.reset_peak_memory_stats()
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
monitor.start()
m = Omni(
model=model_name,
@@ -33,7 +33,7 @@ def inference(model_name: str, offload: bool = True):
# cache_backend="cache_dit",
enable_cpu_offload=offload,
)
torch.cuda.reset_peak_memory_stats(device=device_index)
current_omni_platform.reset_peak_memory_stats()
height = 256
width = 256
@@ -44,7 +44,7 @@ def inference(model_name: str, offload: bool = True):
width=width,
num_inference_steps=9,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
),
)
peak = monitor.peak_used_mb
@@ -70,6 +70,11 @@ def test_cpu_offload_diffusion_model(model_name: str):
pytest.fail("Inference failed")
print(f"Offload peak memory: {offload_peak_memory} MB")
print(f"No offload peak memory: {no_offload_peak_memory} MB")
assert offload_peak_memory + 2500 < no_offload_peak_memory, (
f"Offload peak memory {offload_peak_memory} MB should be less than no offload peak memory {no_offload_peak_memory} MB"
# Set platform-specific VRAM saving thresholds to account
# for varying runtime memory overhead and fragmentation between CUDA and ROCm.
is_rocm = torch.version.hip is not None
threshold = 2500 if not is_rocm else 2100
assert offload_peak_memory + threshold < no_offload_peak_memory, (
f"Offload peak memory {offload_peak_memory} MB should be less than "
f"no offload peak memory {no_offload_peak_memory} MB by {threshold} MB"
)
@@ -5,7 +5,7 @@ import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.utils import GPUMemoryMonitor
from tests.utils import DeviceMemoryMonitor
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
@@ -28,11 +28,9 @@ def run_inference(
layerwise_offload: bool = False,
num_inference_steps: int = 3,
) -> float:
# For now, only support on GPU, so apply torch.cuda operations here
# NPU / ROCm platforms are expected to be detected and skipped this test function
current_omni_platform.empty_cache()
device_index = torch.cuda.current_device()
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
device_index = current_omni_platform.current_device()
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
monitor.start()
m = Omni(
@@ -44,7 +42,7 @@ def run_inference(
flow_shift=5.0,
)
torch.cuda.reset_peak_memory_stats(device=device_index)
current_omni_platform.reset_peak_memory_stats()
# Refer to tests/e2e/offline_inference/test_t2v_model.py
# Use minimal settings for testing
@@ -57,7 +55,7 @@ def run_inference(
OmniDiffusionSamplingParams(
height=height,
width=width,
generator=torch.Generator("cuda").manual_seed(42),
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
guidance_scale=1.0,
num_inference_steps=num_inference_steps,
num_frames=num_frames,
@@ -146,9 +146,6 @@ def _run_inference(
# - warmup: whether to run warmup for this SP config
# - is_perf_test: whether this is a performance test (show speedup metrics)
SP_CONFIGS_L2 = [
# Ulysses-2 - performance test
(2, 1, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True),
(1, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True), # Ring-2 - performance test
# Hybrid - correctness only
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
]
@@ -22,7 +22,7 @@ import torch
from PIL import Image
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.utils import GPUMemoryMonitor, hardware_test
from tests.utils import DeviceMemoryMonitor, hardware_test
from vllm_omni import Omni
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -54,12 +54,13 @@ def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor:
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
"""Return (mean_abs_diff, p99_abs_diff) over RGB pixels in [0, 1]."""
ta = _pil_to_float_rgb_tensor(a)
tb = _pil_to_float_rgb_tensor(b)
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
abs_diff = torch.abs(ta - tb)
return abs_diff.mean().item(), abs_diff.max().item()
p99_abs_diff = torch.quantile(abs_diff.flatten(), 0.99).item()
return abs_diff.mean().item(), p99_abs_diff
def _extract_single_image(outputs) -> Image.Image:
@@ -93,9 +94,9 @@ def _run_zimage_generate(
if num_requests < 2:
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
torch.cuda.empty_cache()
device_index = torch.cuda.current_device()
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
current_omni_platform.empty_cache()
device_index = current_omni_platform.current_device()
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
monitor.start()
m = Omni(
model=_get_zimage_model(),
@@ -161,8 +162,8 @@ def _run_zimage_generate(
def test_zimage_tensor_parallel_tp2(tmp_path: Path):
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
pytest.skip("Z-Image TP e2e test is only supported on CUDA for now.")
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
pytest.skip("Z-Image TP=2 requires >= 2 CUDA devices.")
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
pytest.skip("Z-Image TP=2 requires >= 2 devices.")
enforce_eager = False
@@ -196,18 +197,18 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
assert tp1_img.width == width and tp1_img.height == height
assert tp2_img.width == width and tp2_img.height == height
mean_abs_diff, max_abs_diff = _diff_metrics(tp1_img, tp2_img)
mean_abs_diff, p99_abs_diff = _diff_metrics(tp1_img, tp2_img)
mean_threshold = 3e-2
max_threshold = 5e-1
p99_threshold = 2.5e-1
print(
"Z-Image TP image diff stats (TP=1 vs TP=2): "
f"mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e}; "
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
f"tp1_img={tp1_path}, tp2_img={tp2_path}"
)
assert mean_abs_diff <= mean_threshold and max_abs_diff <= max_threshold, (
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e} "
f"(thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e})"
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
)
print(f"Z-Image TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}")
@@ -219,59 +220,59 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
)
@pytest.mark.integration
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA for now.")
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 CUDA devices.")
# @pytest.mark.integration
# def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
# if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
# pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA for now.")
# if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
# pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 devices.")
enforce_eager = False
# enforce_eager = False
# Use a larger image to ensure there are multiple VAE tiles.
height = 1152
width = 1152
num_inference_steps = 2
seed = 42
# # Use a larger image to ensure there are multiple VAE tiles.
# height = 1152
# width = 1152
# num_inference_steps = 2
# seed = 42
baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
tp_size=2,
height=height,
width=width,
num_inference_steps=num_inference_steps,
seed=seed,
enforce_eager=enforce_eager,
vae_use_tiling=True,
vae_patch_parallel_size=1,
num_requests=2,
)
pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
tp_size=2,
height=height,
width=width,
num_inference_steps=num_inference_steps,
seed=seed,
enforce_eager=enforce_eager,
vae_use_tiling=True,
vae_patch_parallel_size=2,
num_requests=2,
)
# baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
# tp_size=2,
# height=height,
# width=width,
# num_inference_steps=num_inference_steps,
# seed=seed,
# enforce_eager=enforce_eager,
# vae_use_tiling=True,
# vae_patch_parallel_size=1,
# num_requests=2,
# )
# pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
# tp_size=2,
# height=height,
# width=width,
# num_inference_steps=num_inference_steps,
# seed=seed,
# enforce_eager=enforce_eager,
# vae_use_tiling=True,
# vae_patch_parallel_size=2,
# num_requests=2,
# )
baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
baseline_img.save(baseline_path)
pp2_img.save(pp2_path)
# baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
# pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
# baseline_img.save(baseline_path)
# pp2_img.save(pp2_path)
mean_abs_diff, max_abs_diff = _diff_metrics(baseline_img, pp2_img)
mean_threshold = 5e-3
max_threshold = 1e-1
print(
"Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
f"mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e}; "
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
f"pp1_img={baseline_path}, pp2_img={pp2_path}"
)
assert mean_abs_diff <= mean_threshold and max_abs_diff <= max_threshold, (
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e} "
f"(thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e})"
)
# mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img)
# mean_threshold = 5e-3
# p99_threshold = 1e-1
# print(
# "Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
# f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
# f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
# f"pp1_img={baseline_path}, pp2_img={pp2_path}"
# )
# assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
# f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
# f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
# )
+6 -4
View File
@@ -8,6 +8,7 @@ from pathlib import Path
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
generate_synthetic_audio,
generate_synthetic_image,
@@ -51,7 +52,9 @@ else:
stage_configs = [get_chunk_config()]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
test_params = [
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
]
def get_system_prompt():
@@ -115,12 +118,11 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"stream": True,
"key_words": {
"audio": ["test"],
"image": ["square", "quadrate"],
},
}
# Test single completion
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -146,4 +148,4 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
"key_words": {"text": ["beijing"]},
}
openai_client.send_request(request_config, request_num=get_max_batch_size())
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@@ -12,6 +12,7 @@ from pathlib import Path
import pytest
from tests.conftest import (
OmniServerParams,
dummy_messages_from_mix_data,
generate_synthetic_audio,
generate_synthetic_image,
@@ -50,7 +51,9 @@ def get_chunk_config(default_path):
default_path = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
stage_configs = [default_path, get_chunk_config(default_path)]
# Create parameter combinations for model and stage config
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
test_params = [
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
]
def get_system_prompt():
@@ -106,7 +109,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
"key_words": {"text": ["beijing"]},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -128,7 +131,7 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None:
"key_words": {"text": ["beijing"]},
}
openai_client.send_request(request_config, request_num=get_max_batch_size())
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@pytest.mark.advanced_model
@@ -153,7 +156,7 @@ def test_image_to_text_001(omni_server, openai_client) -> None:
"key_words": {"image": IMAGE_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -177,7 +180,7 @@ def test_image_to_audio_001(omni_server, openai_client) -> None:
"key_words": {"image": IMAGE_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -201,7 +204,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
"key_words": {"image": IMAGE_KEY},
}
openai_client.send_request(request_config, request_num=get_max_batch_size())
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@pytest.mark.advanced_model
@@ -225,7 +228,7 @@ def test_video_to_text_001(omni_server, openai_client) -> None:
"key_words": {"video": VIDEO_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -249,7 +252,7 @@ def test_video_to_audio_001(omni_server, openai_client) -> None:
"key_words": {"video": VIDEO_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -273,7 +276,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
"key_words": {"video": VIDEO_KEY},
}
openai_client.send_request(request_config, request_num=get_max_batch_size())
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@pytest.mark.advanced_model
@@ -298,7 +301,7 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None:
"key_words": {"audio": AUDIO_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -324,7 +327,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
"key_words": {"image": IMAGE_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -351,7 +354,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
"key_words": {"video": VIDEO_KEY},
}
openai_client.send_request(request_config)
openai_client.send_omni_request(request_config)
@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
@@ -383,4 +386,4 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"stream": True,
"key_words": {"audio": AUDIO_KEY, "image": IMAGE_KEY, "video": VIDEO_KEY},
}
openai_client.send_request(request_config, request_num=get_max_batch_size())
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E online tests for Qwen3-TTS WebSocket streaming speech.
"""
import asyncio
import json
import os
from pathlib import Path
import pytest
import websockets
from tests.conftest import OmniServer
from tests.utils import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
def get_stage_config() -> str:
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
)
@pytest.fixture(scope="module")
def omni_server():
stage_config_path = get_stage_config()
with OmniServer(
MODEL,
[
"--stage-configs-path",
stage_config_path,
"--stage-init-timeout",
"120",
"--trust-remote-code",
"--enforce-eager",
"--disable-log-stats",
],
env_dict={"VLLM_DISABLE_COMPILE_CACHE": "1"},
) as server:
yield server
async def _run_ws_session(host: str, port: int) -> dict:
uri = f"ws://{host}:{port}/v1/audio/speech/stream"
starts: list[dict] = []
dones: list[dict] = []
chunk_lengths: dict[int, list[int]] = {}
session_done: dict | None = None
async with websockets.connect(uri, max_size=None) as ws:
await ws.send(
json.dumps(
{
"type": "session.config",
"model": MODEL,
"voice": "vivian",
"language": "English",
"response_format": "pcm",
"stream_audio": True,
}
)
)
await ws.send(
json.dumps(
{
"type": "input.text",
"text": (
"Hello, this is a websocket streaming test for Qwen three TTS, "
"and this sentence is intentionally long enough to produce audio chunks. "
"This is the second sentence."
),
}
)
)
await ws.send(json.dumps({"type": "input.done"}))
while True:
message = await asyncio.wait_for(ws.recv(), timeout=180)
if isinstance(message, bytes):
if not starts:
raise AssertionError("Received audio bytes before audio.start")
sentence_index = starts[-1]["sentence_index"]
chunk_lengths.setdefault(sentence_index, []).append(len(message))
continue
payload = json.loads(message)
msg_type = payload.get("type")
if msg_type == "audio.start":
starts.append(payload)
chunk_lengths.setdefault(payload["sentence_index"], [])
elif msg_type == "audio.done":
dones.append(payload)
elif msg_type == "session.done":
session_done = payload
break
elif msg_type == "error":
raise AssertionError(f"WebSocket error: {payload['message']}")
else:
raise AssertionError(f"Unexpected WebSocket message: {payload}")
return {
"starts": starts,
"dones": dones,
"chunk_lengths": chunk_lengths,
"session_done": session_done,
}
class TestQwen3TTSWebSocket:
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
def test_streaming_pcm_output(self, omni_server) -> None:
result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port))
starts = result["starts"]
dones = result["dones"]
chunk_lengths = result["chunk_lengths"]
session_done = result["session_done"]
assert session_done is not None
assert session_done["total_sentences"] == 2
assert len(starts) == 2
assert len(dones) == 2
for idx, start in enumerate(starts):
assert start["type"] == "audio.start"
assert start["sentence_index"] == idx
assert start["format"] == "pcm"
assert start["sample_rate"] == 24000
assert start["sentence_text"]
for done in dones:
sentence_index = done["sentence_index"]
total_bytes = done["total_bytes"]
assert done["error"] is False
assert total_bytes > 0
assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}"
assert sum(chunk_lengths[sentence_index]) == total_bytes
@@ -0,0 +1,164 @@
"""
Comprehensive tests of diffusion features that are available in online serving mode
and are supported by the following models:
- Qwen-Image-Edit: single image input
- Qwen-Image-Edit-2509: two image inputs
"""
import pytest
from tests.conftest import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
generate_synthetic_image,
)
from tests.utils import hardware_marks
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
"Transform the second image into a Vincent van Gogh style painting. "
"Then juxtapose the two transformed images into a single artwork for visual contrast."
)
NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
# This test file targets two models, so I write a helper function.
# If a similar test only involves one model, one can just define a global list variable.
def _get_diffusion_feature_cases(model: str):
return [
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--cache-backend",
"tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together.
"--enable-layerwise-offload",
],
),
id="single_card_001",
marks=SINGLE_CARD_FEATURE_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--cache-backend",
"cache_dit",
"--ulysses-degree",
"2",
],
),
id="parallel_001",
marks=PARALLEL_FEATURE_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--cache-backend",
"cache_dit",
"--ring",
"2",
],
),
id="parallel_002",
marks=PARALLEL_FEATURE_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--cache-backend",
"tea_cache",
"--cfg-parallel-size",
"2",
],
),
id="parallel_003",
marks=PARALLEL_FEATURE_MARKS,
),
pytest.param(
OmniServerParams(
model=model,
server_args=[
"--cache-backend",
"cache_dit",
"--tensor-parallel-size",
"2",
"--vae-patch-parallel-size",
"2",
"--vae-use-tiling",
],
),
id="parallel_004",
marks=PARALLEL_FEATURE_MARKS,
),
]
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
indirect=True,
)
def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios."""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
indirect=True,
)
def test_qwen_image_edit_2509(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test all diffusion features with Qwen-Image-Edit-2509 in regular end-user scenarios."""
image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(
image_data_url=[image_data_url_1, image_data_url_2], content_text=MULTI_EDIT_PROMPT
)
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@@ -136,7 +136,6 @@ def mock_async_diffusion(mocker: MockerFixture):
async def generate(**kwargs):
# Return n PIL images wrapped in result object
print("!!!!!!!!!!!!!!!!!!!!! kwargs", kwargs)
n = kwargs["sampling_params_list"][0].num_outputs_per_prompt
mock.captured_sampling_params_list = kwargs["sampling_params_list"]
mock.captured_prompt = kwargs["prompt"]
@@ -1,23 +1,29 @@
# tests/entrypoints/openai/test_serving_speech.py
import asyncio
import logging
import os
import struct
from inspect import Signature, signature
from pathlib import Path
from unittest.mock import MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, patch
import numpy as np
import pytest
import torch
from fastapi import FastAPI, HTTPException, UploadFile
from fastapi import FastAPI, HTTPException, Request, UploadFile
from fastapi.params import File, Form
from fastapi.responses import JSONResponse
from fastapi.testclient import TestClient
from pydantic import ValidationError
from pytest_mock import MockerFixture
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm_omni.entrypoints.openai import api_server as api_server_module
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio, OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.serving_speech import (
OmniOpenAIServingSpeech,
_create_wav_header,
)
from vllm_omni.outputs import OmniRequestOutput
@@ -916,17 +922,20 @@ class TestStreamingProtocolValidation:
"""Unit tests for the stream field validators in OpenAICreateSpeechRequest."""
def test_stream_validation_errors(self):
"""stream=True requires response_format='pcm' and speed=1.0."""
with pytest.raises(ValidationError, match="response_format='pcm'"):
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="wav")
"""stream=True requires response_format not in ('pcm', 'wav') and speed=1.0."""
with pytest.raises(ValidationError, match="requires response_format not in \\('pcm', 'wav'\\)"):
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="mp3")
with pytest.raises(ValidationError, match="Speed adjustment is not supported"):
OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm", speed=2.0)
def test_stream_valid(self):
"""stream=True + response_format='pcm' + speed=1.0 is accepted."""
"""stream=True + response_format in ('pcm', 'wav') + speed=1.0 is accepted."""
req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm")
assert req.stream is True
req = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="wav")
assert req.stream is True
def test_sse_stream_format_is_blocked(self):
"""stream_format='sse' is blocked."""
with pytest.raises(ValidationError, match="sse"):
@@ -1017,3 +1026,256 @@ class TestStreamingResponse:
response = client.post("/v1/audio/speech", json={"input": "Hello", "response_format": "wav"})
assert response.status_code == 200
assert "audio/wav" in response.headers["content-type"]
class TestAsyncOmniSupportedTasks:
"""Test that AsyncOmni reports correct supported tasks based on output modalities."""
@pytest.mark.asyncio
async def test_tts_only_no_generate_task(self):
"""TTS-only models (audio output, no text) should not include 'generate'."""
from unittest.mock import MagicMock
from vllm_omni.entrypoints.async_omni import AsyncOmni
omni = AsyncOmni.__new__(AsyncOmni)
omni.output_modalities = [None, "audio"]
stage = MagicMock()
stage.is_comprehension = False
omni.stage_list = [stage]
tasks = await omni.get_supported_tasks()
assert "generate" not in tasks
assert "speech" in tasks
@pytest.mark.asyncio
async def test_omni_model_includes_generate(self):
"""Models with text output (e.g. Qwen3-Omni) should include 'generate'."""
from unittest.mock import MagicMock
from vllm_omni.entrypoints.async_omni import AsyncOmni
omni = AsyncOmni.__new__(AsyncOmni)
omni.output_modalities = ["text", None, "audio"]
stage = MagicMock()
stage.is_comprehension = True
omni.stage_list = [stage]
tasks = await omni.get_supported_tasks()
assert "generate" in tasks
def test_api_server_create_speech_wraps_error_response_status():
handler = MagicMock()
handler.create_speech = AsyncMock(
return_value=ErrorResponse(
error=ErrorInfo(message="bad request", type="BadRequestError", param=None, code=400),
)
)
app = FastAPI()
app.state.openai_serving_speech = handler
scope = {
"type": "http",
"app": app,
"method": "POST",
"path": "/v1/audio/speech",
"headers": [],
"query_string": b"",
"client": ("127.0.0.1", 12345),
"server": ("testserver", 80),
"scheme": "http",
}
raw_request = Request(scope)
request = OpenAICreateSpeechRequest(input="Hello")
response = asyncio.run(api_server_module.create_speech(request, raw_request))
assert isinstance(response, JSONResponse)
assert response.status_code == 400
class TestWAVHeaderGeneration:
"""Unit tests for WAV header generation with placeholder values."""
def test_wav_header_basic_structure(self):
"""Test basic WAV header structure with default parameters."""
header = _create_wav_header(sample_rate=24000, num_channels=1, bits_per_sample=16)
# Verify header length (should be 44 bytes)
assert len(header) == 44, f"Expected 44 bytes, got {len(header)}"
# Parse and verify header structure
(
chunk_id,
chunk_size,
format_type,
subchunk1_id,
subchunk1_size,
audio_format,
num_channels,
sample_rate,
byte_rate,
block_align,
bits_per_sample,
subchunk2_id,
subchunk2_size,
) = struct.unpack("<4sI4s4sIHHIIHH4sI", header)
# Verify RIFF header
assert chunk_id == b"RIFF", f"Expected RIFF, got {chunk_id}"
assert chunk_size == 0xFFFFFFFF, f"Expected placeholder 0xFFFFFFFF, got {chunk_size:#x}"
assert format_type == b"WAVE", f"Expected WAVE, got {format_type}"
# Verify fmt chunk
assert subchunk1_id == b"fmt ", f"Expected 'fmt ', got {subchunk1_id}"
assert subchunk1_size == 16, f"Expected 16, got {subchunk1_size}"
assert audio_format == 1, f"Expected PCM (1), got {audio_format}"
assert num_channels == 1, f"Expected 1 channel, got {num_channels}"
assert sample_rate == 24000, f"Expected 24000 Hz, got {sample_rate}"
assert byte_rate == 48000, f"Expected 48000 byte/s, got {byte_rate}"
assert block_align == 2, f"Expected 2 bytes block align, got {block_align}"
assert bits_per_sample == 16, f"Expected 16 bits, got {bits_per_sample}"
# Verify data chunk
assert subchunk2_id == b"data", f"Expected 'data', got {subchunk2_id}"
assert subchunk2_size == 0xFFFFFFFF, f"Expected placeholder 0xFFFFFFFF, got {subchunk2_size:#x}"
def test_wav_header_different_sample_rates(self):
"""Test WAV header with different sample rates."""
test_cases = [
(16000, 1, 16),
(22050, 1, 16),
(24000, 1, 16),
(44100, 1, 16),
(48000, 1, 16),
]
for sample_rate, num_channels, bits_per_sample in test_cases:
header = _create_wav_header(sample_rate, num_channels, bits_per_sample)
assert len(header) == 44, f"Header length mismatch for {sample_rate} Hz"
# Parse sample rate from header
parsed_sample_rate = struct.unpack("<I", header[24:28])[0]
assert parsed_sample_rate == sample_rate, (
f"Sample rate mismatch: expected {sample_rate}, got {parsed_sample_rate}"
)
def test_wav_header_stereo(self):
"""Test WAV header with stereo audio."""
header = _create_wav_header(sample_rate=44100, num_channels=2, bits_per_sample=16)
# Parse header
parsed = struct.unpack("<4sI4s4sIHHIIHH4sI", header)
num_channels = parsed[6]
byte_rate = parsed[8]
block_align = parsed[9]
assert num_channels == 2, f"Expected 2 channels, got {num_channels}"
assert byte_rate == 44100 * 2 * 16 // 8, "Byte rate mismatch"
assert block_align == 2 * 16 // 8, "Block align mismatch"
def test_wav_header_placeholder_values(self):
"""Test that placeholder values are correctly set to 0xFFFFFFFF."""
header = _create_wav_header(sample_rate=24000)
# Extract size fields
chunk_size = struct.unpack("<I", header[4:8])[0]
subchunk2_size = struct.unpack("<I", header[40:44])[0]
assert chunk_size == 0xFFFFFFFF, "ChunkSize should be 0xFFFFFFFF for streaming"
assert subchunk2_size == 0xFFFFFFFF, "Subchunk2Size should be 0xFFFFFFFF for streaming"
class TestWAVStreaming:
"""Integration tests for WAV format streaming."""
@pytest.fixture
def wav_streaming_app(self, mocker: MockerFixture):
"""Test app configured for WAV streaming."""
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
def _make_output(finished: bool) -> OmniRequestOutput:
chunk = torch.sin(torch.linspace(0, 440 * 2 * torch.pi, 24000))
class MockCompletionOutput:
def __init__(self, index: int = 0):
self.index = index
self.text = ""
self.token_ids = []
self.finish_reason = "stop"
self.stop_reason = None
self.logprobs = None
class MockRequestOutput:
def __init__(self, audio_tensor: torch.Tensor):
self.request_id = "speech-wav-stream-test"
self.outputs = [MockCompletionOutput(index=0)]
self.multimodal_output = {"audio": audio_tensor, "sr": 24000}
self.finished = finished
self.prompt_token_ids = None
self.encoder_prompt_token_ids = None
self.num_cached_tokens = None
self.prompt_logprobs = None
self.kv_transfer_params = None
return OmniRequestOutput(
stage_id=0,
final_output_type="audio",
request_output=MockRequestOutput(audio_tensor=chunk),
finished=finished,
)
async def mock_generate_streaming(*args, **kwargs):
yield _make_output(finished=False)
yield _make_output(finished=True)
mock_engine_client.generate = mocker.MagicMock(side_effect=mock_generate_streaming)
mock_engine_client.default_sampling_params_list = [{}]
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
speech_server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=mocker.MagicMock(),
)
original_create_speech = speech_server.create_speech
sig = signature(original_create_speech)
new_parameters = [p for name, p in sig.parameters.items() if name != "raw_request"]
new_sig = Signature(parameters=new_parameters, return_annotation=sig.return_annotation)
async def awaitable_create_speech(*args, **kwargs):
return await original_create_speech(*args, **kwargs)
awaitable_create_speech.__signature__ = new_sig
speech_server.create_speech = awaitable_create_speech
app = FastAPI()
app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None)
return app
def test_wav_streaming_success(self, wav_streaming_app):
"""Test WAV format streaming returns correct content type and includes WAV header."""
client = TestClient(wav_streaming_app)
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": "wav"})
assert response.status_code == 200
assert "audio/wav" in response.headers["content-type"]
assert len(response.content) > 44 # Should have WAV header + audio data
# Verify WAV header is present
header = response.content[:44]
chunk_id = header[0:4]
format_type = header[8:12]
assert chunk_id == b"RIFF", "Should start with RIFF"
assert format_type == b"WAVE", "Should contain WAVE format"
def test_streaming_unsupported_format_rejected(self, wav_streaming_app):
"""Test that unsupported formats are rejected for streaming."""
client = TestClient(wav_streaming_app)
unsupported_formats = ["mp3"]
for fmt in unsupported_formats:
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": fmt})
assert response.status_code == 422
@@ -0,0 +1,387 @@
import asyncio
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import FastAPI, WebSocket
from starlette.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
from vllm_omni.entrypoints.openai import serving_speech_stream as streaming_speech_module
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _build_test_app(speech_service=None, *, idle_timeout=30.0, config_timeout=10.0):
if speech_service is None:
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"RIFF" + b"\x00" * 32, "audio/wav"))
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-1", object(), {}))
async def mock_generate_pcm_chunks(_generator, _request_id):
for chunk in (b"\x01\x02", b"\x03\x04\x05"):
yield chunk
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
speech_service.engine_client = MagicMock()
speech_service.engine_client.abort = AsyncMock()
handler = OmniStreamingSpeechHandler(
speech_service=speech_service,
idle_timeout=idle_timeout,
config_timeout=config_timeout,
)
app = FastAPI()
@app.websocket("/v1/audio/speech/stream")
async def ws_endpoint(websocket: WebSocket):
await handler.handle_session(websocket)
return app, speech_service
class TestStreamingSpeechWebSocket:
def test_non_streaming_single_frame(self):
app, speech_service = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": "Hello world. "})
start = ws.receive_json()
assert start["type"] == "audio.start"
assert start["sentence_index"] == 0
assert start["sentence_text"] == "Hello world."
assert start["format"] == "wav"
audio = ws.receive_bytes()
assert audio.startswith(b"RIFF")
done = ws.receive_json()
assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": len(audio), "error": False}
ws.send_json({"type": "input.done"})
session_done = ws.receive_json()
assert session_done == {"type": "session.done", "total_sentences": 1}
assert speech_service._generate_audio_bytes.await_count == 1
def test_streaming_multiple_binary_frames(self):
captured_requests = []
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"", "audio/wav"))
speech_service.engine_client = MagicMock()
speech_service.engine_client.abort = AsyncMock()
async def mock_prepare_speech_generation(request):
captured_requests.append(request)
return "req-stream", object(), {}
speech_service._prepare_speech_generation = mock_prepare_speech_generation
async def mock_generate_pcm_chunks(_generator, _request_id):
for chunk in (b"\x01\x02", b"\x03\x04\x05", b"\x06"):
yield chunk
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
app, _ = _build_test_app(speech_service)
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json(
{
"type": "session.config",
"voice": "Vivian",
"stream_audio": True,
"response_format": "pcm",
"initial_codec_chunk_frames": 12,
}
)
ws.send_json({"type": "input.text", "text": "Hello world. "})
start = ws.receive_json()
assert start["type"] == "audio.start"
assert start["format"] == "pcm"
assert start["sample_rate"] == 24000
assert ws.receive_bytes() == b"\x01\x02"
assert ws.receive_bytes() == b"\x03\x04\x05"
assert ws.receive_bytes() == b"\x06"
done = ws.receive_json()
assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": 6, "error": False}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
assert len(captured_requests) == 1
assert captured_requests[0].stream is True
assert captured_requests[0].response_format == "pcm"
assert captured_requests[0].initial_codec_chunk_frames == 12
assert speech_service._generate_audio_bytes.await_count == 0
def test_flush_on_input_done(self):
app, _ = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": "Hello world without punctuation"})
ws.send_json({"type": "input.done"})
assert ws.receive_json()["type"] == "audio.start"
assert ws.receive_bytes()
assert ws.receive_json() == {
"type": "audio.done",
"sentence_index": 0,
"total_bytes": 36,
"error": False,
}
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
def test_invalid_streaming_config(self):
app, _ = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json(
{
"type": "session.config",
"voice": "Vivian",
"stream_audio": True,
"response_format": "wav",
}
)
error = ws.receive_json()
assert error["type"] == "error"
assert "response_format='pcm'" in error["message"]
def test_empty_input_text_emits_no_audio(self):
app, speech_service = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": ""})
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 0}
assert speech_service._generate_audio_bytes.await_count == 0
def test_multiple_sentences_increment_indices(self):
app, _ = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": "First sentence. Second sentence. "})
first_start = ws.receive_json()
assert first_start["sentence_index"] == 0
ws.receive_bytes()
assert ws.receive_json() == {
"type": "audio.done",
"sentence_index": 0,
"total_bytes": 36,
"error": False,
}
second_start = ws.receive_json()
assert second_start["sentence_index"] == 1
ws.receive_bytes()
assert ws.receive_json() == {
"type": "audio.done",
"sentence_index": 1,
"total_bytes": 36,
"error": False,
}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 2}
def test_unknown_message_type_keeps_session_open(self):
app, _ = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "unknown"})
error = ws.receive_json()
assert error == {"type": "error", "message": "Unknown message type: unknown"}
ws.send_json({"type": "input.text", "text": "Hello world. "})
assert ws.receive_json()["type"] == "audio.start"
ws.receive_bytes()
assert ws.receive_json() == {
"type": "audio.done",
"sentence_index": 0,
"total_bytes": 36,
"error": False,
}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
def test_config_timeout_closes_session(self):
app, _ = _build_test_app(config_timeout=0.01)
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
error = ws.receive_json()
assert error == {"type": "error", "message": "Timeout waiting for session.config"}
def test_generation_error_marks_audio_done(self):
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
speech_service._generate_audio_bytes = AsyncMock(side_effect=RuntimeError("boom"))
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-err", object(), {}))
speech_service._generate_pcm_chunks = AsyncMock()
speech_service.engine_client = MagicMock()
speech_service.engine_client.abort = AsyncMock()
app, _ = _build_test_app(speech_service)
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": "Hello world. "})
assert ws.receive_json()["type"] == "audio.start"
assert ws.receive_json() == {"type": "error", "message": "Generation failed for sentence 0: boom"}
assert ws.receive_json() == {"type": "audio.done", "sentence_index": 0, "total_bytes": 0, "error": True}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
def test_streaming_generation_error_marks_audio_done(self):
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"", "audio/wav"))
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-stream-err", object(), {}))
speech_service.engine_client = MagicMock()
speech_service.engine_client.abort = AsyncMock()
async def mock_generate_pcm_chunks(_generator, _request_id):
yield b"\x01\x02"
raise RuntimeError("stream boom")
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
app, _ = _build_test_app(speech_service)
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json(
{
"type": "session.config",
"voice": "Vivian",
"stream_audio": True,
"response_format": "pcm",
}
)
ws.send_json({"type": "input.text", "text": "Hello world. "})
assert ws.receive_json()["type"] == "audio.start"
assert ws.receive_bytes() == b"\x01\x02"
assert ws.receive_json() == {
"type": "error",
"message": "Generation failed for sentence 0: stream boom",
}
assert ws.receive_json() == {
"type": "audio.done",
"sentence_index": 0,
"total_bytes": 2,
"error": True,
}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 1}
def test_invalid_input_text_type_returns_validation_error(self):
app, speech_service = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": 123})
assert ws.receive_json() == {
"type": "error",
"message": "input.text requires a string value",
}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 0}
assert speech_service._generate_audio_bytes.await_count == 0
def test_input_text_message_too_large(self, monkeypatch):
monkeypatch.setattr(streaming_speech_module, "_MAX_INPUT_TEXT_MESSAGE_SIZE", 32)
app, speech_service = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian"})
ws.send_json({"type": "input.text", "text": "x" * 128})
assert ws.receive_json() == {
"type": "error",
"message": "input.text message too large",
}
ws.send_json({"type": "input.done"})
assert ws.receive_json() == {"type": "session.done", "total_sentences": 0}
assert speech_service._generate_audio_bytes.await_count == 0
def test_session_config_message_too_large(self, monkeypatch):
monkeypatch.setattr(streaming_speech_module, "_MAX_CONFIG_MESSAGE_SIZE", 64)
app, _ = _build_test_app()
with TestClient(app) as client:
with client.websocket_connect("/v1/audio/speech/stream") as ws:
ws.send_json({"type": "session.config", "voice": "Vivian", "ref_audio": "x" * 512})
assert ws.receive_json() == {
"type": "error",
"message": "session.config message too large",
}
def test_disconnect_aborts_streaming_request(self):
speech_service = MagicMock(spec=OmniOpenAIServingSpeech)
speech_service._generate_audio_bytes = AsyncMock(return_value=(b"", "audio/wav"))
speech_service._prepare_speech_generation = AsyncMock(return_value=("req-abort", object(), {}))
speech_service.engine_client = MagicMock()
speech_service.engine_client.abort = AsyncMock()
async def mock_generate_pcm_chunks(_generator, _request_id):
yield b"\x01\x02"
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
handler = OmniStreamingSpeechHandler(speech_service=speech_service)
websocket = MagicMock()
websocket.send_json = AsyncMock(side_effect=[None, WebSocketDisconnect()])
websocket.send_bytes = AsyncMock(side_effect=WebSocketDisconnect())
config = MagicMock()
config.model = None
config.voice = "Vivian"
config.task_type = None
config.language = None
config.instructions = None
config.response_format = "pcm"
config.speed = 1.0
config.max_new_tokens = None
config.initial_codec_chunk_frames = None
config.ref_audio = None
config.ref_text = None
config.x_vector_only_mode = None
config.stream_audio = True
with pytest.raises(WebSocketDisconnect):
asyncio.run(handler._generate_and_send(websocket, config, "Hello world.", 0))
speech_service.engine_client.abort.assert_awaited_once_with("req-abort")
assert websocket.send_json.await_count == 2
@@ -0,0 +1,253 @@
"""Tests for SentenceSplitter used in streaming TTS input."""
import pytest
from vllm_omni.entrypoints.openai.text_splitter import SentenceSplitter
pytestmark = [pytest.mark.openai, pytest.mark.speech]
class TestSentenceSplitterEnglish:
"""Tests for English sentence splitting."""
def test_single_sentence_no_boundary(self):
splitter = SentenceSplitter()
result = splitter.add_text("Hello world")
assert result == []
assert splitter.buffer == "Hello world"
def test_single_sentence_with_boundary(self):
splitter = SentenceSplitter()
result = splitter.add_text("Hello world. How are you?")
assert len(result) == 1
assert result[0] == "Hello world."
def test_multiple_sentences(self):
splitter = SentenceSplitter()
result = splitter.add_text("Hello. How are you? I am fine! ")
assert len(result) == 3
assert result[0] == "Hello."
assert result[1] == "How are you?"
assert result[2] == "I am fine!"
def test_exclamation_mark(self):
splitter = SentenceSplitter()
result = splitter.add_text("Wow, that is great! Tell me more.")
assert len(result) == 1
assert result[0] == "Wow, that is great!"
def test_question_mark(self):
splitter = SentenceSplitter()
result = splitter.add_text("Can you hear me? I hope so.")
assert len(result) == 1
assert result[0] == "Can you hear me?"
class TestSentenceSplitterChinese:
"""Tests for CJK sentence splitting."""
def test_chinese_period(self):
splitter = SentenceSplitter()
result = splitter.add_text("你好世界。你好吗")
assert len(result) == 1
assert result[0] == "你好世界。"
def test_chinese_exclamation(self):
splitter = SentenceSplitter()
result = splitter.add_text("太好了!谢谢你")
assert len(result) == 1
assert result[0] == "太好了!"
def test_chinese_question(self):
splitter = SentenceSplitter()
result = splitter.add_text("你是谁?我是小明")
assert len(result) == 1
assert result[0] == "你是谁?"
def test_chinese_comma_no_split(self):
"""Chinese commas are clause-level and should not trigger a split."""
splitter = SentenceSplitter()
result = splitter.add_text("你好,世界")
assert result == []
assert splitter.buffer == "你好,世界"
def test_chinese_semicolon_no_split(self):
"""Chinese semicolons are clause-level and should not trigger a split."""
splitter = SentenceSplitter()
result = splitter.add_text("第一点;第二点")
assert result == []
assert splitter.buffer == "第一点;第二点"
def test_chinese_multiple(self):
splitter = SentenceSplitter()
result = splitter.add_text("你好!你好吗?我很好。")
assert len(result) == 3
assert result[0] == "你好!"
assert result[1] == "你好吗?"
assert result[2] == "我很好。"
class TestSentenceSplitterMixed:
"""Tests for mixed-language sentence splitting."""
def test_mixed_english_chinese(self):
splitter = SentenceSplitter()
result = splitter.add_text("Hello世界。How are you? ")
assert len(result) == 2
assert result[0] == "Hello世界。"
assert result[1] == "How are you?"
class TestSentenceSplitterIncremental:
"""Tests for incremental (multi-chunk) text input."""
def test_accumulation_across_chunks(self):
splitter = SentenceSplitter()
# First chunk: no boundary
result1 = splitter.add_text("Hello ")
assert result1 == []
# Second chunk: completes a sentence
result2 = splitter.add_text("world. How")
assert len(result2) == 1
assert result2[0] == "Hello world."
assert splitter.buffer == "How"
def test_word_by_word(self):
splitter = SentenceSplitter()
words = ["Hello, ", "how ", "are ", "you? ", "I ", "am ", "fine."]
all_sentences = []
for word in words:
all_sentences.extend(splitter.add_text(word))
assert len(all_sentences) == 1
assert all_sentences[0] == "Hello, how are you?"
# "I am fine." stays in buffer (no trailing whitespace after period)
def test_three_chunks(self):
splitter = SentenceSplitter()
splitter.add_text("The quick brown ")
splitter.add_text("fox jumps. ")
result = splitter.add_text("Over the lazy dog. ")
# "The quick brown fox jumps." should have been returned on second chunk
# "Over the lazy dog." on third chunk
assert len(result) == 1
assert result[0] == "Over the lazy dog."
class TestSentenceSplitterFlush:
"""Tests for flush behavior."""
def test_flush_returns_remaining(self):
splitter = SentenceSplitter()
splitter.add_text("Hello world")
result = splitter.flush()
assert result == "Hello world"
assert splitter.buffer == ""
def test_flush_empty_buffer(self):
splitter = SentenceSplitter()
result = splitter.flush()
assert result is None
def test_flush_after_sentence(self):
splitter = SentenceSplitter()
splitter.add_text("Hello world. Remaining text")
result = splitter.flush()
assert result == "Remaining text"
def test_flush_whitespace_only(self):
splitter = SentenceSplitter()
splitter.add_text("Hello. ")
# "Hello." extracted, buffer is " "
result = splitter.flush()
# Whitespace-only should return None
assert result is None
def test_flush_clears_buffer(self):
splitter = SentenceSplitter()
splitter.add_text("some text")
splitter.flush()
assert splitter.buffer == ""
# Second flush should return None
assert splitter.flush() is None
class TestSentenceSplitterEdgeCases:
"""Edge case tests."""
def test_empty_input(self):
splitter = SentenceSplitter()
result = splitter.add_text("")
assert result == []
assert splitter.buffer == ""
def test_none_like_empty(self):
"""Empty string should not affect buffer."""
splitter = SentenceSplitter()
splitter.add_text("Hello")
splitter.add_text("")
assert splitter.buffer == "Hello"
def test_only_punctuation(self):
splitter = SentenceSplitter()
result = splitter.add_text(". ")
# "." is 1 char, below default min_sentence_length of 2
# It will be carried forward
assert result == []
def test_min_sentence_length(self):
splitter = SentenceSplitter(min_sentence_length=10)
result = splitter.add_text("Hi. Hello world. ")
# "Hi." is 3 chars (< 10), so it gets carried to "Hello world."
assert len(result) == 1
assert "Hi." in result[0]
assert "Hello world." in result[0]
def test_short_segments_are_carried_until_long_enough(self):
splitter = SentenceSplitter(min_sentence_length=10)
result = splitter.add_text("Hi. Ok. Hello there. ")
assert result == ["Hi.Ok.Hello there."]
assert splitter.buffer == ""
def test_min_sentence_length_zero(self):
splitter = SentenceSplitter(min_sentence_length=0)
result = splitter.add_text("A. B. ")
assert len(result) == 2
def test_no_boundary_then_flush(self):
splitter = SentenceSplitter()
result = splitter.add_text("Hello world how are you")
assert result == []
flushed = splitter.flush()
assert flushed == "Hello world how are you"
def test_consecutive_punctuation(self):
splitter = SentenceSplitter()
result = splitter.add_text("Really?! Yes, really. ")
assert len(result) >= 1
def test_reuse_after_flush(self):
"""Splitter can be reused after flush."""
splitter = SentenceSplitter()
splitter.add_text("First session.")
splitter.flush()
result = splitter.add_text("Second session. More text")
assert len(result) == 1
assert result[0] == "Second session."
assert splitter.buffer == "More text"
class TestSentenceSplitterBufferLimit:
"""Tests for buffer overflow protection."""
def test_buffer_overflow_raises(self):
from vllm_omni.entrypoints.openai.text_splitter import _MAX_BUFFER_SIZE
splitter = SentenceSplitter()
# Fill buffer just under the limit
splitter.add_text("x" * (_MAX_BUFFER_SIZE - 1))
# One more char should exceed the limit
with pytest.raises(ValueError, match="exceeded maximum size"):
splitter.add_text("xx")
+1 -1
View File
@@ -345,7 +345,7 @@ def _setup_multiprocessing_mocks(monkeypatch: pytest.MonkeyPatch, mocker: Mocker
fake_process_instance = mocker.MagicMock()
fake_process_instance.start = mocker.MagicMock()
fake_process_instance.join = mocker.MagicMock()
fake_process_instance.is_alive = mocker.MagicMock(return_value=False)
fake_process_instance.is_alive = mocker.MagicMock(return_value=True)
fake_process_instance.terminate = mocker.MagicMock()
fake_process_class.return_value = fake_process_instance
@@ -12,7 +12,7 @@ from pathlib import Path
import pytest
from tests.conftest import convert_audio_file_to_text, cosine_similarity_text
from tests.conftest import OmniServerParams, convert_audio_file_to_text, cosine_similarity_text
from tests.utils import hardware_test
models = ["Qwen/Qwen2.5-Omni-7B"]
@@ -22,7 +22,11 @@ stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_config
example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "qwen2_5_omni")
# Create parameter combinations for model and stage config
test_params = [(8091, model, stage_config) for model in models for stage_config in stage_configs]
test_params = [
OmniServerParams(model=model, port=8091, stage_config_path=stage_config)
for model in models
for stage_config in stage_configs
]
def run_cmd(command):
@@ -12,7 +12,7 @@ from pathlib import Path
import pytest
from tests.conftest import convert_audio_file_to_text, cosine_similarity_text
from tests.conftest import OmniServerParams, convert_audio_file_to_text, cosine_similarity_text
from tests.utils import hardware_test
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
@@ -22,7 +22,11 @@ stage_configs = [str(Path(__file__).parent.parent.parent / "e2e" / "stage_config
example_dir = str(Path(__file__).parent.parent.parent.parent / "examples" / "online_serving" / "qwen3_omni")
# Create parameter combinations for model and stage config
test_params = [(8091, model, stage_config) for model in models for stage_config in stage_configs]
test_params = [
OmniServerParams(model=model, port=8091, stage_config_path=stage_config)
for model in models
for stage_config in stage_configs
]
def run_cmd(command):
+22 -9
View File
@@ -107,8 +107,14 @@ def omni_server(request):
print("OmniServer stopped")
def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_prompt) -> Any:
"""Generate synthetic image with random values."""
def run_benchmark(
args: list,
test_name: str,
flow,
dataset_name: str,
num_prompt,
) -> Any:
"""Run a single benchmark iteration and return the parsed result JSON."""
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_{current_dt}.json"
if "--result-filename" in args:
@@ -117,10 +123,6 @@ def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_promp
["vllm", "bench", "serve", "--omni"]
+ args
+ [
"--backend",
"openai-chat-omni",
"--endpoint",
"/v1/chat/completions",
"--save-result",
"--result-dir",
os.environ.get("BENCHMARK_DIR", "tests"),
@@ -196,7 +198,10 @@ def benchmark_params(request, omni_server):
total = len(all_params)
print(f"\n Running benchmark {current}/{total} for {test_name}")
return {"test_name": test_name, "params": all_params[param_index]}
return {
"test_name": test_name,
"params": all_params[param_index],
}
def assert_result(result, params, num_prompt):
@@ -266,7 +271,11 @@ def test_performance_benchmark(omni_server, benchmark_params):
for qps, num_prompt in zip(qps_list, num_prompt_list):
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
result = run_benchmark(
args=args, test_name=test_name, flow=qps, dataset_name=dataset_name, num_prompt=num_prompt
args=args,
test_name=test_name,
flow=qps,
dataset_name=dataset_name,
num_prompt=num_prompt,
)
assert_result(result, params, num_prompt=num_prompt)
@@ -274,6 +283,10 @@ def test_performance_benchmark(omni_server, benchmark_params):
for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
result = run_benchmark(
args=args, test_name=test_name, flow=concurrency, dataset_name=dataset_name, num_prompt=num_prompt
args=args,
test_name=test_name,
flow=concurrency,
dataset_name=dataset_name,
num_prompt=num_prompt,
)
assert_result(result, params, num_prompt=num_prompt)
+99
View File
@@ -0,0 +1,99 @@
# Stage config for running Qwen3-TTS with 2-stage architecture
# Stage 0: Talker (text -> 8-layer RVQ codec codes)
# Stage 1: Code2Wav (codec codes -> audio waveform)
#
# The following config has been verified on 1x H100-80G GPU.
async_chunk: true
stage_args:
- stage_id: 0
stage_type: llm
runtime:
devices: "0"
max_batch_size: 4
engine_args:
model_stage: qwen3_tts
model_arch: Qwen3TTSTalkerForConditionalGeneration
hf_overrides:
architectures: [Qwen3TTSTalkerForConditionalGeneration]
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
enforce_eager: false
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: latent
gpu_memory_utilization: 0.3
distributed_executor_backend: "mp"
max_num_batched_tokens: 512
max_model_len: 4096
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
output_connectors:
to_stage_1: connector_of_shared_memory
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 4096
seed: 42
detokenize: false
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 1
stage_type: llm
runtime:
devices: "0"
max_batch_size: 4
engine_args:
model_stage: code2wav
model_arch: Qwen3TTSCode2Wav
hf_overrides:
architectures: [Qwen3TTSCode2Wav]
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
enforce_eager: true
trust_remote_code: true
async_scheduling: false
enable_prefix_caching: false
engine_output_type: audio
gpu_memory_utilization: 0.2
distributed_executor_backend: "mp"
max_num_batched_tokens: 8192
max_model_len: 32768
engine_input_source: [0]
final_output: true
final_output_type: audio
input_connectors:
from_stage_0: connector_of_shared_memory
tts_args:
max_instructions_length: 500
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
detokenize: true
repetition_penalty: 1.0
runtime:
enabled: true
defaults:
window_size: -1
max_inflight: 4
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
shm_threshold_bytes: 65536
codec_streaming: true
connector_get_sleep_s: 0.01
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
codec_chunk_frames: 25
codec_left_context_frames: 25
edges:
- from: 0
to: 1
window_size: -1
+41
View File
@@ -8,6 +8,8 @@
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
10,
40
@@ -28,6 +30,8 @@
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
10,
40
@@ -88,6 +92,8 @@
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
10,
40
@@ -108,6 +114,8 @@
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [
10,
40
@@ -140,5 +148,38 @@
}
}
]
},
{
"test_name": "test_qwen3_tts",
"server_params": {
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
"stage_config_name": "qwen3_tts.yaml"
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"num_prompts": [
10,
40
],
"max_concurrency": [
1,
4
],
"random_input_len": 100,
"random_output_len": 100,
"extra_body": {
"voice": "Vivian",
"language": "English"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
}
]
+570
View File
@@ -0,0 +1,570 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for StageConfigFactory and related classes.
"""
from vllm_omni.config.stage_config import (
ModelPipeline,
StageConfig,
StageConfigFactory,
StageType,
)
class TestStageType:
"""Tests for StageType enum."""
def test_stage_type_values(self):
"""Test StageType enum values."""
assert StageType.LLM.value == "llm"
assert StageType.DIFFUSION.value == "diffusion"
def test_stage_type_from_string(self):
"""Test creating StageType from string."""
assert StageType("llm") == StageType.LLM
assert StageType("diffusion") == StageType.DIFFUSION
class TestStageConfig:
"""Tests for StageConfig dataclass."""
def test_minimal_config(self):
"""Test creating StageConfig with minimal required fields."""
config = StageConfig(stage_id=0, model_stage="thinker")
assert config.stage_id == 0
assert config.model_stage == "thinker"
assert config.stage_type == StageType.LLM
assert config.input_sources == []
assert config.final_output is False
assert config.worker_type is None
def test_full_config(self):
"""Test creating StageConfig with all fields."""
config = StageConfig(
stage_id=1,
model_stage="talker",
stage_type=StageType.LLM,
input_sources=[0],
custom_process_input_func="module.path.func",
final_output=True,
final_output_type="audio",
worker_type="ar",
scheduler_cls="path.to.Scheduler",
hf_config_name="talker_config",
is_comprehension=False,
)
assert config.stage_id == 1
assert config.model_stage == "talker"
assert config.input_sources == [0]
assert config.final_output_type == "audio"
assert config.worker_type == "ar"
def test_to_omegaconf_basic(self):
"""Test converting StageConfig to OmegaConf format."""
config = StageConfig(
stage_id=0,
model_stage="thinker",
stage_type=StageType.LLM,
worker_type="ar",
final_output=True,
final_output_type="text",
)
omega_config = config.to_omegaconf()
assert omega_config.stage_id == 0
assert omega_config.stage_type == "llm"
assert omega_config.engine_args.model_stage == "thinker"
assert omega_config.engine_args.worker_type == "ar"
assert omega_config.final_output is True
assert omega_config.final_output_type == "text"
# Legacy field name for backward compatibility
assert omega_config.engine_input_source == []
def test_to_omegaconf_with_runtime_overrides(self):
"""Test that runtime overrides are applied to OmegaConf output."""
config = StageConfig(
stage_id=0,
model_stage="thinker",
runtime_overrides={
"gpu_memory_utilization": 0.9,
"tensor_parallel_size": 2,
"devices": "0,1",
"max_batch_size": 64,
},
)
omega_config = config.to_omegaconf()
assert omega_config.engine_args.gpu_memory_utilization == 0.9
assert omega_config.engine_args.tensor_parallel_size == 2
assert omega_config.runtime.devices == "0,1"
assert omega_config.runtime.max_batch_size == 64
class TestModelPipeline:
"""Tests for ModelPipeline class."""
def test_valid_linear_dag(self):
"""Test validation of a valid linear DAG."""
stages = [
StageConfig(stage_id=0, model_stage="thinker", input_sources=[]),
StageConfig(stage_id=1, model_stage="talker", input_sources=[0]),
StageConfig(stage_id=2, model_stage="code2wav", input_sources=[1]),
]
pipeline = ModelPipeline(model_type="test", stages=stages)
errors = pipeline.validate_pipeline()
assert errors == [], f"Unexpected errors: {errors}"
def test_valid_branching_dag(self):
"""Test validation of a valid branching DAG."""
stages = [
StageConfig(stage_id=0, model_stage="input", input_sources=[]),
StageConfig(stage_id=1, model_stage="branch_a", input_sources=[0]),
StageConfig(stage_id=2, model_stage="branch_b", input_sources=[0]),
]
pipeline = ModelPipeline(model_type="test", stages=stages)
errors = pipeline.validate_pipeline()
assert errors == [], f"Unexpected errors: {errors}"
def test_missing_entry_point(self):
"""Test that missing entry point is detected."""
stages = [
StageConfig(stage_id=0, model_stage="stage_a", input_sources=[1]),
StageConfig(stage_id=1, model_stage="stage_b", input_sources=[0]),
]
pipeline = ModelPipeline(model_type="test", stages=stages)
errors = pipeline.validate_pipeline()
assert any("entry point" in e.lower() for e in errors)
def test_missing_dependency(self):
"""Test that missing stage reference is detected."""
stages = [
StageConfig(stage_id=0, model_stage="input", input_sources=[]),
StageConfig(stage_id=1, model_stage="output", input_sources=[99]), # Invalid
]
pipeline = ModelPipeline(model_type="test", stages=stages)
errors = pipeline.validate_pipeline()
assert any("non-existent" in e.lower() for e in errors)
def test_duplicate_stage_ids(self):
"""Test that duplicate stage IDs are detected."""
stages = [
StageConfig(stage_id=0, model_stage="stage_a", input_sources=[]),
StageConfig(stage_id=0, model_stage="stage_b", input_sources=[]), # Duplicate
]
pipeline = ModelPipeline(model_type="test", stages=stages)
errors = pipeline.validate_pipeline()
assert any("duplicate" in e.lower() for e in errors)
def test_self_reference(self):
"""Test that self-references are detected."""
stages = [
StageConfig(stage_id=0, model_stage="entry", input_sources=[]),
StageConfig(stage_id=1, model_stage="self_ref", input_sources=[1]), # Self
]
pipeline = ModelPipeline(model_type="test", stages=stages)
errors = pipeline.validate_pipeline()
assert any("itself" in e.lower() for e in errors)
def test_get_stage_by_id(self):
"""Test getting stage by ID."""
stages = [
StageConfig(stage_id=0, model_stage="thinker", input_sources=[]),
StageConfig(stage_id=1, model_stage="talker", input_sources=[0]),
]
pipeline = ModelPipeline(model_type="test", stages=stages)
stage = pipeline.get_stage(1)
assert stage is not None
assert stage.model_stage == "talker"
missing = pipeline.get_stage(99)
assert missing is None
def test_empty_pipeline(self):
"""Test validation of empty pipeline."""
pipeline = ModelPipeline(model_type="test", stages=[])
errors = pipeline.validate_pipeline()
assert any("no stages" in e.lower() for e in errors)
class TestStageConfigFactory:
"""Tests for StageConfigFactory class."""
def test_default_diffusion_no_yaml(self):
"""Test single-stage diffusion works without YAML config (@ZJY0516)."""
kwargs = {
"cache_backend": "none",
"cache_config": None,
"dtype": "bfloat16",
}
configs = StageConfigFactory.create_default_diffusion(kwargs)
assert len(configs) == 1
cfg = configs[0]
assert cfg["stage_id"] == 0
assert cfg["stage_type"] == "diffusion"
assert cfg["final_output"] is True
assert cfg["final_output_type"] == "image"
def test_default_diffusion_with_parallel_config(self):
"""Test diffusion config calculates devices from parallel_config."""
class MockParallelConfig:
world_size = 4
kwargs = {
"parallel_config": MockParallelConfig(),
"cache_backend": "tea_cache",
}
configs = StageConfigFactory.create_default_diffusion(kwargs)
assert configs[0]["runtime"]["devices"] == "0,1,2,3"
def test_per_stage_override_precedence(self):
"""Test that --stage-0-gpu-memory-utilization overrides global."""
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
cli_overrides = {
"gpu_memory_utilization": 0.5, # Global
"stage_0_gpu_memory_utilization": 0.9, # Per-stage override
}
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
# Per-stage should override global
assert overrides["gpu_memory_utilization"] == 0.9
def test_cli_override_forwards_engine_registered_args(self):
"""Test that any engine-registered CLI arg is forwarded (@wuhang2014)."""
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
cli_overrides = {
"gpu_memory_utilization": 0.9, # Well-known param
"custom_engine_flag": True, # Not in _INTERNAL_KEYS, so forwarded
}
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
assert overrides["gpu_memory_utilization"] == 0.9
assert overrides["custom_engine_flag"] is True
def test_cli_override_excludes_internal_keys(self):
"""Test that internal/orchestrator keys are not forwarded."""
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
cli_overrides = {
"gpu_memory_utilization": 0.9,
"model": "some_model", # Internal
"stage_configs_path": "/path", # Internal
"batch_timeout": 10, # Internal
}
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
assert overrides["gpu_memory_utilization"] == 0.9
assert "model" not in overrides
assert "stage_configs_path" not in overrides
assert "batch_timeout" not in overrides
def test_per_stage_override_excludes_internal_keys(self):
"""Test that per-stage overrides also skip internal keys."""
stage = StageConfig(stage_id=0, model_stage="thinker", input_sources=[])
cli_overrides = {
"stage_0_gpu_memory_utilization": 0.9,
"stage_0_model": "override_model", # Internal, should be skipped
"stage_0_batch_timeout": 5, # Internal, should be skipped
}
overrides = StageConfigFactory._merge_cli_overrides(stage, cli_overrides)
assert overrides["gpu_memory_utilization"] == 0.9
assert "model" not in overrides
assert "batch_timeout" not in overrides
class TestPipelineYamlParsing:
"""Tests for pipeline YAML file parsing (@ZJY0516)."""
def test_parse_qwen3_omni_moe_yaml(self, tmp_path):
"""Test parsing the qwen3_omni_moe pipeline YAML."""
yaml_content = """\
model_type: qwen3_omni_moe
async_chunk: false
stages:
- stage_id: 0
model_stage: thinker
stage_type: llm
input_sources: []
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
hf_config_name: thinker_config
final_output: true
final_output_type: text
is_comprehension: true
- stage_id: 1
model_stage: talker
stage_type: llm
input_sources: [0]
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
hf_config_name: talker_config
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
- stage_id: 2
model_stage: code2wav
stage_type: llm
input_sources: [1]
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
hf_config_name: thinker_config
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
final_output: true
final_output_type: audio
"""
yaml_file = tmp_path / "qwen3_omni_moe.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "qwen3_omni_moe")
assert pipeline.model_type == "qwen3_omni_moe"
assert len(pipeline.stages) == 3
# Stage 0: thinker
s0 = pipeline.stages[0]
assert s0.stage_id == 0
assert s0.model_stage == "thinker"
assert s0.stage_type == StageType.LLM
assert s0.input_sources == []
assert s0.worker_type == "ar"
assert s0.final_output is True
assert s0.final_output_type == "text"
assert s0.is_comprehension is True
# Stage 1: talker
s1 = pipeline.stages[1]
assert s1.stage_id == 1
assert s1.input_sources == [0]
assert s1.custom_process_input_func == (
"vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker"
)
assert s1.final_output is False
# Stage 2: code2wav
s2 = pipeline.stages[2]
assert s2.stage_id == 2
assert s2.input_sources == [1]
assert s2.worker_type == "generation"
assert s2.final_output_type == "audio"
def test_parse_yaml_with_legacy_engine_input_source(self, tmp_path):
"""Test backward compatibility with engine_input_source field."""
yaml_content = """\
model_type: legacy_model
stages:
- stage_id: 0
model_stage: entry
stage_type: llm
- stage_id: 1
model_stage: downstream
stage_type: llm
engine_input_source: [0]
"""
yaml_file = tmp_path / "legacy.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "legacy_model")
assert pipeline.stages[1].input_sources == [0]
def test_parse_yaml_with_connectors_and_edges(self, tmp_path):
"""Test parsing pipeline with optional connectors and edges."""
yaml_content = """\
model_type: test_model
stages:
- stage_id: 0
model_stage: entry
stage_type: llm
input_sources: []
connectors:
type: ray
edges:
- from: 0
to: 1
"""
yaml_file = tmp_path / "with_connectors.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "test_model")
assert pipeline.connectors == {"type": "ray"}
assert pipeline.edges == [{"from": 0, "to": 1}]
def test_parsed_pipeline_passes_validation(self, tmp_path):
"""Test that a well-formed YAML produces a valid pipeline."""
yaml_content = """\
model_type: valid_model
stages:
- stage_id: 0
model_stage: entry
stage_type: llm
input_sources: []
final_output: true
final_output_type: text
- stage_id: 1
model_stage: next
stage_type: llm
input_sources: [0]
"""
yaml_file = tmp_path / "valid.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "valid_model")
errors = pipeline.validate_pipeline()
assert errors == [], f"Unexpected validation errors: {errors}"
def test_parse_diffusion_stage_type(self, tmp_path):
"""Test parsing a diffusion stage type from YAML."""
yaml_content = """\
model_type: diff_model
stages:
- stage_id: 0
model_stage: dit
stage_type: diffusion
input_sources: []
final_output: true
final_output_type: image
"""
yaml_file = tmp_path / "diffusion.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "diff_model")
assert pipeline.stages[0].stage_type == StageType.DIFFUSION
def test_parse_mimo_audio_yaml(self, tmp_path):
"""Test parsing the MiMo Audio pipeline YAML."""
yaml_content = """\
model_type: mimo_audio
stages:
- stage_id: 0
model_stage: fused_thinker_talker
stage_type: llm
input_sources: []
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
is_comprehension: true
final_output: true
final_output_type: text
- stage_id: 1
model_stage: code2wav
stage_type: llm
input_sources: [0]
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav
final_output: true
final_output_type: audio
"""
yaml_file = tmp_path / "mimo_audio.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "mimo_audio")
assert pipeline.model_type == "mimo_audio"
assert len(pipeline.stages) == 2
# Stage 0: fused_thinker_talker
s0 = pipeline.stages[0]
assert s0.stage_id == 0
assert s0.model_stage == "fused_thinker_talker"
assert s0.input_sources == []
assert s0.worker_type == "ar"
assert s0.is_comprehension is True
assert s0.final_output is True
assert s0.final_output_type == "text"
# Stage 1: code2wav
s1 = pipeline.stages[1]
assert s1.stage_id == 1
assert s1.input_sources == [0]
assert s1.worker_type == "generation"
assert s1.custom_process_input_func == (
"vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav"
)
assert s1.final_output is True
assert s1.final_output_type == "audio"
# Pipeline validation
errors = pipeline.validate_pipeline()
assert errors == [], f"Unexpected errors: {errors}"
class TestAsyncChunk:
"""Tests for async_chunk pipeline flag."""
def test_model_pipeline_async_chunk_default(self):
"""Test ModelPipeline defaults async_chunk to False."""
stages = [StageConfig(stage_id=0, model_stage="entry", input_sources=[])]
pipeline = ModelPipeline(model_type="test", stages=stages)
assert pipeline.async_chunk is False
def test_model_pipeline_async_chunk_set(self):
"""Test ModelPipeline with async_chunk=True."""
stages = [StageConfig(stage_id=0, model_stage="entry", input_sources=[])]
pipeline = ModelPipeline(model_type="test", stages=stages, async_chunk=True)
assert pipeline.async_chunk is True
def test_parse_async_chunk_from_yaml(self, tmp_path):
"""Test that async_chunk is parsed from pipeline YAML."""
yaml_content = """\
model_type: test_async
async_chunk: true
stages:
- stage_id: 0
model_stage: entry
stage_type: llm
input_sources: []
"""
yaml_file = tmp_path / "async.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "test_async")
assert pipeline.async_chunk is True
def test_parse_missing_async_chunk_defaults_false(self, tmp_path):
"""Test that missing async_chunk defaults to False."""
yaml_content = """\
model_type: test_no_async
stages:
- stage_id: 0
model_stage: entry
stage_type: llm
input_sources: []
"""
yaml_file = tmp_path / "no_async.yaml"
yaml_file.write_text(yaml_content)
pipeline = StageConfigFactory._parse_pipeline_yaml(yaml_file, "test_no_async")
assert pipeline.async_chunk is False
class TestArchitectureFallback:
"""Tests for architecture-based model detection fallback."""
def test_architecture_models_mapping_exists(self):
"""Test that _ARCHITECTURE_MODELS contains expected entries."""
assert "MiMoAudioForConditionalGeneration" in StageConfigFactory._ARCHITECTURE_MODELS
assert StageConfigFactory._ARCHITECTURE_MODELS["MiMoAudioForConditionalGeneration"] == "mimo_audio"
assert "HunyuanImage3ForCausalMM" in StageConfigFactory._ARCHITECTURE_MODELS
assert StageConfigFactory._ARCHITECTURE_MODELS["HunyuanImage3ForCausalMM"] == "hunyuan_image3"
def test_mimo_audio_in_pipeline_models(self):
"""Test that mimo_audio is registered in PIPELINE_MODELS."""
assert "mimo_audio" in StageConfigFactory.PIPELINE_MODELS
+39 -27
View File
@@ -20,6 +20,8 @@ from typing_extensions import ParamSpec
from vllm.platforms import current_platform
from vllm.utils.torch_utils import cuda_device_count_stateless
from vllm_omni.platforms import current_omni_platform
_P = ParamSpec("_P")
if current_platform.is_rocm():
@@ -441,27 +443,11 @@ def npu_marks(*, res: str, num_cards: int):
return [mark for mark in [test_platform, test_resource, test_distributed] if mark is not None]
def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
"""
Decorate a test for multiple hardware platforms with a single call.
Automatically wraps the test with @create_new_process_for_each_test() for distributed tests.
Args:
res: Mapping from platform to resource type. Supported platforms/resources:
- cuda: L4, H100
- rocm: MI325
- npu: A2, A3
num_cards: Number of cards required. Can be:
- int: same card count for all platforms (default: 1)
- dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2}
Example:
@hardware_test(
res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
num_cards={"cuda": 2, "rocm": 2, "npu": 2},
)
def test_multi_platform():
...
Get a collection of pytest marks to apply for `@hardware_test`,
including CUDA, ROCm, and NPU,
based on the specified platforms and resources.
"""
# Validate platforms
# Don't validate platform details in this decorator
@@ -484,7 +470,7 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
num_cards_dict[platform] = 1
# Collect marks from all platforms
all_marks: list[Callable[[Callable[_P, None]], Callable[_P, None]]] = []
all_marks: list[pytest.MarkDecorator] = []
for platform, resource in res.items():
cards = num_cards_dict[platform]
if platform == "cuda" or platform == "rocm":
@@ -494,6 +480,32 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
else:
raise ValueError(f"Unsupported platform: {platform}")
all_marks.extend(marks)
return all_marks
def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
"""
Decorate a test for multiple hardware platforms with a single call.
Automatically wraps the test with @create_new_process_for_each_test() for distributed tests.
Args:
res: Mapping from platform to resource type. Supported platforms/resources:
- cuda: L4, H100
- rocm: MI325
- npu: A2, A3
num_cards: Number of cards required. Can be:
- int: same card count for all platforms (default: 1)
- dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2}
Example:
@hardware_test(
res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
num_cards={"cuda": 2, "rocm": 2, "npu": 2},
)
def test_multi_platform():
...
"""
all_marks = hardware_marks(res=res, num_cards=num_cards)
def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
func = f
@@ -504,8 +516,8 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
return wrapper
class GPUMemoryMonitor:
"""Poll global device memory usage via CUDA APIs."""
class DeviceMemoryMonitor:
"""Poll global device memory usage."""
def __init__(self, device_index: int, interval: float = 0.05):
self.device_index = device_index
@@ -518,8 +530,8 @@ class GPUMemoryMonitor:
def monitor_loop() -> None:
while not self._stop_event.is_set():
try:
with torch.cuda.device(self.device_index):
free_bytes, total_bytes = torch.cuda.mem_get_info()
with current_omni_platform.device(self.device_index):
free_bytes, total_bytes = current_omni_platform.mem_get_info()
used_mb = (total_bytes - free_bytes) / (1024**2)
self._peak_used_mb = max(self._peak_used_mb, used_mb)
except Exception:
@@ -537,8 +549,8 @@ class GPUMemoryMonitor:
@property
def peak_used_mb(self) -> float:
fallback_alloc = torch.cuda.max_memory_allocated(device=self.device_index) / (1024**2)
fallback_reserved = torch.cuda.max_memory_reserved(device=self.device_index) / (1024**2)
fallback_alloc = current_omni_platform.max_memory_allocated(device=self.device_index) / (1024**2)
fallback_reserved = current_omni_platform.max_memory_reserved(device=self.device_index) / (1024**2)
return max(self._peak_used_mb, fallback_alloc, fallback_reserved)
def __del__(self):