[Test] Unify L2/L3 test layout, Buildkite steps, and test helpers (#2556)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Co-authored-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
wangyuandHuang, Zeyu authored and GitHub committed 2026-05-08 14:12:23 +08:00
1 parent b85833eeec
commit 6f2ad7b403
134 files changed
+4904 -4185

No files matched your search

@@ -282,7 +282,7 @@ def _run_with_custom_transformer(model: str, output_dir: Path) -> list[Image.Ima
return frames
@pytest.mark.advanced_model
@pytest.mark.full_model
@pytest.mark.benchmark
@pytest.mark.diffusion
@pytest.mark.skipif(
@@ -381,7 +381,7 @@ def _run_vllm_omni_serving(model: str, output_dir: Path) -> list[Image.Image]:
return frames
@pytest.mark.advanced_model
@pytest.mark.full_model
@pytest.mark.benchmark
@pytest.mark.diffusion
@pytest.mark.skipif(
@@ -2,44 +2,40 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import sys
from pathlib import Path
import numpy as np
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from vllm_omni import Omni
# Prefer a tiny/random checkpoint for CI.
# Override in CI if needed: AUDIOX_TEST_MODEL=<model-or-local-path>
models = [os.environ.get("AUDIOX_TEST_MODEL", "zhangj1an/audiox_random")]
# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize
_OMNI_RUNNER_PARAMS = [(m, None, {"model_class_name": "AudioXPipeline"}) for m in models]
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True, ids=models),
]
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_audiox_model(model_name: str):
m = Omni(model=model_name, model_class_name="AudioXPipeline")
def test_audiox_model(omni_runner: OmniRunner) -> None:
# Keep runtime short for CI.
seconds_total = 2.0
# AudioXPipeline always emits 44.1 kHz stereo (advertised via class-level
# ``audio_sample_rate``); the trimmed output should match this rate.
sample_rate = 44100
outputs = m.generate(
outputs = omni_runner.omni.generate(
prompts={"prompt": "A dog barking in a quiet park."},
sampling_params_list=OmniDiffusionSamplingParams(
num_inference_steps=4,
+110
View File
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end tests for Bagel text2text and img2text (understanding) tasks.
These tests validate that the Bagel multistage pipeline correctly generates
text output for understanding tasks, matching reference results.
Equivalent to running:
python3 examples/offline_inference/bagel/end2end.py \
--modality text2text \
--prompts "Where is the capital of France?"
python3 examples/offline_inference/bagel/end2end.py \
--modality img2text \
--prompts "Please describe this image" \
--image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import pytest
from vllm.assets.image import ImageAsset
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml")
REFERENCE_TEXT_TEXT2TEXT = "The capital of France is Paris."
REFERENCE_TEXT_IMG2TEXT = (
"This is a photo of a wooden boardwalk or pathway that leads through "
"tall green grass. The path appears to be in a natural setting, possibly "
"a wetland or marsh area. The sky above is blue with some scattered "
"clouds, suggesting it might be a sunny day. The overall scene looks "
"peaceful and serene."
)
# (model, stage_config_path, extra_omni_kwargs) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
_OMNI_RUNNER_PARAM = (MODEL_NAME, STAGE_CONFIG)
pytestmark = [
pytest.mark.advanced_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def _extract_text(omni_outputs: list) -> str:
"""Extract generated text from OmniRequestOutput list."""
for req_output in omni_outputs:
ro = getattr(req_output, "request_output", None)
if ro and getattr(ro, "outputs", None):
return "".join(getattr(o, "text", "") or "" for o in ro.outputs)
return ""
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_text2text(run_level, omni_runner: OmniRunner) -> None:
"""Test Bagel text2text produces correct text output."""
omni = omni_runner.omni
prompt = "<|im_start|>user\nWhere is the capital of France?<|im_end|>\n<|im_start|>assistant\n"
params_list = omni.default_sampling_params_list
omni_outputs = list(
omni.generate(
prompts=[{"prompt": prompt, "modalities": ["text"]}],
sampling_params_list=params_list,
)
)
assert len(omni_outputs) > 0, "No outputs returned"
text = _extract_text(omni_outputs)
assert len(text) > 0, "Generated text is empty"
if run_level == "advanced_model":
assert text == REFERENCE_TEXT_TEXT2TEXT, f"Text mismatch: expected {REFERENCE_TEXT_TEXT2TEXT!r}, got {text!r}"
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_img2text(run_level, omni_runner: OmniRunner) -> None:
"""Test Bagel img2text produces correct text output."""
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
omni = omni_runner.omni
prompt = "<|im_start|>user\n<|image_pad|>\nPlease describe this image<|im_end|>\n<|im_start|>assistant\n"
params_list = omni.default_sampling_params_list
omni_outputs = list(
omni.generate(
prompts=[
{
"prompt": prompt,
"multi_modal_data": {"image": input_image},
"modalities": ["text"],
}
],
sampling_params_list=params_list,
)
)
assert len(omni_outputs) > 0, "No outputs returned"
text = _extract_text(omni_outputs)
assert len(text) > 0, "Generated text is empty"
if run_level in ["advanced_model", "full_model"]:
assert "wooden boardwalk" in text.lower(), f"Text mismatch: expected 'wooden boardwalk' in {text!r}"
@@ -2,7 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for BAGEL LoRA support (Stage 1 / DiT).
Expanded end-to-end test for BAGEL in offline mode.
This test file primarily covers end-to-end tests for LoRA support (Stage 1 / DiT).
Validates that LoRA adapters are correctly loaded, applied with controllable
scale, and cleanly deactivated. Uses a synthetic rank-1 adapter targeting the
@@ -32,7 +34,7 @@ from safetensors.torch import save_file
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
@@ -41,26 +43,15 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
BAGEL_STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml")
DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>"
# (model, stage_configs_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
_OMNI_RUNNER_PARAM = (MODEL, BAGEL_STAGE_CONFIG)
# ---------------------------------------------------------------------------
# Helpers (reused from test_bagel_text2img.py patterns)
# ---------------------------------------------------------------------------
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
if run_level == "advanced_model":
return modify_stage_config(
config_path,
deletes={
"stages": {
0: ["load_format"],
1: ["load_format"],
}
},
)
return config_path
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 10) -> list[OmniSamplingParams]:
params_list = omni.default_sampling_params_list
if len(params_list) > 1:
@@ -146,50 +137,52 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest:
# ---------------------------------------------------------------------------
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_lora_scale_and_deactivation(run_level, tmp_path):
def test_bagel_lora_scale_and_deactivation(omni_runner: OmniRunner, tmp_path) -> None:
"""Validate LoRA effect, bounded perturbation, and clean deactivation."""
config_path = _resolve_deploy_config(BAGEL_STAGE_CONFIG, run_level)
with OmniRunner(MODEL, stage_configs_path=config_path) as runner:
omni = runner.omni
lora_request = _make_file_lora_request(tmp_path / "bagel_lora")
omni = omni_runner.omni
lora_request = _make_file_lora_request(tmp_path / "bagel_lora")
# 1) Baseline (no LoRA)
baseline = _generate_bagel_image(omni)
# 1) Baseline (no LoRA)
baseline = _generate_bagel_image(omni)
# 2) LoRA with scale=1.0
img_1x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=1.0)
# 2) LoRA with scale=1.0
img_1x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=1.0)
# 3) LoRA with scale=2.0
img_2x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=2.0)
# 3) LoRA with scale=2.0
img_2x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=2.0)
# 4) No LoRA again (deactivation)
restored = _generate_bagel_image(omni)
# 4) No LoRA again (deactivation)
restored = _generate_bagel_image(omni)
baseline_arr = np.array(baseline, dtype=np.int16)
img_1x_arr = np.array(img_1x, dtype=np.int16)
img_2x_arr = np.array(img_2x, dtype=np.int16)
restored_arr = np.array(restored, dtype=np.int16)
baseline_arr = np.array(baseline, dtype=np.int16)
img_1x_arr = np.array(img_1x, dtype=np.int16)
img_2x_arr = np.array(img_2x, dtype=np.int16)
restored_arr = np.array(restored, dtype=np.int16)
diff_1x = np.abs(baseline_arr - img_1x_arr).mean()
diff_2x = np.abs(baseline_arr - img_2x_arr).mean()
diff_restored = np.abs(baseline_arr - restored_arr).mean()
diff_1x = np.abs(baseline_arr - img_1x_arr).mean()
diff_2x = np.abs(baseline_arr - img_2x_arr).mean()
diff_restored = np.abs(baseline_arr - restored_arr).mean()
# (a) Adapter has visible effect at both scales
assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}"
assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}"
# (a) Adapter has visible effect at both scales
assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}"
assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}"
# (b) Different scales produce different outputs
assert not np.isclose(diff_1x, diff_2x, atol=1.0), (
f"LoRA scale has no effect: diff_1x={diff_1x:.2f}, diff_2x={diff_2x:.2f}"
)
# (b) Different scales produce different outputs
assert not np.isclose(diff_1x, diff_2x, atol=1.0), (
f"LoRA scale has no effect: diff_1x={diff_1x:.2f}, diff_2x={diff_2x:.2f}"
)
# (c) Output is not corrupted (scale=2.0 can produce ~2x the diff of scale=1.0)
assert diff_1x < 80, f"LoRA output looks corrupted: diff_1x={diff_1x}"
assert diff_2x < 120, f"LoRA output looks corrupted: diff_2x={diff_2x}"
# (c) Output is not corrupted (scale=2.0 can produce ~2x the diff of scale=1.0)
assert diff_1x < 80, f"LoRA output looks corrupted: diff_1x={diff_1x}"
assert diff_2x < 120, f"LoRA output looks corrupted: diff_2x={diff_2x}"
# (d) Deactivation fully restores base model
assert diff_restored == 0.0, f"Base model not restored after LoRA deactivation: diff={diff_restored}"
# (d) Deactivation fully restores base model
assert diff_restored == 0.0, f"Base model not restored after LoRA deactivation: diff={diff_restored}"
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for Bagel img2img generation.
This test validates that the Bagel model generates images from an input image
and text prompt that match expected reference pixel values within a ±10 tolerance.
Equivalent to running:
python3 examples/offline_inference/bagel/end2end.py \
--prompts "Change the grass color to red" \
--modality img2img --step 15 \
--image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
"""
import socket
from typing import Any
import pytest
from PIL import Image
from vllm.assets.image import ImageAsset
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
# Reference pixel data extracted from the known-good output image
# Generated with seed=52, num_inference_steps=15,
# prompt='Change the grass color to red',
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (156, 172, 217)},
{"position": (400, 50), "rgb": (105, 144, 217)},
{"position": (700, 100), "rgb": (118, 159, 232)},
{"position": (150, 400), "rgb": (180, 22, 52)},
{"position": (512, 336), "rgb": (221, 211, 194)},
{"position": (700, 400), "rgb": (192, 10, 46)},
{"position": (100, 600), "rgb": (102, 12, 22)},
{"position": (400, 600), "rgb": (161, 28, 47)},
{"position": (700, 600), "rgb": (100, 87, 94)},
{"position": (256, 256), "rgb": (181, 201, 221)},
]
if current_omni_platform.is_rocm():
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (156, 172, 217)},
{"position": (400, 50), "rgb": (105, 144, 217)},
{"position": (700, 100), "rgb": (118, 159, 232)},
{"position": (150, 400), "rgb": (180, 22, 52)},
{"position": (512, 336), "rgb": (221, 211, 194)},
{"position": (700, 400), "rgb": (192, 10, 46)},
{"position": (100, 600), "rgb": (102, 12, 22)},
{"position": (400, 600), "rgb": (161, 28, 47)},
{"position": (700, 600), "rgb": (100, 87, 94)},
{"position": (256, 256), "rgb": (181, 201, 221)},
]
PIXEL_TOLERANCE = 10
DEFAULT_PROMPT = "<|fim_middle|><|im_start|>Change the grass color to red<|im_end|>"
EXPECTED_OUTPUT_SIZE = (1024, 672)
def _load_input_image() -> Image.Image:
"""Load the test input image via vllm's ImageAsset."""
return ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
def _find_free_port() -> int:
"""Find and return a free ephemeral port by binding to port 0."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
s.listen(1)
port = s.getsockname()[1]
return port
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
"""Configure sampling parameters for Bagel img2img generation.
Args:
omni: The Omni instance to get default params from.
num_inference_steps: Number of inference steps for the diffusion stage.
Returns:
Configured sampling params list.
"""
params_list = omni.default_sampling_params_list
if len(params_list) > 1:
params_list[1].num_inference_steps = num_inference_steps # type: ignore
params_list[1].extra_args = { # type: ignore
"cfg_text_scale": 4.0,
"cfg_img_scale": 1.5,
}
return params_list
def _extract_generated_image(omni_outputs: list) -> Image.Image | None:
"""Extract the generated image from Omni outputs.
Args:
omni_outputs: List of outputs from omni.generate().
Returns:
The first generated PIL Image, or None if no image found.
"""
for req_output in omni_outputs:
if images := getattr(req_output, "images", None):
return images[0]
if hasattr(req_output, "request_output") and req_output.request_output:
stage_out = req_output.request_output
if hasattr(stage_out, "images") and stage_out.images:
return stage_out.images[0]
return None
def _validate_pixels(
image: Image.Image,
reference_pixels: list[dict[str, Any]] = REFERENCE_PIXELS,
tolerance: int = PIXEL_TOLERANCE,
) -> None:
"""Validate that image pixels match expected reference values.
Args:
image: The PIL Image to validate.
reference_pixels: List of dicts with 'position' (x, y) and 'rgb' (R, G, B).
tolerance: Maximum allowed difference per color channel.
Raises:
AssertionError: If any pixel differs beyond tolerance.
"""
for ref in reference_pixels:
x, y = ref["position"]
expected = ref["rgb"]
actual = image.getpixel((x, y))[:3]
assert all(abs(a - e) <= tolerance for a, e in zip(actual, expected)), (
f"Pixel mismatch at ({x}, {y}): expected {expected}, got {actual}"
)
def _generate_bagel_img2img(
omni: Omni,
input_image: Image.Image,
prompt: str = DEFAULT_PROMPT,
) -> Image.Image:
"""Generate an image using Bagel model with img2img pipeline.
Args:
omni: The Omni instance to use for generation.
input_image: The input PIL Image for img2img.
prompt: The text prompt for image editing.
Returns:
The generated PIL Image.
Raises:
AssertionError: If no image is generated or size is incorrect.
"""
params_list = _configure_sampling_params(omni)
omni_outputs = list(
omni.generate(
prompts=[
{
"prompt": prompt,
"multi_modal_data": {"img2img": input_image},
"modalities": ["img2img"],
}
],
sampling_params_list=params_list,
)
)
generated_image = _extract_generated_image(omni_outputs)
assert generated_image is not None, "No images generated"
assert generated_image.size == EXPECTED_OUTPUT_SIZE, f"Expected {EXPECTED_OUTPUT_SIZE}, got {generated_image.size}"
return generated_image
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
"""Resolve deploy config based on run level.
For advanced_model (real weights), strip load_format: dummy so the model
falls back to loading real weights from HuggingFace.
"""
if run_level == "advanced_model":
return modify_stage_config(
config_path,
deletes={
"stages": {
0: ["load_format"],
1: ["load_format"],
}
},
)
return config_path
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_img2img_shared_memory_connector(run_level):
"""Test Bagel img2img with shared memory connector."""
input_image = _load_input_image()
config_path = _resolve_deploy_config(BAGEL_CI_DEPLOY, run_level)
with OmniRunner(
"ByteDance-Seed/BAGEL-7B-MoT",
stage_configs_path=config_path,
) as runner:
generated_image = _generate_bagel_img2img(runner.omni, input_image)
if run_level == "advanced_model":
_validate_pixels(generated_image)
@@ -1,369 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for Bagel text2img generation.
This test validates that the Bagel model generates images that match
expected reference pixel values within a ±5 tolerance.
Equivalent to running:
python3 examples/offline_inference/bagel/end2end.py \
--prompts "A futuristic city skyline at twilight, cyberpunk style" \
--modality text2img --step 15
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import signal
import socket
import subprocess
import tempfile
import time
from typing import Any
import pytest
from PIL import Image
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.platforms import current_omni_platform
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
# Reference pixel data extracted from the known-good output image
# Each entry contains (x, y) position and expected (R, G, B) values
# "Generated with seed=52, num_inference_steps=15,
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (115, 113, 94)},
{"position": (400, 50), "rgb": (159, 160, 144)},
{"position": (700, 100), "rgb": (164, 151, 123)},
{"position": (150, 400), "rgb": (120, 121, 107)},
{"position": (512, 512), "rgb": (165, 133, 127)},
{"position": (700, 400), "rgb": (217, 130, 66)},
{"position": (100, 700), "rgb": (191, 168, 152)},
{"position": (400, 700), "rgb": (130, 96, 77)},
{"position": (700, 700), "rgb": (247, 203, 140)},
{"position": (256, 256), "rgb": (167, 156, 150)},
]
if current_omni_platform.is_rocm():
REFERENCE_PIXELS = [
{"position": (100, 100), "rgb": (115, 113, 94)},
{"position": (400, 50), "rgb": (159, 160, 144)},
{"position": (700, 100), "rgb": (164, 151, 123)},
{"position": (150, 400), "rgb": (120, 121, 107)},
{"position": (512, 512), "rgb": (165, 133, 127)},
{"position": (700, 400), "rgb": (217, 130, 66)},
{"position": (100, 700), "rgb": (191, 168, 152)},
{"position": (400, 700), "rgb": (130, 96, 77)},
{"position": (700, 700), "rgb": (247, 203, 140)},
{"position": (256, 256), "rgb": (167, 156, 150)},
]
# Maximum allowed difference per color channel
PIXEL_TOLERANCE = 10
# Default test prompt
DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>"
def _find_free_port() -> int:
"""Find and return a free ephemeral port by binding to port 0."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
s.listen(1)
port = s.getsockname()[1]
return port
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
"""Configure sampling parameters for Bagel text2img generation.
Args:
omni: The Omni instance to get default params from.
num_inference_steps: Number of inference steps for the diffusion stage.
Returns:
Configured sampling params list.
"""
params_list = omni.default_sampling_params_list
if len(params_list) > 1:
params_list[1].num_inference_steps = num_inference_steps # type: ignore
params_list[1].extra_args = { # type: ignore
"cfg_text_scale": 4.0,
"cfg_img_scale": 1.5,
}
return params_list
def _extract_generated_image(omni_outputs: list) -> Image.Image | None:
"""Extract the generated image from Omni outputs.
Args:
omni_outputs: List of outputs from omni.generate().
Returns:
The first generated PIL Image, or None if no image found.
"""
for req_output in omni_outputs:
if images := getattr(req_output, "images", None):
return images[0]
if hasattr(req_output, "request_output") and req_output.request_output:
stage_out = req_output.request_output
if hasattr(stage_out, "images") and stage_out.images:
return stage_out.images[0]
return None
def _validate_pixels(
image: Image.Image,
reference_pixels: list[dict[str, Any]] = REFERENCE_PIXELS,
tolerance: int = PIXEL_TOLERANCE,
) -> None:
"""Validate that image pixels match expected reference values.
Args:
image: The PIL Image to validate.
reference_pixels: List of dicts with 'position' (x, y) and 'rgb' (R, G, B).
tolerance: Maximum allowed difference per color channel.
Raises:
AssertionError: If any pixel differs beyond tolerance.
"""
for ref in reference_pixels:
x, y = ref["position"]
expected = ref["rgb"]
actual = image.getpixel((x, y))[:3]
assert all(abs(a - e) <= tolerance for a, e in zip(actual, expected)), (
f"Pixel mismatch at ({x}, {y}): expected {expected}, got {actual}"
)
def _generate_bagel_image(omni: Omni, prompt: str = DEFAULT_PROMPT) -> Image.Image:
"""Generate an image using Bagel model with configured parameters.
Args:
omni: The Omni instance to use for generation.
prompt: The text prompt for image generation.
Returns:
The generated PIL Image.
Raises:
AssertionError: If no image is generated or size is incorrect.
"""
params_list = _configure_sampling_params(omni)
omni_outputs = list(
omni.generate(
prompts=[{"prompt": prompt, "modalities": ["image"]}],
sampling_params_list=params_list,
)
)
generated_image = _extract_generated_image(omni_outputs)
assert generated_image is not None, "No images generated"
assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}"
return generated_image
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
"""Resolve deploy config based on run level.
For advanced_model (real weights), strip load_format: dummy so the model
falls back to loading real weights from HuggingFace.
"""
if run_level == "advanced_model":
return modify_stage_config(
config_path,
deletes={
"stages": {
0: ["load_format"],
1: ["load_format"],
}
},
)
return config_path
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_text2img_shared_memory_connector(run_level):
"""Test Bagel text2img with shared memory connector."""
config_path = _resolve_deploy_config(BAGEL_CI_DEPLOY, run_level)
with OmniRunner(
"ByteDance-Seed/BAGEL-7B-MoT",
stage_configs_path=config_path,
) as runner:
generated_image = _generate_bagel_image(runner.omni)
if run_level == "advanced_model":
_validate_pixels(generated_image)
def _wait_for_port(host: str, port: int, timeout: int = 30) -> bool:
"""Wait for a port to become available.
Args:
host: The host address.
port: The port number.
timeout: Maximum seconds to wait.
Returns:
True if port becomes available, False otherwise.
"""
for _ in range(timeout):
try:
with socket.create_connection((host, port), timeout=1):
return True
except (TimeoutError, ConnectionRefusedError):
time.sleep(1)
return False
def _is_mooncake_master_available() -> bool:
"""Check if mooncake_master binary is present and can actually execute."""
import shutil
binary = shutil.which("mooncake_master")
if binary is None:
return False
try:
result = subprocess.run(
[binary, "--help"],
capture_output=True,
timeout=5,
)
return result.returncode != 127
except (subprocess.TimeoutExpired, OSError):
return True
def _cleanup_mooncake_processes(timeout_secs: int = 5) -> None:
"""Clean up any existing mooncake_master processes.
Args:
timeout_secs: Maximum seconds to wait for graceful termination.
"""
subprocess.run(
["pkill", "-f", "mooncake_master"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
start_time = time.time()
while time.time() - start_time < timeout_secs:
result = subprocess.run(
["pgrep", "-f", "mooncake_master"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
if result.returncode != 0:
break
time.sleep(0.5)
else:
subprocess.run(
["pkill", "-9", "-f", "mooncake_master"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
time.sleep(1)
def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
"""Load Mooncake config from CI overlay and substitute placeholders.
Args:
host: Mooncake host address.
rpc_port: RPC port for Mooncake master.
http_port: HTTP metadata server port.
Returns:
Path to the temporary config file with substituted values.
"""
with open(BAGEL_MOONCAKE_CI_DEPLOY) as f:
config_content = f.read()
config_content = config_content.replace("${MOONCAKE_HOST}", host)
config_content = config_content.replace("${MOONCAKE_RPC_PORT}", str(rpc_port))
config_content = config_content.replace("${MOONCAKE_HTTP_PORT}", str(http_port))
temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False)
temp_file.write(config_content)
temp_file.close()
return temp_file.name
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_bagel_text2img_mooncake_connector(run_level):
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
if not _is_mooncake_master_available():
raise RuntimeError(
"mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)"
)
MOONCAKE_HOST = "127.0.0.1"
MOONCAKE_RPC_PORT = _find_free_port()
MOONCAKE_HTTP_PORT = _find_free_port()
MOONCAKE_METRICS_PORT = _find_free_port()
mooncake_master_proc = None
temp_config_file = None
try:
_cleanup_mooncake_processes()
# Start mooncake_master
mooncake_master_proc = subprocess.Popen(
[
"mooncake_master",
f"--rpc_port={MOONCAKE_RPC_PORT}",
"--enable_http_metadata_server=true",
"--http_metadata_server_host=0.0.0.0",
f"--http_metadata_server_port={MOONCAKE_HTTP_PORT}",
f"--metrics_port={MOONCAKE_METRICS_PORT}",
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
preexec_fn=os.setsid,
)
assert _wait_for_port(MOONCAKE_HOST, MOONCAKE_RPC_PORT), "mooncake_master failed to start"
# Create temp config and initialize Omni
temp_config_file = _load_mooncake_config(
host=MOONCAKE_HOST,
rpc_port=MOONCAKE_RPC_PORT,
http_port=MOONCAKE_HTTP_PORT,
)
temp_config_file = _resolve_deploy_config(temp_config_file, run_level)
with OmniRunner(
"ByteDance-Seed/BAGEL-7B-MoT",
stage_configs_path=temp_config_file,
stage_init_timeout=300,
) as runner:
generated_image = _generate_bagel_image(runner.omni)
if run_level == "advanced_model":
_validate_pixels(generated_image)
finally:
if temp_config_file:
try:
os.unlink(temp_config_file)
except OSError:
pass
if mooncake_master_proc:
try:
os.killpg(os.getpgid(mooncake_master_proc.pid), signal.SIGKILL)
except OSError:
pass
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end tests for Bagel text2text and img2text (understanding) tasks.
These tests validate that the Bagel multistage pipeline correctly generates
text output for understanding tasks, matching reference results.
Equivalent to running:
python3 examples/offline_inference/bagel/end2end.py \
--modality text2text \
--prompts "Where is the capital of France?"
python3 examples/offline_inference/bagel/end2end.py \
--modality img2text \
--prompts "Please describe this image" \
--image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
import pytest
from vllm.assets.image import ImageAsset
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml")
REFERENCE_TEXT_TEXT2TEXT = "The capital of France is Paris."
REFERENCE_TEXT_IMG2TEXT = (
"This is a photo of a wooden boardwalk or pathway that leads through "
"tall green grass. The path appears to be in a natural setting, possibly "
"a wetland or marsh area. The sky above is blue with some scattered "
"clouds, suggesting it might be a sunny day. The overall scene looks "
"peaceful and serene."
)
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
"""Strip load_format: dummy for advanced_model (real weights)."""
if run_level == "advanced_model":
return modify_stage_config(
config_path,
deletes={
"stages": {
0: ["load_format"],
1: ["load_format"],
}
},
)
return config_path
def _extract_text(omni_outputs: list) -> str:
"""Extract generated text from OmniRequestOutput list."""
for req_output in omni_outputs:
ro = getattr(req_output, "request_output", None)
if ro and getattr(ro, "outputs", None):
return "".join(getattr(o, "text", "") or "" for o in ro.outputs)
return ""
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_text2text(run_level):
"""Test Bagel text2text produces correct text output."""
config_path = _resolve_deploy_config(STAGE_CONFIG, run_level)
with OmniRunner(
MODEL_NAME,
stage_configs_path=config_path,
) as runner:
omni = runner.omni
prompt = "<|im_start|>user\nWhere is the capital of France?<|im_end|>\n<|im_start|>assistant\n"
params_list = omni.default_sampling_params_list
omni_outputs = list(
omni.generate(
prompts=[{"prompt": prompt, "modalities": ["text"]}],
sampling_params_list=params_list,
)
)
assert len(omni_outputs) > 0, "No outputs returned"
text = _extract_text(omni_outputs)
assert len(text) > 0, "Generated text is empty"
if run_level == "advanced_model":
assert text == REFERENCE_TEXT_TEXT2TEXT, (
f"Text mismatch: expected {REFERENCE_TEXT_TEXT2TEXT!r}, got {text!r}"
)
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_bagel_img2text(run_level):
"""Test Bagel img2text produces correct text output."""
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
config_path = _resolve_deploy_config(STAGE_CONFIG, run_level)
with OmniRunner(
MODEL_NAME,
stage_configs_path=config_path,
stage_init_timeout=300,
) as runner:
omni = runner.omni
prompt = "<|im_start|>user\n<|image_pad|>\nPlease describe this image<|im_end|>\n<|im_start|>assistant\n"
params_list = omni.default_sampling_params_list
omni_outputs = list(
omni.generate(
prompts=[
{
"prompt": prompt,
"multi_modal_data": {"image": input_image},
"modalities": ["text"],
}
],
sampling_params_list=params_list,
)
)
assert len(omni_outputs) > 0, "No outputs returned"
text = _extract_text(omni_outputs)
assert len(text) > 0, "Generated text is empty"
if run_level == "advanced_model":
assert text == REFERENCE_TEXT_IMG2TEXT, f"Text mismatch: expected {REFERENCE_TEXT_IMG2TEXT!r}, got {text!r}"
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
System test for cache-dit backend.
This test verifies that cache-dit acceleration works correctly with diffusion models.
It uses minimal settings to keep test time short for CI.
"""
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# Use random weights model for testing
models = ["riverclouds/qwen_image_random"]
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.cache
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_cache_dit(model_name: str):
"""Test cache-dit backend with diffusion model."""
# Configure cache-dit with minimal settings for fast testing
cache_config = {
"Fn_compute_blocks": 1,
"Bn_compute_blocks": 0,
"max_warmup_steps": 2, # Minimal warmup for fast test
"residual_diff_threshold": 0.24,
"max_continuous_cached_steps": 3,
}
with OmniRunner(
model_name,
cache_backend="cache_dit",
cache_config=cache_config,
) as runner:
# Use minimal settings for fast testing
height = 256
width = 256
num_inference_steps = 4 # Minimal steps for fast test
outputs = runner.omni.generate(
"a photo of a cat sitting on a laptop keyboard",
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
),
)
# Extract images from request_output['images']
first_output = outputs[0]
assert first_output.final_output_type == "image"
if not hasattr(first_output, "request_output") or not first_output.request_output:
raise ValueError("No request_output found in OmniRequestOutput")
req_out = first_output.request_output
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
raise ValueError("Invalid request_output structure or missing 'images' key")
images = req_out.images
# Verify generation succeeded
assert images is not None
assert len(images) == 1
# Check image size
assert images[0].width == width
assert images[0].height == height
@@ -6,8 +6,9 @@ Offline E2E smoke test for CosyVoice3 zero-shot reference inference.
This test uses the official upstream zero-shot prompt text/audio pair and
verifies a stable reference recipe:
- config-derived top_p/top_k and token-length ratios
- model EOS token as the stop token
- model EOS token as the stop token for stage-0 (talker) sampling
- a conservative repetition penalty to avoid degenerate loops
- stage-1 (code2wav) completion reports a normal terminal ``finish_reason``
"""
from __future__ import annotations
@@ -23,18 +24,19 @@ from huggingface_hub import snapshot_download
from vllm.sampling_params import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.media import test_asset_path
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
from vllm_omni.outputs import OmniRequestOutput
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
MODEL_DIR_ENV = "VLLM_OMNI_COSYVOICE3_MODEL_DIR"
# Vendored under tests/assets/cosyvoice3/ so the test does not depend on
# raw.githubusercontent.com being reachable from CI runners.
REFERENCE_PROMPT_WAV_PATH = test_asset_path("cosyvoice3/zero_shot_prompt.wav")
REFERENCE_PROMPT_WAV_PATH = get_asset_path("cosyvoice3/zero_shot_prompt.wav")
REFERENCE_PROMPT_TEXT = "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。"
REFERENCE_SYNTH_TEXT = (
"CosyVoice is undergoing a comprehensive upgrade, providing more accurate, "
@@ -44,12 +46,6 @@ REFERENCE_STAGE0_TEMPERATURE = 1.0
REFERENCE_STAGE0_REPETITION_PENALTY = 2.0
ASYNC_CHUNK_MODES = [
pytest.param(False, id="sync"),
pytest.param(True, id="async_chunk"),
]
@functools.lru_cache(maxsize=1)
def _load_reference_prompt_wav() -> tuple[np.ndarray, int]:
audio, sr = sf.read(str(REFERENCE_PROMPT_WAV_PATH), dtype="float32", always_2d=False)
@@ -120,18 +116,6 @@ def _concat_audio(audio_val) -> np.ndarray:
return audio_np.reshape(-1)
def _get_stage_engine_outputs(omni_runner: OmniRunner, stage_id: int):
stage_list = getattr(omni_runner.omni, "stage_list", None)
if stage_list is not None:
return getattr(stage_list[stage_id], "engine_outputs", None) or []
stage_clients = getattr(getattr(omni_runner.omni, "engine", None), "stage_clients", None)
if stage_clients is not None:
return getattr(stage_clients[stage_id], "engine_outputs", None) or []
raise AttributeError("Unable to locate stage outputs on Omni runner")
def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[str, object]]:
return [
{
@@ -143,57 +127,67 @@ def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[s
]
@pytest.mark.core_model
@pytest.mark.omni
# (model, stage_configs_path, extra_omni_kwargs) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
_cosy_deployment = get_deploy_config_path("cosyvoice3.yaml")
_cosy_model_path = str(_resolve_model_dir())
_OMNI_RUNNER_PARAMS = [
pytest.param(
(_cosy_model_path, _cosy_deployment, {"async_chunk": False}),
id="sync",
),
pytest.param(
(_cosy_model_path, _cosy_deployment, {"async_chunk": True}),
id="async_chunk",
),
]
pytestmark = [
pytest.mark.full_model,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
]
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("async_chunk", ASYNC_CHUNK_MODES)
def test_cosyvoice3_offline_reference_zero_shot(async_chunk: bool) -> None:
def test_cosyvoice3_offline_reference_zero_shot(omni_runner: OmniRunner) -> None:
"""CosyVoice3 zero-shot reference inference should stop cleanly and produce sane audio."""
prompt_audio, prompt_sr = _load_reference_prompt_wav()
model_dir = _resolve_model_dir()
expected_stop_token = int(CosyVoice3Config().llm["eos_token_id"])
with OmniRunner(
str(model_dir),
seed=42,
stage_configs_path=get_deploy_config_path("cosyvoice3.yaml"),
async_chunk=async_chunk,
stage_init_timeout=300,
) as omni_runner:
sampling_params_list = omni_runner.get_default_sampling_params_list()
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
sampling_params_list = omni_runner.get_default_sampling_params_list()
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list)
outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list)
assert outputs, "No outputs returned"
audio_mm = outputs[0].multimodal_output
assert "audio" in audio_mm, "No audio output found"
assert outputs, "No outputs returned"
audio_mm = outputs[0].multimodal_output
assert "audio" in audio_mm, "No audio output found"
audio = _concat_audio(audio_mm["audio"])
assert audio.size > 0, "Generated audio is empty"
audio = _concat_audio(audio_mm["audio"])
assert audio.size > 0, "Generated audio is empty"
sr_val = audio_mm.get("sr", 24000)
if isinstance(sr_val, list) and sr_val:
sr_val = sr_val[-1]
if hasattr(sr_val, "item"):
sr_val = sr_val.item()
sr = int(sr_val)
assert sr == 24000, f"Unexpected sample_rate={sr}"
sr_val = audio_mm.get("sr", 24000)
if isinstance(sr_val, list) and sr_val:
sr_val = sr_val[-1]
if hasattr(sr_val, "item"):
sr_val = sr_val.item()
sr = int(sr_val)
assert sr == 24000, f"Unexpected sample_rate={sr}"
duration_s = audio.size / sr
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
duration_s = audio.size / sr
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
if stage0_outputs:
completion = stage0_outputs[0].outputs[0]
finish_reason = getattr(completion, "finish_reason", None)
stop_reason = getattr(completion, "stop_reason", None)
num_tokens = len(getattr(completion, "token_ids", []) or [])
assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
assert int(stop_reason) == expected_stop_token, (
f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
)
assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
else:
assert async_chunk, "Stage-0 produced no engine outputs"
# Code2wav is ``final_output`` (stage 1); ``Omni.generate`` yields its ``request_output`` / completions.
pipeline_out = OmniRequestOutput.unwrap_result(outputs[0])
assert pipeline_out.stage_id == 1, f"expected final stage 1, got {pipeline_out.stage_id}"
assert pipeline_out.final_output_type == "audio", (
f"expected audio final_output_type, got {pipeline_out.final_output_type!r}"
)
ro = pipeline_out.request_output
assert ro is not None, "stage 1 should include request_output"
completions = getattr(ro, "outputs", None) or []
assert completions, "stage 1 request_output should include at least one completion"
completion = completions[0]
finish_reason = getattr(completion, "finish_reason", None)
assert finish_reason == "stop", f"Stage-1 finish_reason={finish_reason}, expected 'stop'"
token_ids = getattr(completion, "token_ids", None) or []
if token_ids:
assert 1 <= len(token_ids) <= 4096, f"Stage-1 num_tokens={len(token_ids)}, unexpected range"
@@ -1,122 +0,0 @@
import gc
import numpy as np
import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
AUDIO_MODEL = {
"stabilityai/stable-audio-open-1.0": {"cuda": 100, "rocm": None},
}
IMAGE_VIDEO_MODELS = {
"riverclouds/qwen_image_random": {"cuda": 2500, "rocm": 2100},
}
MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS}
AUDIO_MODEL_PARAMS = {
"runner_params": {},
"sampler_params": {},
}
IMAGE_VIDEO_MODELS_PARAMS = {
"runner_params": {},
"sampler_params": {
"height": 256,
"width": 256,
},
}
def inference(model_name: str, offload: bool = True):
gc.collect()
current_omni_platform.empty_cache()
device_index = current_omni_platform.current_device()
current_omni_platform.reset_peak_memory_stats()
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
if model_name in AUDIO_MODEL:
params = AUDIO_MODEL_PARAMS
else:
params = IMAGE_VIDEO_MODELS_PARAMS
with OmniRunner(
model_name,
# TODO: we might want to add overlapped feature e2e tests
# cache_backend="cache_dit",
enable_cpu_offload=offload,
**params["runner_params"],
) as runner:
current_omni_platform.reset_peak_memory_stats()
monitor.start()
output = runner.omni.generate(
"a photo of a cat sitting on a laptop keyboard",
OmniDiffusionSamplingParams(
num_inference_steps=9,
guidance_scale=0.0,
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
**params["sampler_params"],
),
)
peak = monitor.peak_used_mb
monitor.stop()
gc.collect()
current_omni_platform.empty_cache()
return peak, output
def check_audio_determinism(audio1, audio2, atol=1e-2):
device = current_omni_platform.device_type
if isinstance(audio1, np.ndarray):
audio1 = torch.from_numpy(audio1).to(device)
if isinstance(audio2, np.ndarray):
audio2 = torch.from_numpy(audio2).to(device)
if not torch.allclose(audio1, audio2, atol=atol):
diff = torch.abs(audio1 - audio2)
print(f"Max difference: {diff.max().item()}")
print(f"Mean difference: {diff.mean().item()}")
raise AssertionError(f"Audio outputs differ beyond tolerance atol={atol}")
return True
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
@pytest.mark.parametrize("model_name", list(MODELS.keys()))
def test_cpu_offload_diffusion_model(model_name: str):
try:
offload_peak_memory, output_offload = inference(model_name, offload=True)
cleanup_dist_env_and_memory()
no_offload_peak_memory, output_no_offload = inference(model_name, offload=False)
except Exception:
pytest.fail("Inference failed")
print(f"Offload peak memory: {offload_peak_memory} MB")
print(f"No offload peak memory: {no_offload_peak_memory} MB")
if model_name == "stabilityai/stable-audio-open-1.0":
audio_offload = output_offload[0].request_output.multimodal_output.get("audio")
audio_no_offload = output_no_offload[0].request_output.multimodal_output.get("audio")
check_audio_determinism(audio_offload, audio_no_offload, atol=1e-2)
# Set platform-specific VRAM saving thresholds to account
# for varying runtime memory overhead and fragmentation between CUDA and ROCm.
is_rocm = torch.version.hip is not None
platform = "rocm" if is_rocm else "cuda"
threshold = MODELS[model_name][platform]
if threshold is None:
pytest.skip(f"Threshold not defined for {platform} on {model_name}")
assert offload_peak_memory + threshold < no_offload_peak_memory, (
f"Offload peak memory {offload_peak_memory} MB should be less than "
f"no offload peak memory {no_offload_peak_memory} MB by {threshold} MB"
)
@@ -1,118 +0,0 @@
import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.e2e.offline_inference.test_diffusion_cpu_offload import check_audio_determinism
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
AUDIO_MODEL = {
"stabilityai/stable-audio-open-1.0": {"cuda": 1500, "rocm": 1500},
}
IMAGE_VIDEO_MODELS = {
"riverclouds/qwen_image_random": {"cuda": 4500, "rocm": None},
# "Wan-AI/Wan2.2-T2V-A14B-Diffusers": {"cuda": 45000, "rocm": None},
}
MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS}
AUDIO_MODEL_PARAMS = {
"runner_params": {},
"sampler_params": {},
}
IMAGE_VIDEO_MODELS_PARAMS = {
"runner_params": {"boundary_ratio": 0.875, "flow_shift": 5.0},
"sampler_params": {"height": 480, "width": 640, "num_frames": 5},
}
def run_inference(
model_name: str,
layerwise_offload: bool = False,
num_inference_steps: int = 3,
) -> float:
current_omni_platform.empty_cache()
device_index = current_omni_platform.current_device()
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
monitor.start()
if model_name in AUDIO_MODEL:
params = AUDIO_MODEL_PARAMS
else:
params = IMAGE_VIDEO_MODELS_PARAMS
with OmniRunner(
model_name,
enable_layerwise_offload=layerwise_offload,
# TODO: we might want to add overlapped feature e2e tests
# cache_backend="cache_dit",
**params["runner_params"],
) as runner:
current_omni_platform.reset_peak_memory_stats()
# Refer to tests/e2e/offline_inference/test_t2v_model.py
# Use minimal settings for testing
output = runner.omni.generate(
"A cat sitting on a table",
OmniDiffusionSamplingParams(
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
guidance_scale=1.0,
num_inference_steps=num_inference_steps,
**params["sampler_params"],
),
)
peak = monitor.peak_used_mb
monitor.stop()
return peak, output
@pytest.mark.parametrize("model_name", list(MODELS.keys()))
def test_layerwise_offload_diffusion_model(model_name: str):
"""Test that layerwise offloading reduces GPU memory usage.
This test verifies that layerwise offloading significantly reduces peak
GPU memory usage compared to loading the entire model on GPU. The layerwise
offloader keeps only a single transformer block on GPU at a time, with
prefetching for compute-memory overlap.
"""
try:
# Run without layerwise offloading (baseline)
no_offload_peak_memory, output_no_offload = run_inference(model_name, layerwise_offload=False)
cleanup_dist_env_and_memory()
# Run with layerwise offloading (1 layer on device)
layerwise_offload_peak_memory, output_offload = run_inference(model_name, layerwise_offload=True)
cleanup_dist_env_and_memory()
except Exception:
pytest.fail("Inference failed")
print(f"Layerwise offload peak memory (1 GPU layer): {layerwise_offload_peak_memory} MB")
print(f"No offload peak memory: {no_offload_peak_memory} MB")
if model_name == "stabilityai/stable-audio-open-1.0":
audio_offload = output_offload[0].request_output.multimodal_output.get("audio")
audio_no_offload = output_no_offload[0].request_output.multimodal_output.get("audio")
# Match the sibling cpu-offload test's tolerance: layerwise offload moves
# blocks across the PCIe bus on a side stream, which can perturb cuBLAS
# algorithm selection and produce ~ULP-level drift larger than 1e-3.
check_audio_determinism(audio_offload, audio_no_offload, atol=1e-2)
is_rocm = torch.version.hip is not None
platform = "rocm" if is_rocm else "cuda"
expected_saved_memory = MODELS[model_name][platform]
if expected_saved_memory is None:
pytest.skip(f"Threshold not defined for {platform} on {model_name}")
# Verify that layerwise offloading significantly reduces memory usage
# Passes only if the actual savings exceeds the expected savings
assert layerwise_offload_peak_memory + expected_saved_memory < no_offload_peak_memory, (
f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB "
f"should be significantly less than no offload peak memory {no_offload_peak_memory} MB"
)
@@ -1,136 +0,0 @@
import json
import os
import sys
from pathlib import Path
import pytest
import torch
from safetensors.torch import save_file
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
# This test is specific to Z-Image LoRA behavior. Keep it focused on a single
# model to reduce runtime and avoid extra downloads.
models = ["Tongyi-MAI/Z-Image-Turbo"]
@pytest.mark.parametrize("model_name", models)
def test_diffusion_model(model_name: str, tmp_path: Path):
def _extract_images(outputs: list[OmniRequestOutput]):
if not outputs:
raise ValueError("Empty outputs from Omni.generate()")
first_output = outputs[0]
assert first_output.final_output_type == "image"
if not hasattr(first_output, "request_output") or not first_output.request_output:
raise ValueError("No request_output found in OmniRequestOutput")
req_out = first_output.request_output
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
raise ValueError("Invalid request_output structure or missing 'images' key")
return req_out.images
def _write_zimage_lora(adapter_dir: Path) -> str:
adapter_dir.mkdir(parents=True, exist_ok=True)
# Z-Image transformer uses dim=3840 by default (see ZImageTransformer2DModel).
dim = 3840
module_name = "transformer.layers.0.attention.to_qkv"
rank = 1
lora_a = torch.zeros((rank, dim), dtype=torch.float32)
lora_a[0, 0] = 1.0
# QKVParallelLinear packs (Q, K, V). With tp=1 and n_kv_heads==n_heads in Z-Image,
# each slice is `dim`, so total out dim is `3 * dim`.
lora_b = torch.zeros((3 * dim, rank), dtype=torch.float32)
# Apply a visible delta to the Q slice only to keep the perturbation bounded.
lora_b[:dim, 0] = 0.1
save_file(
{
f"base_model.model.{module_name}.lora_A.weight": lora_a,
f"base_model.model.{module_name}.lora_B.weight": lora_b,
},
str(adapter_dir / "adapter_model.safetensors"),
)
(adapter_dir / "adapter_config.json").write_text(
json.dumps(
{
"r": rank,
"lora_alpha": rank,
"target_modules": [module_name],
}
),
encoding="utf-8",
)
return str(adapter_dir)
with OmniRunner(model_name) as runner:
m = runner.omni
# high resolution may cause OOM on L4
height = 256
width = 256
prompt = "a photo of a cat sitting on a laptop keyboard"
outputs = m.generate(
prompt,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=1,
),
)
images = _extract_images(outputs)
assert len(images) == 1
# check image size
assert images[0].width == width
assert images[0].height == height
# Real LoRA E2E: generate again with a real on-disk PEFT adapter and
# verify that output changes.
if model_name == "Tongyi-MAI/Z-Image-Turbo":
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
lora_dir = _write_zimage_lora(tmp_path / "zimage_lora")
lora_request = LoRARequest(
lora_name="test",
lora_int_id=stable_lora_int_id(lora_dir),
lora_path=lora_dir,
)
outputs_lora = m.generate(
prompt,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=1,
lora_request=lora_request,
lora_scale=2.0,
),
)
images_lora = _extract_images(outputs_lora)
assert len(images_lora) == 1
assert images_lora[0].width == width
assert images_lora[0].height == height
import numpy as np
diff = np.abs(np.array(images[0], dtype=np.int16) - np.array(images_lora[0], dtype=np.int16)).mean()
assert diff > 0.0
@@ -4,7 +4,7 @@
E2E offline smoke tests for Dynin-Omni.
- model: "snu-aidas/Dynin-Omni"
- stage config: tests/e2e/stage_configs/dynin_omni_ci.yaml
- stage config: vllm_omni/deploy/dynin_omni_ci.yaml
"""
from __future__ import annotations
@@ -19,13 +19,14 @@ import torch
from transformers import AutoTokenizer
from tests.helpers.mark import hardware_test
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
_REPO_ROOT = Path(__file__).resolve().parents[3]
_DEFAULT_DYNIN_CONFIG_PATH: Path | None = None
_DEFAULT_STAGE_CONFIG_PATH = _REPO_ROOT / "tests" / "e2e" / "stage_configs" / "dynin_omni_ci.yaml"
_DEFAULT_STAGE_CONFIG_PATH = Path(get_deploy_config_path("dynin_omni_ci.yaml"))
models = ["snu-aidas/Dynin-Omni"]
stage_configs = [str(_DEFAULT_STAGE_CONFIG_PATH)]
@@ -34,7 +35,7 @@ test_params = [(model, stage_config) for model in models for stage_config in sta
DYNIN_CONFIG_PATH = str(_DEFAULT_DYNIN_CONFIG_PATH) if _DEFAULT_DYNIN_CONFIG_PATH is not None else None
pytestmark = [
pytest.mark.core_model,
pytest.mark.full_model,
pytest.mark.omni,
pytest.mark.parametrize("omni_runner", test_params, indirect=True),
]
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for ErnieImage text-to-image generation.
Equivalent to running:
vllm serve baidu/ERNIE-Image --omni
"""
import pytest
from PIL import Image
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "baidu/ERNIE-Image"
@pytest.mark.core_model
@pytest.mark.diffusion
def test_ernie_image_text_to_image():
with OmniRunner(
MODEL,
parallel_config=DiffusionParallelConfig(
tensor_parallel_size=2,
),
enable_cpu_offload=True,
) as runner:
omni_outputs = list(
runner.omni.generate(
prompts=["A photo of a cat sitting on a laptop"],
sampling_params_list=OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
guidance_scale=4.0,
seed=42,
),
)
)
assert len(omni_outputs) > 0
output = omni_outputs[0]
images = None
if output.images:
images = output.images
elif hasattr(output, "request_output") and output.request_output:
for stage_out in output.request_output:
if hasattr(stage_out, "images") and stage_out.images:
images = stage_out.images
break
assert images is not None
assert len(images) > 0
assert isinstance(images[0], Image.Image)
assert images[0].size == (512, 512)
@@ -0,0 +1,52 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for ErnieImage text-to-image generation.
Equivalent to running:
vllm serve baidu/ERNIE-Image --omni
"""
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunnerHandler
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "baidu/ERNIE-Image"
# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize
_OMNI_RUNNER_PARAM = (
MODEL,
None,
{
"parallel_config": DiffusionParallelConfig(
tensor_parallel_size=2,
),
"enable_cpu_offload": True,
},
)
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
@hardware_test(res={"cuda": "L4"}, num_cards=2)
def test_ernie_image_text_to_image(omni_runner_handler: OmniRunnerHandler) -> None:
request_config = {
"model": omni_runner_handler.runner.model_name,
"prompt": "A photo of a cat sitting on a laptop",
"sampling_params": OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
guidance_scale=4.0,
seed=42,
),
}
omni_runner_handler.send_diffusion_request(request_config)
@@ -1,247 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
System test for Expert Parallel (EP).
Tests verify that EP inference produces correct outputs compared to baseline.
"""
import gc
import os
import time
from typing import NamedTuple
import numpy as np
import pytest
import torch
import torch.distributed as dist
from PIL import Image
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
# Test configuration
MODELS = ["tencent/HunyuanImage-3.0"]
PROMPT = "A brown and white dog is running on the grass"
DEFAULT_HEIGHT = 1024
DEFAULT_WIDTH = 1024
DEFAULT_SEED = 1234
DEFAULT_STEPS = 50
DEFAULT_GUIDANCE_SCALE = 0.0
COS_SIM_MEAN_THRESHOLD = 1e-2
COS_SIM_MAX_THRESHOLD = 1e-2
MSE_THRESHOLD = 1e-2
DIFF_MEAN_THRESHOLD = 5e-2
DIFF_MAX_THRESHOLD = 1
TOTAL_CARD_NUM = 8
class InferenceResult(NamedTuple):
"""Result of an inference run."""
images: list[Image.Image]
elapsed_ms: float
def _cleanup_distributed():
"""Clean up distributed environment and GPU resources."""
if dist.is_initialized():
dist.destroy_process_group()
for key in ["MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"]:
os.environ.pop(key, None)
gc.collect()
if current_omni_platform.is_available():
current_omni_platform.empty_cache()
current_omni_platform.synchronize()
time.sleep(5)
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float, float, float, float]:
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
ta = torch.from_numpy(np.asarray(a.convert("RGB"), dtype=np.float32) / 255.0)
tb = torch.from_numpy(np.asarray(b.convert("RGB"), dtype=np.float32) / 255.0)
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
abs_diff = torch.abs(ta - tb)
vec_a = ta.reshape(-1)
vec_b = tb.reshape(-1)
cos_sim = torch.nn.functional.cosine_similarity(vec_a, vec_b, dim=0, eps=1e-8)
mse = torch.nn.functional.mse_loss(ta, tb)
return abs_diff.mean().item(), abs_diff.max().item(), cos_sim.mean().item(), cos_sim.max().item(), mse.item()
def _run_inference(
model_name: str,
tensor_parallel_size: int,
enable_expert_parallel: bool,
guidance_scale: float = DEFAULT_GUIDANCE_SCALE,
height: int = DEFAULT_HEIGHT,
width: int = DEFAULT_WIDTH,
seed: int = DEFAULT_SEED,
warmup: bool = True,
) -> InferenceResult:
"""Run inference with specified configuration.
Args:
warmup: If True, run one warmup iteration before the timed run.
"""
parallel_config = DiffusionParallelConfig(
tensor_parallel_size=tensor_parallel_size,
enable_expert_parallel=enable_expert_parallel,
)
try:
with OmniRunner(model_name, parallel_config=parallel_config) as runner:
omni = runner.omni
# Warmup run (not timed)
if warmup:
_ = omni.generate(
PROMPT,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=DEFAULT_STEPS,
guidance_scale=guidance_scale,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
# Timed run
start = time.time()
outputs = omni.generate(
PROMPT,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=DEFAULT_STEPS,
guidance_scale=guidance_scale,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
elapsed_ms = (time.time() - start) * 1000
return InferenceResult(
images=outputs[0].images,
elapsed_ms=elapsed_ms,
)
finally:
_cleanup_distributed()
# EP configurations: (tensor_parallel_size, enable_ep, height, width, is_perf_test)
EP_TEST_CONFIG = [
(TOTAL_CARD_NUM, True, DEFAULT_HEIGHT, DEFAULT_WIDTH, True),
]
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": TOTAL_CARD_NUM})
@pytest.mark.parametrize("model_name", MODELS)
def test_ep(model_name):
baseline_cache: dict[str, InferenceResult] = {}
results: list[dict] = []
# baseline_key = PROMPT + "_" + str(DEFAULT_HEIGHT) + "*" + str(DEFAULT_WIDTH) + "_" + str(DEFAULT_SEED) + "_" + str(DEFAULT_STEPS)
# baseline_cache[baseline_key] = _run_inference(
# model_name=model_name,
# tensor_parallel_size=TOTAL_CARD_NUM,
# enable_expert_parallel=False,
# warmup=True,
# )
device_count = current_omni_platform.get_device_count()
print("\n" + "=" * 90)
print(f"Sequence Parallel Test - Model: {model_name}")
print(f"Available GPUs: {device_count}")
print("=" * 90)
for tensor_parallel_size, enable_ep, height, width, is_perf_test in EP_TEST_CONFIG:
cache_key = PROMPT + "_" + str(height) + "*" + str(width) + "_" + str(DEFAULT_SEED) + "_" + str(DEFAULT_STEPS)
if cache_key not in baseline_cache:
print(
f"\n--- Running baseline{{ prompt: {PROMPT}, resolution: {height}x{width}, seed: {DEFAULT_SEED}, steps: {DEFAULT_STEPS} }}---"
)
baseline = _run_inference(
model_name=model_name,
tensor_parallel_size=TOTAL_CARD_NUM,
enable_expert_parallel=False,
warmup=True,
)
assert len(baseline.images) == 1
baseline_cache[cache_key] = baseline
print(f"[baseline] {height}x{width}: {baseline.elapsed_ms:.0f}ms")
else:
baseline = baseline_cache[cache_key]
print(f"\n--- Running test: {{ enable_ep: {enable_ep} }} ---")
infer_result = _run_inference(
model_name=model_name,
tensor_parallel_size=tensor_parallel_size,
enable_expert_parallel=enable_ep,
warmup=is_perf_test,
)
assert len(infer_result.images) == 1
mean_diff, max_diff, cos_sim_mean, cos_sim_max, mse = _diff_metrics(baseline.images[0], infer_result.images[0])
print(f"{cos_sim_mean}, {cos_sim_max}, {mse}")
# Build result entry
result = {
"prompt": PROMPT,
"enable_ep": enable_ep,
"height": height,
"width": width,
"baseline_ms": baseline.elapsed_ms,
"ep_ms": infer_result.elapsed_ms,
"mean_diff": mean_diff,
"max_diff": max_diff,
"cos_sim_mean": cos_sim_mean,
"cos_sim_max": cos_sim_max,
"mse": mse,
"is_perf_test": is_perf_test,
}
results.append(result)
# Output based on test type
if is_perf_test:
speedup = baseline.elapsed_ms / infer_result.elapsed_ms if infer_result.elapsed_ms > 0 else 0
result["speedup"] = speedup
print(
f"[enable_ep: {enable_ep}] {tensor_parallel_size} GPUs | "
f"baseline: {baseline.elapsed_ms:.0f}ms, ep: {infer_result.elapsed_ms:.0f}ms, "
f"speedup: {speedup:.2f}x"
)
else:
print(
f"[enable_ep: {enable_ep}] {tensor_parallel_size} GPUs | ep: {infer_result.elapsed_ms:.0f}ms (correctness only)"
)
print(
f"[enable_ep: {enable_ep}] diff: [mean={mean_diff:.6e}, max={max_diff:.6e}], cos_sim: [mean={cos_sim_mean:.6e}, max={cos_sim_max:.6e}], mse: {mse:.6e}"
)
assert mean_diff <= DIFF_MEAN_THRESHOLD and max_diff <= DIFF_MAX_THRESHOLD, (
f"[enable_ep: {enable_ep}] output differs from baseline: mean={mean_diff:.6e}, max={max_diff:.6e}"
)
print("\n" + "=" * 90)
print("SUMMARY")
print("=" * 90)
print(f"{'Mode':<15} {'GPUs':<6} {'Size':<10} {'Baseline':<12} {'EP':<12} {'Speedup':<10} {'Status'}")
print("-" * 90)
for r in results:
speedup_str = f"{r['speedup']:.2f}x" if r.get("speedup") else "N/A"
baseline_str = f"{r['baseline_ms']:.0f}ms" if r["is_perf_test"] else "N/A"
status = "PASS" if r["mean_diff"] <= DIFF_MEAN_THRESHOLD else "FAIL"
print(
f"{r['prompt']:<15.10} {r['enable_ep']:<6} {r['height']}x{r['width']:<5} "
f"{baseline_str:<12} {r['ep_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
)
print("=" * 90)
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Flux1 Schnell."""
import pytest
from PIL import Image
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "black-forest-labs/FLUX.1-schnell"
@pytest.mark.core_model
@pytest.mark.diffusion
def test_flux_schnell_text_to_image():
"""Test FLUX.1-schnell text-to-image generation."""
omni = Omni(model=MODEL)
omni_outputs = list(
omni.generate(
prompts=["A photo of a cat sitting on a laptop"],
sampling_params_list=OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
seed=42,
),
)
)
assert len(omni_outputs) > 0
images = omni_outputs[0].images
assert len(images) == 1
assert isinstance(images[0], Image.Image)
assert images[0].size == (512, 512)
@@ -5,11 +5,16 @@
These tests require:
- A CUDA GPU
- The quantized model checkpoint (vllm-project-org/FLUX.1-dev-AutoRound-w4a16)
``test_flux_autoround_w4a16_generates_image`` uses the shared ``omni_runner`` fixture
(quantized model). ``test_flux_autoround_w4a16_memory_savings`` must load two different
checkpoints in sequence, so it keeps explicit ``OmniRunner`` context managers.
"""
import gc
import os as _os
import numpy as np
import pytest
import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
@@ -21,6 +26,10 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
pytestmark = pytest.mark.skip(
reason="Current Hugging Face account is not yet authorized to access this model.",
)
QUANTIZED_MODEL = "vllm-project-org/FLUX.1-dev-AutoRound-w4a16"
BASELINE_MODEL = "black-forest-labs/FLUX.1-dev"
@@ -32,6 +41,27 @@ HEIGHT = 256
WIDTH = 256
NUM_STEPS = 2 # minimal for smoke-test
# Indirect param: (model_id, stage_config_path, extra OmniRunner / Omni kwargs)
_OMNI_RUNNER_PARAM = (QUANTIZED_MODEL, None, {"enforce_eager": True})
def _sampling_params() -> OmniDiffusionSamplingParams:
return OmniDiffusionSamplingParams(
height=HEIGHT,
width=WIDTH,
num_inference_steps=NUM_STEPS,
guidance_scale=0.0,
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
)
def _first_request_images(outputs) -> list:
first_output = outputs[0]
assert first_output.final_output_type == "image"
req_out = first_output.request_output
assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
return req_out.images
def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
"""Load a FLUX model, generate one image, return (images, peak_memory_mb)."""
@@ -44,25 +74,12 @@ def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
with OmniRunner(model_name, enforce_eager=True, **extra_kwargs) as runner:
current_omni_platform.reset_peak_memory_stats()
outputs = runner.omni.generate(
"a photo of a cat sitting on a laptop keyboard",
OmniDiffusionSamplingParams(
height=HEIGHT,
width=WIDTH,
num_inference_steps=NUM_STEPS,
guidance_scale=0.0,
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
),
)
outputs = runner.omni.generate("a photo of a cat sitting on a laptop keyboard", _sampling_params())
peak = monitor.peak_used_mb
monitor.stop()
first_output = outputs[0]
assert first_output.final_output_type == "image"
req_out = first_output.request_output
assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
images = req_out.images
images = _first_request_images(outputs)
gc.collect()
current_omni_platform.empty_cache()
@@ -70,26 +87,25 @@ def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
return images, peak
@pytest.mark.advanced_model
@pytest.mark.full_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"})
def test_flux_autoround_w4a16_generates_image():
@pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)
def test_flux_autoround_w4a16_generates_image(omni_runner: OmniRunner):
"""Load the W4A16 quantized FLUX model and verify it produces a valid image."""
images, _ = _generate_image(QUANTIZED_MODEL)
outputs = omni_runner.omni.generate("a photo of a cat sitting on a laptop keyboard", _sampling_params())
images = _first_request_images(outputs)
assert len(images) >= 1, "Expected at least one generated image"
img = images[0]
assert img.width == WIDTH, f"Expected width {WIDTH}, got {img.width}"
assert img.height == HEIGHT, f"Expected height {HEIGHT}, got {img.height}"
# Sanity: image should not be blank (all-zero)
import numpy as np
arr = np.array(img)
assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)"
@pytest.mark.advanced_model
@pytest.mark.full_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"})
def test_flux_autoround_w4a16_memory_savings():
@@ -98,7 +114,7 @@ def test_flux_autoround_w4a16_memory_savings():
The W4A16 model should use meaningfully less memory than the
BF16/FP16 baseline since weights are 4-bit instead of 16-bit.
"""
quant_images, quant_peak = _generate_image(QUANTIZED_MODEL)
_, quant_peak = _generate_image(QUANTIZED_MODEL)
cleanup_dist_env_and_memory()
_, baseline_peak = _generate_image(BASELINE_MODEL)
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for FluxKontext model pipeline.
FluxKontext is a text-to-image and image-to-image diffusion model that supports:
- Text-to-image generation
- Image editing with text guidance
"""
from __future__ import annotations
import pytest
from vllm.assets.image import ImageAsset
from tests.helpers.runtime import OmniRunnerHandler
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "black-forest-labs/FLUX.1-Kontext-dev"
_OMNI_RUNNER_PARAM = (
MODEL,
None,
{
"parallel_config": DiffusionParallelConfig(tensor_parallel_size=2),
"enable_cpu_offload": False,
},
)
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def _sampling_512() -> OmniDiffusionSamplingParams:
return OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
seed=42,
)
def test_flux_kontext_text_to_image(omni_runner_handler: OmniRunnerHandler):
"""Test FluxKontext text-to-image generation with real model."""
request_config = {
"model": MODEL,
"prompt": "A photo of a cat sitting on a laptop",
"sampling_params": _sampling_512(),
}
omni_runner_handler.send_diffusion_request(request_config)
def test_flux_kontext_image_edit(omni_runner_handler: OmniRunnerHandler):
"""Test FluxKontext image-to-image editing with real model."""
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
request_config = {
"prompt": "Transform this image into a Vincent van Gogh style painting",
"multi_modal_data": {"img2img": input_image},
"modalities": ["img2img"],
"sampling_params": _sampling_512(),
}
omni_runner_handler.send_diffusion_request(request_config)
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Flux1 Schnell."""
import pytest
from tests.helpers.runtime import OmniRunnerHandler
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "black-forest-labs/FLUX.1-schnell"
_OMNI_RUNNER_PARAM = (MODEL, None)
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def test_flux_schnell_text_to_image(omni_runner_handler: OmniRunnerHandler):
"""Test FLUX.1-schnell text-to-image generation."""
request_config = {
"model": MODEL,
"prompt": "A photo of a cat sitting on a laptop",
"sampling_params": OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
seed=42,
),
}
omni_runner_handler.send_diffusion_request(request_config)
+79 -161
View File
@@ -4,31 +4,26 @@
"""
End-to-end test for Flux2 Klein inpainting.
Inpainting uses ``omni_runner_handler.send_diffusion_request`` with
``multi_modal_data`` containing ``image`` and ``mask_image``; see
:meth:`OmniRunnerHandler.send_diffusion_request` in ``tests.helpers.runtime``.
"""
# ruff: noqa: E402
import os
import sys
from pathlib import Path
import pytest
import torch
from PIL import Image, ImageDraw
from vllm_omni.entrypoints.omni import Omni
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import DiffusionResponse, OmniRunnerHandler
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
MODEL = "black-forest-labs/FLUX.2-klein-4B"
_OMNI_RUNNER_PARAM = (MODEL, None)
pytestmark = [pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)]
_HEIGHT = 512
_WIDTH = 512
_NUM_INFERENCE_STEPS = 4
@@ -49,50 +44,56 @@ def _create_test_inputs(color: tuple = (100, 150, 200)):
return _create_test_image(_WIDTH, _HEIGHT, color), _create_test_mask(_WIDTH, _HEIGHT)
def _extract_images_from_output(outputs: list) -> list[Image.Image]:
images = []
for req_output in outputs:
if hasattr(req_output, "images") and req_output.images:
images.extend(req_output.images)
elif hasattr(req_output, "request_output") and req_output.request_output:
stage_out = req_output.request_output
if isinstance(stage_out, OmniRequestOutput) and hasattr(stage_out, "images"):
images.extend(stage_out.images)
elif isinstance(stage_out, list):
for s in stage_out:
if hasattr(s, "images") and s.images:
images.extend(s.images)
return images
def _images_from_response(response: DiffusionResponse) -> list[Image.Image]:
if isinstance(response.images[0], list):
return [f for fr in response.images for f in fr]
return list(response.images)
def _send_inpaint_with_generator(
omni_runner_handler: OmniRunnerHandler, prompt: str, input_image, mask_image, generator: torch.Generator
) -> DiffusionResponse:
return omni_runner_handler.send_diffusion_request(
{
"model": MODEL,
"prompt": prompt,
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
"sampling_params": OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=generator,
num_outputs_per_prompt=1,
),
}
)
# Regression test for https://github.com/vllm-project/vllm-omni/issues/3097
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
def test_flux2_klein_can_accept_text_inputs():
model = Omni(model=MODEL)
outputs = model.generate(
"a cup of coffee on the table",
OmniDiffusionSamplingParams(num_inference_steps=2, seed=42),
def test_flux2_klein_can_accept_text_inputs(omni_runner_handler: OmniRunnerHandler):
omni_runner_handler.send_diffusion_request(
{
"model": MODEL,
"prompt": "a cup of coffee on the table",
"sampling_params": OmniDiffusionSamplingParams(num_inference_steps=2, seed=42),
}
)
assert len(outputs[0].images) == 1
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
def test_flux2_klein_inpaint_basic():
m = None
try:
m = Omni(model=MODEL)
input_image, mask_image = _create_test_inputs()
outputs = m.generate(
prompts=[
{
"prompt": "Fill in the masked area with a beautiful garden",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_flux2_klein_inpaint_basic(omni_runner_handler: OmniRunnerHandler):
input_image, mask_image = _create_test_inputs()
omni_runner_handler.send_diffusion_request(
{
"model": MODEL,
"prompt": "Fill in the masked area with a beautiful garden",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
"sampling_params": OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
@@ -100,128 +101,45 @@ def test_flux2_klein_inpaint_basic():
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=1,
),
)
images = _extract_images_from_output(list(outputs))
assert len(images) == 1
assert images[0].size == (_WIDTH, _HEIGHT)
finally:
if m is not None and hasattr(m, "close"):
m.close()
}
)
@pytest.mark.advanced_model
@pytest.mark.diffusion
def test_flux2_klein_inpaint_deterministic():
m = None
try:
m = Omni(model=MODEL)
input_image, mask_image = _create_test_inputs()
seed = 12345
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandler):
input_image, mask_image = _create_test_inputs()
seed = 12345
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
outputs1 = m.generate(
prompts=[
{
"prompt": "A red flower in a field",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen1,
num_outputs_per_prompt=1,
),
)
r1 = _send_inpaint_with_generator(omni_runner_handler, "A red flower in a field", input_image, mask_image, gen1)
r2 = _send_inpaint_with_generator(omni_runner_handler, "A red flower in a field", input_image, mask_image, gen2)
outputs2 = m.generate(
prompts=[
{
"prompt": "A red flower in a field",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen2,
num_outputs_per_prompt=1,
),
)
images1 = _images_from_response(r1)
images2 = _images_from_response(r2)
images1 = _extract_images_from_output(list(outputs1))
images2 = _extract_images_from_output(list(outputs2))
assert len(images1) == 1
assert len(images2) == 1
assert list(images1[0].getdata()) == list(images2[0].getdata()), (
"Same input with same seed should produce identical output. "
"This is critical for offline/online consistency."
)
finally:
if m is not None and hasattr(m, "close"):
m.close()
assert list(images1[0].getdata()) == list(images2[0].getdata()), (
"Same input with same seed should produce identical output. This is critical for offline/online consistency."
)
@pytest.mark.advanced_model
@pytest.mark.diffusion
def test_flux2_klein_inpaint_different_seeds_different_output():
m = None
try:
m = Omni(model=MODEL)
input_image, mask_image = _create_test_inputs()
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_flux2_klein_inpaint_different_seeds_different_output(omni_runner_handler: OmniRunnerHandler):
input_image, mask_image = _create_test_inputs()
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(42)
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(99999)
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(42)
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(99999)
outputs1 = m.generate(
prompts=[
{
"prompt": "A beautiful landscape",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen1,
num_outputs_per_prompt=1,
),
)
r1 = _send_inpaint_with_generator(omni_runner_handler, "A beautiful landscape", input_image, mask_image, gen1)
r2 = _send_inpaint_with_generator(omni_runner_handler, "A beautiful landscape", input_image, mask_image, gen2)
outputs2 = m.generate(
prompts=[
{
"prompt": "A beautiful landscape",
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=_HEIGHT,
width=_WIDTH,
num_inference_steps=_NUM_INFERENCE_STEPS,
guidance_scale=0.0,
generator=gen2,
num_outputs_per_prompt=1,
),
)
images1 = _images_from_response(r1)
images2 = _images_from_response(r2)
images1 = _extract_images_from_output(list(outputs1))
images2 = _extract_images_from_output(list(outputs2))
assert len(images1) == 1
assert len(images2) == 1
different_pixel_count = sum(1 for p1, p2 in zip(images1[0].getdata(), images2[0].getdata()) if p1 != p2)
assert different_pixel_count > 0, "Different seeds should produce different outputs"
finally:
if m is not None and hasattr(m, "close"):
m.close()
different_pixel_count = sum(1 for p1, p2 in zip(images1[0].getdata(), images2[0].getdata()) if p1 != p2)
assert different_pixel_count > 0, "Different seeds should produce different outputs"
@@ -1,107 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for FluxKontext model pipeline.
FluxKontext is a text-to-image and image-to-image diffusion model that supports:
- Text-to-image generation
- Image editing with text guidance
"""
import pytest
from PIL import Image
from vllm.assets.image import ImageAsset
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "black-forest-labs/FLUX.1-Kontext-dev"
@pytest.mark.core_model
@pytest.mark.diffusion
def test_flux_kontext_text_to_image():
"""Test FluxKontext text-to-image generation with real model."""
with OmniRunner(
MODEL,
parallel_config=DiffusionParallelConfig(
tensor_parallel_size=2,
),
enable_cpu_offload=False,
) as runner:
omni_outputs = list(
runner.omni.generate(
prompts=["A photo of a cat sitting on a laptop"],
sampling_params_list=OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
seed=42,
),
)
)
assert len(omni_outputs) > 0
output = omni_outputs[0]
images = None
if output.images:
images = output.images
elif hasattr(output, "request_output") and output.request_output:
for stage_out in output.request_output:
if hasattr(stage_out, "images") and stage_out.images:
images = stage_out.images
break
assert images is not None
assert len(images) > 0
assert isinstance(images[0], Image.Image)
assert images[0].size == (512, 512)
@pytest.mark.core_model
@pytest.mark.diffusion
def test_flux_kontext_image_edit():
"""Test FluxKontext image-to-image editing with real model."""
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
with OmniRunner(
MODEL,
parallel_config=DiffusionParallelConfig(
tensor_parallel_size=2,
),
enable_cpu_offload=False,
) as runner:
omni_outputs = list(
runner.omni.generate(
prompts=[
{
"prompt": "Transform this image into a Vincent van Gogh style painting",
"multi_modal_data": {"img2img": input_image},
"modalities": ["img2img"],
}
],
sampling_params_list=OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=2,
seed=42,
),
)
)
assert len(omni_outputs) > 0
output = omni_outputs[0]
images = None
if output.images:
images = output.images
elif hasattr(output, "request_output") and output.request_output:
for stage_out in output.request_output:
if hasattr(stage_out, "images") and stage_out.images:
images = stage_out.images
break
assert images is not None
assert len(images) > 0
assert isinstance(images[0], Image.Image)
assert images[0].size == (512, 512)
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Offline E2E test for InternVLA-A1 open-loop inference."""
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
import pytest
from tests.helpers.mark import hardware_test
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
REPO_ROOT = Path(__file__).resolve().parents[3]
EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "internvla_a1" / "end2end.py"
def _required_env(name: str) -> str:
value = os.getenv(name)
if not value:
pytest.skip(f"{name} is not set")
return value
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_internvla_a1_offline_open_loop(run_level: str) -> None:
if run_level != "advanced_model":
pytest.skip("InternVLA-A1 offline evaluation requires real local checkpoints and dataset.")
model_dir = _required_env("INTERNVLA_A1_MODEL_DIR")
dataset_dir = _required_env("INTERNVLA_A1_DATASET_DIR")
processor_dir = _required_env("INTERNVLA_A1_PROCESSOR_DIR")
cosmos_dir = _required_env("INTERNVLA_A1_COSMOS_DIR")
env = os.environ.copy()
env["INTERNVLA_A1_PROCESSOR_DIR"] = processor_dir
env["INTERNVLA_A1_COSMOS_DIR"] = cosmos_dir
with tempfile.TemporaryDirectory(prefix="internvla_a1_e2e_") as tmpdir:
output_dir = Path(tmpdir) / "outputs"
cmd = [
sys.executable,
str(EXAMPLE_SCRIPT),
"--model-dir",
model_dir,
"--dataset-dir",
dataset_dir,
"--output-dir",
str(output_dir),
"--num-samples",
"1",
"--num-episodes",
"1",
"--dtype",
"float32",
"--attn-implementation",
"eager",
"--skip-plots",
]
subprocess.run(cmd, check=True, env=env)
summary_path = output_dir / "summary.json"
assert summary_path.exists(), f"Missing summary file: {summary_path}"
with open(summary_path, encoding="utf-8") as f:
summary = json.load(f)
assert summary["mode"] == "registry"
assert summary["attn_implementation"] == "eager"
assert summary["dtype"] == "float32"
assert len(summary["results"]) == 1
assert "registry" in summary["eval_summaries"]
eval_summary = summary["eval_summaries"]["registry"]
assert eval_summary["num_episodes"] == 1
assert eval_summary["average_mse"] is not None
assert eval_summary["average_mae"] is not None
log_path = output_dir / "registry" / "log.json"
assert log_path.exists(), f"Missing evaluation log: {log_path}"
@@ -1,243 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
import os
import subprocess
import sys
from pathlib import Path
import numpy as np
import pytest
from PIL import Image
from tests.helpers.mark import hardware_test
REPO_ROOT = Path(__file__).resolve().parents[3]
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
I2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py"
T2V_PROMPT = (
"At sunrise, a glowing paper lantern boat drifts through a narrow canal between mossy stone walls, "
"soft fog above the water, the camera slowly gliding forward as golden reflections shimmer across "
"the ripples, cinematic, realistic, highly detailed."
)
T2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion, distorted, oversaturated, artifacts"
I2V_PROMPT = "A cinematic dolly shot of a boat drifting on calm water at sunset"
I2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion"
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
def _get_ltx2_model() -> str:
return os.environ.get("VLLM_TEST_LTX2_MODEL", "Lightricks/LTX-2")
def _md5(path: Path) -> str:
digest = hashlib.md5(usedforsecurity=False)
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _make_deterministic_test_image(path: Path) -> None:
"""Create a deterministic 256x256 test image for I2V tests."""
rng = np.random.RandomState(42)
img = Image.fromarray(rng.randint(0, 255, (256, 256, 3), dtype=np.uint8))
img.save(path)
def _run_and_check(cmd: list[str], env: dict, output_path: Path, expected_md5: str) -> None:
result = subprocess.run(cmd, cwd=REPO_ROOT, env=env, capture_output=True, text=True, check=False)
assert result.returncode == 0, (
f"Command failed (exit {result.returncode}).\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
generated_md5 = _md5(output_path)
assert generated_md5 == expected_md5, (
f"Unexpected output md5: {generated_md5} != {expected_md5}.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
# ── T2V tests ──
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=2)
def test_ltx2_t2v_cfg_parallel(tmp_path: Path):
"""T2V with CFG=4.0, cfg-parallel-size=2."""
output = tmp_path / "t2v_cfg4.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
cmd = [
sys.executable,
str(T2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--prompt",
T2V_PROMPT,
"--negative-prompt",
T2V_NEGATIVE_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"145",
"--num-inference-steps",
"6",
"--guidance-scale",
"4.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--cfg-parallel-size",
"2",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="08e606b9c522fee4b6f30cee8b77db40")
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_ltx2_t2v_no_cfg(tmp_path: Path):
"""T2V with CFG=1.0 (no classifier-free guidance)."""
output = tmp_path / "t2v_nocfg.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
cmd = [
sys.executable,
str(T2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--prompt",
T2V_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"145",
"--num-inference-steps",
"6",
"--guidance-scale",
"1.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="a83994b94b6e67c54a524e0383c45ce8")
# ── I2V tests ──
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=2)
def test_ltx2_i2v_cfg_parallel(tmp_path: Path):
"""I2V with CFG=4.0, cfg-parallel-size=2."""
test_image = tmp_path / "test_input.png"
_make_deterministic_test_image(test_image)
output = tmp_path / "i2v_cfg4.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
cmd = [
sys.executable,
str(I2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--model-class-name",
"LTX2ImageToVideoPipeline",
"--image",
str(test_image),
"--prompt",
I2V_PROMPT,
"--negative-prompt",
I2V_NEGATIVE_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"73",
"--num-inference-steps",
"6",
"--guidance-scale",
"4.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--cfg-parallel-size",
"2",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="aed7e56084b36373244d8f839b16d115")
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_ltx2_i2v_no_cfg(tmp_path: Path):
"""I2V with CFG=1.0 (no classifier-free guidance)."""
test_image = tmp_path / "test_input.png"
_make_deterministic_test_image(test_image)
output = tmp_path / "i2v_nocfg.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
cmd = [
sys.executable,
str(I2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--model-class-name",
"LTX2ImageToVideoPipeline",
"--image",
str(test_image),
"--prompt",
I2V_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"73",
"--num-inference-steps",
"6",
"--guidance-scale",
"1.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="81b21ede12753e9e14a357a6c548b666")
@@ -13,6 +13,28 @@ from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "SII-GAIR/daVinci-MagiHuman-Base-1080p"
# (model, stage_config_path, extra_omni_kwargs) — see ``omni_runner`` in tests.helpers.fixtures.runtime
_OMNI_RUNNER_PARAM = (
MODEL,
None,
{
"init_timeout": 1200,
"tensor_parallel_size": 2,
},
)
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
_SKIP_ISSUE_3236 = pytest.mark.skip(
reason="https://github.com/vllm-project/vllm-omni/issues/3236",
)
def _validate_mp4(video_bytes: bytes, min_frames: int = 10) -> None:
"""Validate that the MP4 contains meaningful video and audio tracks."""
@@ -38,17 +60,10 @@ def _validate_mp4(video_bytes: bytes, min_frames: int = 10) -> None:
container.close()
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@_SKIP_ISSUE_3236
@hardware_test(res={"cuda": "H100"}, num_cards=2)
def test_magi_human_e2e(run_level):
def test_magi_human_e2e(omni_runner: OmniRunner):
"""End-to-end test for MagiHuman generating video and audio."""
if run_level != "advanced_model":
pytest.skip("MagiHuman e2e test requires advanced_model run level with real weights.")
model_path = "SII-GAIR/daVinci-MagiHuman-Base-1080p"
prompt = (
"A young woman with long, wavy golden blonde hair and bright blue eyes, "
"wearing a fitted ivory silk blouse with a delicate lace collar, sits "
@@ -88,54 +103,49 @@ def test_magi_human_e2e(run_level):
},
)
with OmniRunner(
model_path,
init_timeout=1200,
tensor_parallel_size=2,
) as runner:
omni = runner.omni
outputs = list(
omni.generate(
prompts=[prompt],
sampling_params_list=[sampling_params],
)
omni = omni_runner.omni
outputs = list(
omni.generate(
prompts=[prompt],
sampling_params_list=[sampling_params],
)
)
assert len(outputs) > 0, "No outputs returned"
first = outputs[0]
assert len(outputs) > 0, "No outputs returned"
first = outputs[0]
assert hasattr(first, "images") and first.images, "No video frames in output"
video_frames = first.images[0]
assert isinstance(video_frames, np.ndarray), f"Expected numpy array, got {type(video_frames)}"
assert video_frames.ndim == 4, f"Expected 4D array (T,H,W,3), got shape {video_frames.shape}"
assert hasattr(first, "images") and first.images, "No video frames in output"
video_frames = first.images[0]
assert isinstance(video_frames, np.ndarray), f"Expected numpy array, got {type(video_frames)}"
assert video_frames.ndim == 4, f"Expected 4D array (T,H,W,3), got shape {video_frames.shape}"
mm = first.multimodal_output
assert mm, "multimodal_output is empty or missing"
mm = first.multimodal_output
assert mm, "multimodal_output is empty or missing"
audio_waveform = mm.get("audio")
assert audio_waveform is not None, "No audio waveform in multimodal_output"
audio_waveform = mm.get("audio")
assert audio_waveform is not None, "No audio waveform in multimodal_output"
audio_sample_rate = mm.get("audio_sample_rate")
assert audio_sample_rate is not None, (
"audio_sample_rate not found in multimodal_output; model post-process must propagate it"
)
assert isinstance(audio_sample_rate, (int, float)), (
f"audio_sample_rate should be numeric, got {type(audio_sample_rate)}"
)
assert int(audio_sample_rate) > 0, f"audio_sample_rate must be positive, got {audio_sample_rate}"
audio_sample_rate = mm.get("audio_sample_rate")
assert audio_sample_rate is not None, (
"audio_sample_rate not found in multimodal_output; model post-process must propagate it"
)
assert isinstance(audio_sample_rate, (int, float)), (
f"audio_sample_rate should be numeric, got {type(audio_sample_rate)}"
)
assert int(audio_sample_rate) > 0, f"audio_sample_rate must be positive, got {audio_sample_rate}"
fps = mm.get("fps")
assert fps is not None, "fps not found in multimodal_output; model post-process must propagate it"
assert isinstance(fps, (int, float)), f"fps should be numeric, got {type(fps)}"
assert int(fps) > 0, f"fps must be positive, got {fps}"
fps = mm.get("fps")
assert fps is not None, "fps not found in multimodal_output; model post-process must propagate it"
assert isinstance(fps, (int, float)), f"fps should be numeric, got {type(fps)}"
assert int(fps) > 0, f"fps must be positive, got {fps}"
video_bytes = mux_video_audio_bytes(
video_frames,
audio_waveform,
fps=float(fps),
audio_sample_rate=int(audio_sample_rate),
)
assert isinstance(video_bytes, bytes), f"Expected MP4 bytes, got {type(video_bytes)}"
assert len(video_bytes) > 1000, f"MP4 too small ({len(video_bytes)} bytes)"
video_bytes = mux_video_audio_bytes(
video_frames,
audio_waveform,
fps=float(fps),
audio_sample_rate=int(audio_sample_rate),
)
assert isinstance(video_bytes, bytes), f"Expected MP4 bytes, got {type(video_bytes)}"
assert len(video_bytes) > 1000, f"MP4 too small ({len(video_bytes)} bytes)"
_validate_mp4(video_bytes)
_validate_mp4(video_bytes)
@@ -1,212 +0,0 @@
"""
End-to-end test for MammothModa2 text-to-image generation.
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
match a golden reference.
Model weights: env var ``MAMMOTHMODA2_MODEL_PATH``
(default: <repo_root>/MammothModa2-Preview)
Stage config: env var ``MAMMOTHMODA2_T2I_STAGE_CONFIG``
(default: vllm_omni/model_executor/stage_configs/mammoth_moda2.yaml)
Golden pixel file: ``tests/e2e/offline_inference/fixtures/mammoth_moda2_t2i_golden.json``
Regenerate with: ``UPDATE_GOLDEN=1 pytest tests/e2e/offline_inference/test_mammoth_moda2.py``
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
import torch
from vllm.sampling_params import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_IMAGE_TOKEN_ID = 151655 # "<|image_pad|>"
_VIDEO_TOKEN_ID = 151656 # "<|video_pad|>"
_VISION_START_TOKEN_ID = 151652 # "<|vision_start|>"
_VISION_END_TOKEN_ID = 151653 # "<|vision_end|>"
_AR_PATCH_SIZE = 16
_STAGE_CONFIGS_DIR = Path(__file__).resolve().parents[3] / "vllm_omni" / "model_executor" / "stage_configs"
MODEL_PATH = os.environ.get(
"MAMMOTHMODA2_MODEL_PATH",
str(Path(__file__).resolve().parents[3] / "MammothModa2-Preview"),
)
T2I_STAGE_CONFIG = os.environ.get(
"MAMMOTHMODA2_T2I_STAGE_CONFIG",
str(_STAGE_CONFIGS_DIR / "mammoth_moda2.yaml"),
)
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
# Covers corners, centre, and mid-edges across all 3 channels.
_PIXEL_SAMPLE_COORDS = [
(0, 0.0, 0.0),
(0, 0.5, 0.5),
(0, 1.0, 1.0),
(0, 0.25, 0.75),
(1, 0.0, 1.0),
(1, 0.5, 0.0),
(1, 0.75, 0.25),
(1, 1.0, 0.5),
(2, 0.0, 0.5),
(2, 0.5, 1.0),
(2, 0.75, 0.75),
(2, 1.0, 0.0),
]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _load_t2i_gen_config(model_dir: str) -> dict:
cfg_path = Path(model_dir) / "t2i_generation_config.json"
if not cfg_path.exists():
pytest.skip(f"t2i_generation_config.json not found at {cfg_path}")
with cfg_path.open() as f:
return json.load(f)
def _format_t2i_prompt(user_prompt: str, ar_width: int, ar_height: int) -> str:
return (
"<|im_start|>system\nYou are a helpful image generator.<|im_end|>\n"
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
"<|im_start|>assistant\n"
f"<|image start|>{ar_width}*{ar_height}<|image token|>"
)
def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
"""Sample pixel values at fixed fractional coordinates from a (C, H, W) tensor."""
t = img_tensor.float().clamp(0.0, 1.0)
if t.ndim == 4:
t = t[0] # unbatch
C, H, W = t.shape
values = []
for c, rh, rw in _PIXEL_SAMPLE_COORDS:
ri = min(int(rh * (H - 1)), H - 1)
ci = min(int(rw * (W - 1)), W - 1)
values.append(round(float(t[c, ri, ci]), 6))
return values
# ---------------------------------------------------------------------------
# End-to-end test
# ---------------------------------------------------------------------------
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"})
def test_mammothmoda2_t2i_e2e():
"""
End-to-end text-to-image generation with MammothModa2 (AR -> DiT).
Verifies:
- Omni pipeline initialises with the two-stage YAML config.
- DiT stage outputs an image tensor with the correct shape.
- A fixed set of pixel values matches a golden reference
(regenerate with ``UPDATE_GOLDEN=1``).
"""
if not Path(MODEL_PATH).exists():
pytest.skip(f"Model weights not found at {MODEL_PATH}")
if not Path(T2I_STAGE_CONFIG).exists():
pytest.skip(f"Stage config not found at {T2I_STAGE_CONFIG}")
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
eol_token_id = int(gen_cfg["eol_token_id"])
visual_start = int(gen_cfg["visual_token_start_id"])
visual_end = int(gen_cfg["visual_token_end_id"])
height, width = 256, 256 # small for CI speed
ar_height, ar_width = height // _AR_PATCH_SIZE, width // _AR_PATCH_SIZE
expected_grid_tokens = ar_height * (ar_width + 1)
prompt_text = "A cat sitting on a laptop keyboard"
formatted_prompt = _format_t2i_prompt(prompt_text, ar_width, ar_height)
with OmniRunner(MODEL_PATH, stage_configs_path=T2I_STAGE_CONFIG, trust_remote_code=True) as runner:
omni = runner.omni
# Greedy / deterministic sampling so pixel values are reproducible.
ar_sampling = SamplingParams(
temperature=0.0,
top_k=1,
max_tokens=max(1, expected_grid_tokens + 1),
detokenize=False,
)
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
outputs = list(
omni.generate(
[
{
"prompt": formatted_prompt,
"additional_information": {
"omni_task": ["t2i"],
"ar_width": [ar_width],
"ar_height": [ar_height],
"eol_token_id": [eol_token_id],
"visual_token_start_id": [visual_start],
"visual_token_end_id": [visual_end],
"image_height": [height],
"image_width": [width],
"num_inference_steps": [2],
"text_guidance_scale": [1.0],
"cfg_range": [0.0, 1.0],
"visual_ids": [
_IMAGE_TOKEN_ID,
_VIDEO_TOKEN_ID,
_VISION_START_TOKEN_ID,
_VISION_END_TOKEN_ID,
],
},
}
],
[ar_sampling, dit_sampling],
)
)
assert len(outputs) > 0, "Pipeline produced no outputs"
# Find the image tensor produced by the DiT stage and compare pixels.
found_image = False
for out in outputs:
ro_list = getattr(out, "request_output", out)
if not isinstance(ro_list, list):
ro_list = [ro_list]
for ro in ro_list:
completion_outputs = getattr(ro, "outputs", None)
if not isinstance(completion_outputs, list):
continue
for completion in completion_outputs:
mm = getattr(completion, "multimodal_output", None)
if not (isinstance(mm, dict) and "image" in mm):
continue
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
for img_tensor in img_list:
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
sampled = _sample_pixels(img_tensor)
if os.environ.get("UPDATE_GOLDEN"):
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
elif _GOLDEN_T2I_PATH.exists():
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
for i, (got, exp) in enumerate(zip(sampled, golden)):
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
found_image = True
assert found_image, "No image tensor found in pipeline output"
@@ -0,0 +1,204 @@
"""
End-to-end test for MammothModa2 text-to-image generation.
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
match a golden reference.
Model Hub repo id: ``bytedance-research/MammothModa2-Preview``.
Stage config: ``get_deploy_config_path("mammoth_moda2.yaml")`` → ``vllm_omni/deploy/mammoth_moda2.yaml``
Golden pixel file: ``tests/e2e/offline_inference/fixtures/mammoth_moda2_t2i_golden.json``
Regenerate with: ``UPDATE_GOLDEN=1 pytest tests/e2e/offline_inference/test_mammoth_moda2_expansion.py``
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
import torch
from huggingface_hub import snapshot_download
from vllm.sampling_params import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
_IMAGE_TOKEN_ID = 151655 # "<|image_pad|>"
_VIDEO_TOKEN_ID = 151656 # "<|video_pad|>"
_VISION_START_TOKEN_ID = 151652 # "<|vision_start|>"
_VISION_END_TOKEN_ID = 151653 # "<|vision_end|>"
_AR_PATCH_SIZE = 16
MODEL_PATH = "bytedance-research/MammothModa2-Preview"
T2I_STAGE_CONFIG = get_deploy_config_path("mammoth_moda2.yaml")
_OMNI_RUNNER_PARAM = (MODEL_PATH, T2I_STAGE_CONFIG)
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
# Covers corners, centre, and mid-edges across all 3 channels.
_PIXEL_SAMPLE_COORDS = [
(0, 0.0, 0.0),
(0, 0.5, 0.5),
(0, 1.0, 1.0),
(0, 0.25, 0.75),
(1, 0.0, 1.0),
(1, 0.5, 0.0),
(1, 0.75, 0.25),
(1, 1.0, 0.5),
(2, 0.0, 0.5),
(2, 0.5, 1.0),
(2, 0.75, 0.75),
(2, 1.0, 0.0),
]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _load_t2i_gen_config(repo_id: str) -> dict:
weights_dir = Path(snapshot_download(repo_id))
cfg_path = weights_dir / "t2i_generation_config.json"
if not cfg_path.exists():
pytest.skip(f"t2i_generation_config.json not found at {cfg_path}")
with cfg_path.open() as f:
return json.load(f)
def _format_t2i_prompt(user_prompt: str, ar_width: int, ar_height: int) -> str:
return (
"<|im_start|>system\nYou are a helpful image generator.<|im_end|>\n"
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
"<|im_start|>assistant\n"
f"<|image start|>{ar_width}*{ar_height}<|image token|>"
)
def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
"""Sample pixel values at fixed fractional coordinates from a (C, H, W) tensor."""
t = img_tensor.float().clamp(0.0, 1.0)
if t.ndim == 4:
t = t[0] # unbatch
C, H, W = t.shape
values = []
for c, rh, rw in _PIXEL_SAMPLE_COORDS:
ri = min(int(rh * (H - 1)), H - 1)
ci = min(int(rw * (W - 1)), W - 1)
values.append(round(float(t[c, ri, ci]), 6))
return values
# ---------------------------------------------------------------------------
# End-to-end test
# ---------------------------------------------------------------------------
pytestmark = [
pytest.mark.full_model,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3201")
@hardware_test(res={"cuda": "H100"})
def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
"""
End-to-end text-to-image generation with MammothModa2 (AR -> DiT).
Verifies:
- Omni pipeline initialises with the two-stage YAML config.
- DiT stage outputs an image tensor with the correct shape.
- A fixed set of pixel values matches a golden reference
(regenerate with ``UPDATE_GOLDEN=1``).
"""
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
eol_token_id = int(gen_cfg["eol_token_id"])
visual_start = int(gen_cfg["visual_token_start_id"])
visual_end = int(gen_cfg["visual_token_end_id"])
height, width = 256, 256 # small for CI speed
ar_height, ar_width = height // _AR_PATCH_SIZE, width // _AR_PATCH_SIZE
expected_grid_tokens = ar_height * (ar_width + 1)
prompt_text = "A cat sitting on a laptop keyboard"
formatted_prompt = _format_t2i_prompt(prompt_text, ar_width, ar_height)
omni = omni_runner.omni
ar_sampling = SamplingParams(
temperature=0.0,
top_k=1,
max_tokens=max(1, expected_grid_tokens + 1),
detokenize=False,
)
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
outputs = list(
omni.generate(
[
{
"prompt": formatted_prompt,
"additional_information": {
"omni_task": ["t2i"],
"ar_width": [ar_width],
"ar_height": [ar_height],
"eol_token_id": [eol_token_id],
"visual_token_start_id": [visual_start],
"visual_token_end_id": [visual_end],
"image_height": [height],
"image_width": [width],
"num_inference_steps": [2],
"text_guidance_scale": [1.0],
"cfg_range": [0.0, 1.0],
"visual_ids": [
_IMAGE_TOKEN_ID,
_VIDEO_TOKEN_ID,
_VISION_START_TOKEN_ID,
_VISION_END_TOKEN_ID,
],
},
}
],
[ar_sampling, dit_sampling],
)
)
assert len(outputs) > 0, "Pipeline produced no outputs"
found_image = False
for out in outputs:
ro_list = getattr(out, "request_output", out)
if not isinstance(ro_list, list):
ro_list = [ro_list]
for ro in ro_list:
completion_outputs = getattr(ro, "outputs", None)
if not isinstance(completion_outputs, list):
continue
for completion in completion_outputs:
mm = getattr(completion, "multimodal_output", None)
if not (isinstance(mm, dict) and "image" in mm):
continue
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
for img_tensor in img_list:
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
sampled = _sample_pixels(img_tensor)
if os.environ.get("UPDATE_GOLDEN"):
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
elif _GOLDEN_T2I_PATH.exists():
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
for i, (got, exp) in enumerate(zip(sampled, golden)):
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
found_image = True
assert found_image, "No image tensor found in pipeline output"
@@ -63,7 +63,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt("请详细介绍鹦鹉的生活习性。")
request_config = {"prompts": prompt, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.core_model
@@ -80,7 +80,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
request_config = {"prompts": prompt, "images": image, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.core_model
@@ -99,7 +99,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt(f"{AUDIO_TOKEN}Please recognize the language of this speech and transcribe it. Format: oral.")
request_config = {"prompts": prompt, "audios": audio, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.core_model
@@ -116,7 +116,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt(f"{VIDEO_TOKEN}Describe what is happening in this video.")
request_config = {"prompts": prompt, "videos": video, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.core_model
@@ -136,7 +136,7 @@ def test_mixed_to_text(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt(f"{IMAGE_TOKEN}{AUDIO_TOKEN}Describe the image and transcribe the audio.")
request_config = {"prompts": prompt, "images": image, "audios": audio, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -152,7 +152,7 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt("请简单介绍一下北京。")
request_config = {"prompts": prompt, "modalities": ["audio"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -169,4 +169,4 @@ def test_image_to_audio(omni_runner, omni_runner_handler) -> None:
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
request_config = {"prompts": prompt, "images": image, "modalities": ["audio"]}
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@@ -18,9 +18,25 @@ import torch
from vllm import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni import Omni
MODEL_NAME = "OpenMOSS-Team/MOSS-TTS-Nano"
STAGE_CONFIG = get_deploy_config_path("moss_tts_nano.yaml")
# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize
_OMNI_RUNNER_PARAM = (
MODEL_NAME,
STAGE_CONFIG,
)
pytestmark = [
pytest.mark.full_model,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
SAMPLE_RATE = 48000
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS-Nano/main/assets/audio/zh_1.wav"
@@ -90,74 +106,64 @@ def _build_request(
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
"""Run a single request and return (waveform, sample_rate)."""
for stage_outputs in omni.generate(request, DEFAULT_SAMPLING):
for req_output in stage_outputs.request_output:
mm = req_output.outputs[0].multimodal_output
assert mm is not None, "Expected multimodal_output to be non-None"
audio = mm.get("audio")
sr = mm.get("sr")
assert audio is not None, "Expected 'audio' key in multimodal_output"
assert isinstance(audio, torch.Tensor), f"audio should be Tensor, got {type(audio)}"
return audio.cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
# Omni.generate returns list[OmniRequestOutput] by default (py_generator=False).
for omni_out in omni.generate(request, DEFAULT_SAMPLING):
mm = omni_out.multimodal_output
assert mm is not None, "Expected multimodal_output to be non-None"
audio = mm.get("audio")
sr = mm.get("sr")
assert audio is not None, "Expected 'audio' key in multimodal_output"
assert isinstance(audio, torch.Tensor), f"audio should be Tensor, got {type(audio)}"
return audio.cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
raise AssertionError("No stage outputs received")
@pytest.fixture(scope="module")
def omni_engine():
"""Module-scoped Omni engine to avoid re-loading the model for each test."""
return Omni(model=MODEL_NAME, stage_init_timeout=180)
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_nano_english(omni_engine, ref_audio_path):
"""English voice_clone mode (no prompt_text) produces non-empty 48 kHz audio."""
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
"""English TTS produces non-empty 48 kHz stereo audio."""
req = _build_request("Hello, this is a short voice cloning demo for testing.", ref_audio_path)
audio, sr = _collect_audio(omni_engine, req)
audio, sr = _collect_audio(omni_runner.omni, req)
assert sr == SAMPLE_RATE, f"Expected sample_rate={SAMPLE_RATE}, got {sr}"
assert audio.numel() > 0, "Audio tensor should not be empty"
assert not torch.all(audio == 0), "Audio should not be all-zeros (silence)"
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_nano_chinese(omni_engine, ref_audio_path):
"""Chinese voice_clone mode produces non-empty audio."""
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
"""Chinese TTS produces non-empty audio."""
req = _build_request("你好,这是语音合成测试。", ref_audio_path)
audio, sr = _collect_audio(omni_engine, req)
audio, sr = _collect_audio(omni_runner.omni, req)
assert sr == SAMPLE_RATE
assert audio.numel() > 0
assert not torch.all(audio == 0)
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_nano_deterministic(omni_engine, ref_audio_path):
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
"""Same seed produces identical waveforms."""
req = _build_request("Reproducible output test.", ref_audio_path, seed=123)
audio1, _ = _collect_audio(omni_engine, req)
audio2, _ = _collect_audio(omni_engine, req)
audio1, _ = _collect_audio(omni_runner.omni, req)
audio2, _ = _collect_audio(omni_runner.omni, req)
assert audio1.shape == audio2.shape, "Waveform shapes should match with same seed"
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms should match with same seed"
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"})
def test_moss_tts_nano_batch(omni_engine, ref_audio_path):
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
"""Batch of two requests returns audio for each."""
requests = [
_build_request("First request.", ref_audio_path),
_build_request("Second request.", ref_audio_path),
]
results = []
for stage_outputs in omni_engine.generate(requests, [DEFAULT_SAMPLING] * 2):
for req_output in stage_outputs.request_output:
mm = req_output.outputs[0].multimodal_output
assert mm is not None
results.append(mm["audio"].cpu())
# Single-stage pipeline: one SamplingParams object is broadcast to all prompts.
for omni_out in omni_runner.omni.generate(requests, DEFAULT_SAMPLING):
mm = omni_out.multimodal_output
assert mm is not None
results.append(mm["audio"].cpu())
assert len(results) == 2, f"Expected 2 outputs, got {len(results)}"
for i, audio in enumerate(results):
@@ -1,159 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end tests for Omni Sleep Mode across various model architectures.
"""
import pytest
import torch
from vllm import SamplingParams
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
MODEL_DIFF = "riverclouds/qwen_image_random"
def get_ack_info(ack, key, default=None):
if hasattr(ack, key):
return getattr(ack, key)
if isinstance(ack, dict):
return ack.get(key, default)
return default
def get_dynamic_devices(stage_idx, num_stages, tp_size):
total_gpus = torch.accelerator.device_count()
gpus_per_stage = tp_size
start_idx = stage_idx * gpus_per_stage
if start_idx + gpus_per_stage > total_gpus:
start_idx = start_idx % total_gpus
device_ids = [str(start_idx + i) for i in range(gpus_per_stage)]
return ",".join(device_ids)
# Test 1: Diffusion Model (2-Stage BAGEL)
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.parametrize("tp_size", [1])
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)
@pytest.mark.asyncio
async def test_diffusion_model_sleep_tp(tp_size):
num_gpus = torch.accelerator.device_count()
if num_gpus < tp_size:
pytest.skip(f"Skipping TP={tp_size}")
engine_args = {
"model": MODEL,
"enable_sleep_mode": True,
"tensor_parallel_size": tp_size,
"enforce_eager": True,
"trust_remote_code": True,
"dtype": "bfloat16",
"gpu_memory_utilization": 0.5,
}
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
try:
# BAGEL requires 2 params
diff_sp = OmniDiffusionSamplingParams(num_inference_steps=2, height=256, width=256)
llm_sp = SamplingParams()
# Warmup
async for _ in engine.generate("test", sampling_params=[llm_sp, diff_sp]):
pass
# Sleep all
acks = await engine.sleep(level=2)
statuses = [get_ack_info(ack, "status") for ack in acks]
assert all(s == "SUCCESS" for s in statuses), f"Sleep failed. Statuses: {statuses}"
# Wakeup & Verify
await engine.wake_up()
async for _ in engine.generate("verify", sampling_params=[llm_sp, diff_sp]):
pass
print(f"Diffusion TP={tp_size} Lifecycle OK")
finally:
engine.shutdown()
# Test 2: Multi-stage Manual Config
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.parametrize("tp_size", [1, 2])
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.asyncio
async def test_multistage_sleep_h100(tp_size):
num_gpus = torch.accelerator.device_count()
if num_gpus < tp_size * 2:
pytest.skip("Not enough GPUs")
stages = []
for i in range(2):
devs = get_dynamic_devices(i, 2, tp_size)
stages.append(
{
"stage_id": i,
"stage_type": "llm" if i == 0 else "diffusion",
"runtime": {"process": True, "devices": devs},
"engine_args": {
"model": MODEL,
"model_stage": "thinker" if i == 0 else "base",
"tensor_parallel_size": tp_size,
"gpu_memory_utilization": 0.4,
"dtype": "bfloat16",
"enable_sleep_mode": True,
"trust_remote_code": True,
},
}
)
connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
engine = AsyncOmni(
model=MODEL, stages=stages, connectors=connectors, enable_sleep_mode=True, stage_init_timeout=1200
)
try:
sp = OmniDiffusionSamplingParams(num_inference_steps=2)
async for _ in engine.generate("warmup", sampling_params=[SamplingParams(), sp]):
pass
acks = await engine.sleep(stage_ids=[0, 1], level=2)
assert len(acks) == 2
await engine.wake_up(stage_ids=[0, 1])
async for _ in engine.generate("verify", sampling_params=[SamplingParams(), sp]):
pass
finally:
engine.shutdown()
# Test 3: Pure Diffusion Single-Stage
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.parametrize("tp_size", [1, 2])
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.asyncio
async def test_pure_diffusion_scenario(tp_size):
engine_args = {
"model": MODEL_DIFF,
"enable_sleep_mode": True,
"tensor_parallel_size": tp_size,
"enforce_eager": True,
"dtype": "bfloat16",
"gpu_memory_utilization": 0.5,
}
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
try:
await engine.sleep(level=1)
await engine.wake_up()
async for _ in engine.generate("test", sampling_params=[SamplingParams()]):
pass
print("Pure Diffusion OK")
finally:
engine.shutdown()
@@ -11,27 +11,35 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import numpy as np
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "k2-fsa/OmniVoice"
STAGE_CONFIG = get_deploy_config_path("omnivoice.yaml")
# (model, stage_config_path, extra_omni_kwargs) — see ``omni_runner`` in tests.helpers.fixtures.runtime
_OMNI_RUNNER_PARAM = (
MODEL,
STAGE_CONFIG,
{
"trust_remote_code": True,
"log_stats": True,
},
)
pytestmark = [
pytest.mark.full_model,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def get_stage_config():
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "omnivoice.yaml"
)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_omnivoice_text_to_audio() -> None:
def test_omnivoice_text_to_audio(omni_runner: OmniRunner) -> None:
"""
Test OmniVoice text-to-audio generation via offline Omni runner.
Deploy Setting: omnivoice.yaml (enforce_eager=true)
@@ -40,17 +48,10 @@ def test_omnivoice_text_to_audio() -> None:
"""
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
with OmniRunner(
MODEL,
stage_configs_path=get_stage_config(),
trust_remote_code=True,
log_stats=True,
) as runner:
prompts = {"prompt": "Hello, this is a test for text to audio."}
prompts = {"prompt": "Hello, this is a test for text to audio."}
sampling_params_list = [OmniDiffusionSamplingParams()]
sampling_params_list = [OmniDiffusionSamplingParams()]
outputs = list(runner.omni.generate(prompts, sampling_params_list=sampling_params_list))
outputs = list(omni_runner.omni.generate(prompts, sampling_params_list=sampling_params_list))
assert len(outputs) > 0, "No outputs generated"
@@ -1,292 +0,0 @@
"""
Tests for Ovis Image model pipeline.
Strategy:
1. `mock_dependencies` fixture mocks heavy external components (VAE, Scheduler, TextEncoder)
to allow fast testing of the pipeline logic without downloading weights.
- Mocks are configured to return tensors on the correct device.
- Transformer is mocked dynamically to return random noise of correct shape.
2. `test_real_transformer_init_and_forward` tests the actual `OvisImageTransformer2DModel`
initialization and forward pass with a small configuration to ensure code coverage
and correctness of the model definition itself, independent of the pipeline mocks.
"""
import pytest
import torch
from pytest_mock import MockerFixture
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
# Mock the OvisImageTransformer2DModel to avoid complex init if needed,
# or let it run if it's lightweight. It's likely not lightweight.
# Better to mock the transformer forwarding to return random noise.
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image import OvisImagePipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@pytest.fixture
def mock_dependencies(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
"""
Mock external dependencies to avoid loading real models.
"""
device = get_local_device()
# Mock Tokenizer
mock_tokenizer = mocker.MagicMock()
mock_tokenizer.return_value = mocker.MagicMock(
input_ids=torch.zeros((1, 50), dtype=torch.long, device=device),
attention_mask=torch.ones((1, 50), dtype=torch.long, device=device),
)
mock_tokenizer.apply_chat_template.return_value = "dummy prompt"
mock_tokenizer.model_max_length = 1024
# Mock Text Encoder
mock_text_encoder = mocker.MagicMock()
mock_text_encoder.dtype = torch.float32
# Output of text encoder must be on the same device as inputs (which are moved to execution_device)
mock_text_encoder.return_value.last_hidden_state = torch.randn(1, 50, 32, device=device)
# Mock VAE
mock_vae = mocker.MagicMock()
mock_vae.config.block_out_channels = [128, 256, 512, 512] # Scale factor 8
mock_vae.config.scale_factor_temporal = 1
mock_vae.config.scale_factor_spatial = 8
mock_vae.config.scaling_factor = 0.18215
mock_vae.config.shift_factor = 0.0
# Decode return value
mock_vae.decode.return_value = [torch.randn(1, 3, 128, 128, device=device)]
# Ensure .to() returns self so configuration persists
mock_vae.to.return_value = mock_vae
# Mock Scheduler
mock_scheduler = mocker.MagicMock()
mock_scheduler.config = mocker.MagicMock()
# Timesteps on device to match latents during denoising loop interaction if needed
mock_scheduler.timesteps = torch.tensor([1.0, 0.5, 0.0], device=device)
mock_scheduler.set_timesteps.return_value = None
# Make step return dynamic based on input sample shape
def mock_scheduler_step(model_output, timestep, sample, **kwargs):
# sample is the latents, should be preserved
return (torch.randn_like(sample),)
mock_scheduler.step.side_effect = mock_scheduler_step
module_path = "vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image"
monkeypatch.setattr(f"{module_path}.Qwen2TokenizerFast.from_pretrained", lambda *a, **k: mock_tokenizer)
monkeypatch.setattr(f"{module_path}.Qwen3Model.from_pretrained", lambda *a, **k: mock_text_encoder)
monkeypatch.setattr(f"{module_path}.AutoencoderKL.from_pretrained", lambda *a, **k: mock_vae)
monkeypatch.setattr(
f"{module_path}.FlowMatchEulerDiscreteScheduler.from_pretrained", lambda *a, **k: mock_scheduler
)
return {
"tokenizer": mock_tokenizer,
"text_encoder": mock_text_encoder,
"vae": mock_vae,
"scheduler": mock_scheduler,
"device": device,
}
@pytest.fixture
def ovis_pipeline(mocker: MockerFixture, mock_dependencies, monkeypatch: pytest.MonkeyPatch):
"""
Creates an OvisImagePipeline instance with mocked components.
"""
# Create config
tf_config = TransformerConfig(
params={
"in_channels": 4,
"out_channels": 4,
"sample_size": 32,
"patch_size": 2,
"num_attention_heads": 4,
"attention_head_dim": 8,
"num_layers": 1,
"caption_channels": 32,
}
)
od_config = OmniDiffusionConfig(
model="dummy-ovis",
tf_model_config=tf_config,
dtype=torch.float32,
num_gpus=1,
)
# Mock Transformer Layer separately to avoid full init
# We patch OvisImageTransformer2DModel class in the module
mock_transformer_cls = mocker.MagicMock()
mock_transformer_instance = mocker.MagicMock()
mock_transformer_instance.dtype = torch.float32
mock_transformer_instance.in_channels = 16 # Must be 16 so num_channel_latents=4, packed=16
# Forward return: noise prediction
def mock_forward(hidden_states, *args, **kwargs):
# hidden_states shape: (B, SeqLen, Channels)
return (torch.randn_like(hidden_states),)
mock_transformer_instance.forward.side_effect = mock_forward
# Also make the instance itself callable to mimic __call__
mock_transformer_instance.side_effect = mock_forward
mock_transformer_cls.return_value = mock_transformer_instance
monkeypatch.setattr(
"vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image.OvisImageTransformer2DModel", mock_transformer_cls
)
# Initialize pipeline
# We use a dummy model path check override
mocker.patch("os.path.exists", return_value=True)
pipeline = OvisImagePipeline(od_config=od_config)
return pipeline
def test_interface_compliance(ovis_pipeline):
"""Verify methods required by vllm-omni framework."""
assert hasattr(ovis_pipeline, "load_weights")
assert hasattr(ovis_pipeline, "scheduler")
assert hasattr(ovis_pipeline, "transformer")
assert hasattr(ovis_pipeline, "text_encoder")
# assert hasattr(ovis_pipeline, "vae") # Ovis uses VAE
def test_basic_generation(ovis_pipeline):
"""Test the forward pass logic."""
# Setup request
req = OmniDiffusionRequest(
prompts=["A photo of a cat"],
sampling_params=OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=2,
guidance_scale=1.0,
),
)
output = ovis_pipeline(req)
assert output is not None
assert output.output is not None
# Output should be a tensor from mocked VAE decode [torch.randn(1, 3, 128, 128)]
assert isinstance(output.output, torch.Tensor)
assert output.output.shape == (1, 3, 128, 128)
# Check that transformer was called
assert ovis_pipeline.transformer.call_count > 0
def test_guidance_scale(ovis_pipeline):
"""Test that classifier-free guidance path is taken when scale > 1.0."""
req = OmniDiffusionRequest(
prompts=[
{
"prompt": "A photo of a cat",
"negative_prompt": "bad quality",
}
],
sampling_params=OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=1,
guidance_scale=2.0, # Trigger CFG
),
)
ovis_pipeline(req)
assert ovis_pipeline.transformer.call_count >= 2
def test_resolution_check(ovis_pipeline):
"""Test resolution divisible validation logic if present."""
# Pass odd resolution
req = OmniDiffusionRequest(
prompts=["test"],
sampling_params=OmniDiffusionSamplingParams(
height=250, # Not divisible by 16 (8*2)
width=250,
),
)
# Should warn but proceed (as per code I read earlier) or resize?
# The code had `logger.warning(...)`
output = ovis_pipeline(req)
assert output is not None
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
def test_real_transformer_init_and_forward(mocker: MockerFixture):
"""Test the real OvisImageTransformer2DModel initialization and forward pass for coverage."""
from vllm_omni.diffusion.models.ovis_image.ovis_image_transformer import OvisImageTransformer2DModel
device = get_local_device()
tf_config = TransformerConfig(
params={
"patch_size": 2,
"in_channels": 16,
"out_channels": 16,
"num_layers": 1,
"num_single_layers": 1,
"attention_head_dim": 8,
"num_attention_heads": 2,
"joint_attention_dim": 32,
"axes_dims_rope": (4, 4, 4),
}
)
od_config = OmniDiffusionConfig(model="dummy-ovis", tf_model_config=tf_config, dtype=torch.bfloat16, num_gpus=1)
torch.set_default_dtype(torch.bfloat16)
# Mock distributed state for QKVParallelLinear initialization
# We patch get_tp_group because get_tensor_model_parallel_rank calls it and asserts _TP is not None
mock_group = mocker.MagicMock()
mock_group.rank_in_group = 0
mock_group.world_size = 1
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
# Initialize real model
model = OvisImageTransformer2DModel(
od_config=od_config,
patch_size=1,
in_channels=16,
out_channels=16,
num_single_layers=1,
attention_head_dim=8,
num_attention_heads=2,
joint_attention_dim=32,
axes_dims_rope=(2, 2, 4),
).to(device)
# Create dummy inputs
B, Seq, C = 1, 16, 16
hidden_states = torch.randn(B, Seq, C, device=device)
encoder_hidden_states = torch.randn(B, 10, 32, device=device) # joint_attention_dim=32
timestep = torch.tensor([1], device=device)
img_ids = torch.zeros(Seq, 3, device=device)
txt_ids = torch.zeros(10, 3, device=device)
# Run forward
output = model(
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
timestep=timestep,
img_ids=img_ids,
txt_ids=txt_ids,
return_dict=False,
)
assert output is not None
assert isinstance(output, tuple)
assert output[0].shape == hidden_states.shape
@@ -1,319 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end tests for the unified quantization framework (PR #1764).
Validates FP8 quantization works correctly for all supported model types:
- Single-stage diffusion models (FLUX.1-dev, Qwen-Image, Z-Image-Turbo)
- Multi-stage models (BAGEL: LLM + Diffusion)
Tests verify:
1. FP8 quantization produces valid images
2. Memory usage is lower than BF16 baseline
3. Multi-stage models only quantize the diffusion stage (not the LLM stage)
Usage:
# Run all FP8 quantization tests
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v
# Run single-stage tests only (faster, needs ~25GB VRAM)
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v -k "single_stage"
# Run BAGEL multi-stage test (needs ~55GB VRAM, H100 recommended)
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v -k "bagel"
# Run FLUX test only
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v -k "flux"
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
from typing import Any
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
# ─── helpers ──────────────────────────────────────────────────────────────────
def _generate_single_stage_image(
model: str,
quantization: str | None = None,
height: int = 256,
width: int = 256,
num_inference_steps: int = 2,
seed: int = 42,
**extra_omni_kwargs: Any,
) -> tuple[list, float]:
"""Generate an image with a single-stage diffusion model.
Returns (images, peak_memory_gib).
"""
omni_kwargs: dict[str, Any] = dict(extra_omni_kwargs)
if quantization:
omni_kwargs["quantization"] = quantization
with OmniRunner(model, **omni_kwargs) as runner:
torch.accelerator.reset_peak_memory_stats()
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
outputs = runner.omni.generate(
"a photo of a cat sitting on a laptop keyboard",
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=generator,
),
)
peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
first_output = outputs[0]
assert first_output.final_output_type == "image"
assert hasattr(first_output, "request_output") and first_output.request_output
req_out = first_output.request_output[0]
assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
images = req_out.images
assert len(images) >= 1
assert images[0].width == width
assert images[0].height == height
return images, peak_mem
def _generate_bagel_image(
quantization_config: str | None = None,
num_inference_steps: int = 15,
) -> tuple[Any, float]:
"""Generate an image with BAGEL (multi-stage: LLM + Diffusion).
Returns (generated_image, peak_memory_gib).
"""
config_path = get_deploy_config_path("ci/bagel.yaml")
omni_kwargs: dict[str, Any] = {
"model": "ByteDance-Seed/BAGEL-7B-MoT",
"stage_configs_path": config_path,
"stage_init_timeout": 300,
}
if quantization_config:
omni_kwargs["quantization_config"] = quantization_config
model_name = omni_kwargs.pop("model")
with OmniRunner(model_name, **omni_kwargs) as runner:
omni = runner.omni
torch.accelerator.reset_peak_memory_stats()
params_list = omni.default_sampling_params_list
if len(params_list) > 1:
params_list[1].num_inference_steps = num_inference_steps # type: ignore
params_list[1].extra_args = { # type: ignore
"cfg_text_scale": 4.0,
"cfg_img_scale": 1.5,
}
prompt = "<|im_start|>A futuristic city skyline at twilight, cyberpunk style<|im_end|>"
omni_outputs = list(
omni.generate(
prompts=[{"prompt": prompt, "modalities": ["image"]}],
sampling_params_list=params_list,
)
)
peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
# Extract image
generated_image = None
for req_output in omni_outputs:
if images := getattr(req_output, "images", None):
generated_image = images[0]
break
if hasattr(req_output, "request_output") and req_output.request_output:
for stage_out in req_output.request_output:
if hasattr(stage_out, "images") and stage_out.images:
generated_image = stage_out.images[0]
break
if generated_image:
break
assert generated_image is not None, "No images generated from BAGEL"
assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}"
# Check LLM stage output — should have finish_reason=stop (not length)
for req_output in omni_outputs:
if hasattr(req_output, "request_output"):
for stage_out in req_output.request_output:
if hasattr(stage_out, "outputs"):
for comp_out in stage_out.outputs:
if hasattr(comp_out, "finish_reason"):
assert comp_out.finish_reason == "stop", (
f"LLM stage finish_reason={comp_out.finish_reason}, "
f"text={comp_out.text!r}. "
"FP8 may have leaked to the LLM stage."
)
return generated_image, peak_mem
# ─── Single-stage diffusion model tests ──────────────────────────────────────
@hardware_test(res={"cuda": "L4"})
def test_single_stage_zimage_fp8():
"""Z-Image-Turbo with FP8 generates valid images."""
images, _ = _generate_single_stage_image(
model="Tongyi-MAI/Z-Image-Turbo",
quantization="fp8",
)
assert len(images) >= 1
images[0].save("test_zimage_fp8.png")
@hardware_test(res={"cuda": "L4"})
def test_single_stage_zimage_fp8_uses_less_memory():
"""FP8 should use less peak memory than BF16 for Z-Image-Turbo."""
_, mem_bf16 = _generate_single_stage_image(
model="Tongyi-MAI/Z-Image-Turbo",
quantization=None,
)
torch.accelerator.empty_cache()
_, mem_fp8 = _generate_single_stage_image(
model="Tongyi-MAI/Z-Image-Turbo",
quantization="fp8",
)
print(f"Z-Image BF16 peak memory: {mem_bf16:.2f} GiB")
print(f"Z-Image FP8 peak memory: {mem_fp8:.2f} GiB")
assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)"
@hardware_test(res={"cuda": "L4"})
def test_single_stage_qwen_image_fp8():
"""Qwen-Image (random weights) with FP8 generates valid images."""
model = "riverclouds/qwen_image_random"
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
pytest.skip("qwen_image_random not available on this platform")
images, _ = _generate_single_stage_image(
model=model,
quantization="fp8",
)
assert len(images) >= 1
images[0].save("test_qwen_image_fp8.png")
@hardware_test(res={"cuda": "H100"})
def test_single_stage_flux_fp8():
"""FLUX.1-dev with FP8 generates valid images."""
images, _ = _generate_single_stage_image(
model="black-forest-labs/FLUX.1-dev",
quantization="fp8",
height=512,
width=512,
num_inference_steps=4,
)
assert len(images) >= 1
images[0].save("test_flux_fp8.png")
@hardware_test(res={"cuda": "H100"})
def test_single_stage_flux_fp8_uses_less_memory():
"""FP8 should use less peak memory than BF16 for FLUX.1-dev."""
_, mem_bf16 = _generate_single_stage_image(
model="black-forest-labs/FLUX.1-dev",
quantization=None,
height=512,
width=512,
num_inference_steps=4,
)
torch.accelerator.empty_cache()
_, mem_fp8 = _generate_single_stage_image(
model="black-forest-labs/FLUX.1-dev",
quantization="fp8",
height=512,
width=512,
num_inference_steps=4,
)
print(f"FLUX BF16 peak memory: {mem_bf16:.2f} GiB")
print(f"FLUX FP8 peak memory: {mem_fp8:.2f} GiB")
assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)"
# ─── Multi-stage model tests (BAGEL) ─────────────────────────────────────────
@hardware_test(res={"cuda": "H100"})
def test_bagel_fp8_generates_image():
"""BAGEL with FP8 quantization_config generates a valid image.
FP8 should only apply to the diffusion stage (Stage-1), not the
LLM stage (Stage-0). We verify this by checking:
1. Image is generated successfully
2. LLM stage finish_reason is 'stop' (not 'length' from garbled output)
"""
image, _ = _generate_bagel_image(quantization_config="fp8")
image.save("test_bagel_fp8.png")
@hardware_test(res={"cuda": "H100"})
def test_bagel_bf16_generates_image():
"""BAGEL without quantization generates a valid image (baseline)."""
image, _ = _generate_bagel_image(quantization_config=None)
image.save("test_bagel_bf16.png")
# ─── Quantization config routing tests ────────────────────────────────────────
@hardware_test(res={"cuda": "L4"})
def test_quantization_key_maps_to_quantization_config():
"""The old 'quantization' kwarg should map to 'quantization_config'
in OmniDiffusionConfig.from_kwargs for backwards compatibility."""
from vllm_omni.diffusion.data import OmniDiffusionConfig
config = OmniDiffusionConfig.from_kwargs(model="test", quantization="fp8")
assert config.quantization_config is not None
assert config.quantization_config.get_name() == "fp8"
@hardware_test(res={"cuda": "L4"})
def test_quantization_config_key_takes_priority():
"""When both 'quantization' and 'quantization_config' are set,
'quantization_config' takes priority."""
from vllm_omni.diffusion.data import OmniDiffusionConfig
config = OmniDiffusionConfig.from_kwargs(
model="test",
quantization="fp8",
quantization_config={"method": "fp8", "activation_scheme": "static"},
)
assert config.quantization_config is not None
assert config.quantization_config.activation_scheme == "static"
@hardware_test(res={"cuda": "L4"})
def test_single_stage_quantization_config_key():
"""Single-stage model using quantization_config (dict) key generates images."""
images, _ = _generate_single_stage_image(
model="Tongyi-MAI/Z-Image-Turbo",
quantization=None,
quantization_config="fp8",
)
assert len(images) >= 1
@@ -22,6 +22,11 @@ from tests.helpers.media import (
)
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
pytestmark = [
pytest.mark.full_model,
pytest.mark.omni,
]
QUANTIZED_MODEL = "Intel/Qwen2.5-Omni-7B-int4-AutoRound"
BASELINE_MODEL = "Qwen/Qwen2.5-Omni-7B"
@@ -56,8 +61,6 @@ quant_params = [(QUANTIZED_MODEL, stage_config)]
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_text_to_text(omni_runner, omni_runner_handler):
@@ -66,7 +69,7 @@ def test_text_to_text(omni_runner, omni_runner_handler):
"prompts": "What is the capital of China?",
"modalities": ["text"],
}
response = omni_runner_handler.send_request(request_config)
response = omni_runner_handler.send_omni_request(request_config)
assert response.success, f"Request failed: {response.error_message}"
assert response.text_content and len(response.text_content.strip()) > 0
@@ -76,8 +79,6 @@ def test_text_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_audio_to_text(omni_runner, omni_runner_handler):
@@ -91,7 +92,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler):
"audios": (audio, 16000),
"modalities": ["text"],
}
response = omni_runner_handler.send_request(request_config)
response = omni_runner_handler.send_omni_request(request_config)
assert response.success, f"Request failed: {response.error_message}"
assert response.text_content and len(response.text_content.strip()) > 0
@@ -101,8 +102,6 @@ def test_audio_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_image_to_text(omni_runner, omni_runner_handler):
@@ -114,7 +113,7 @@ def test_image_to_text(omni_runner, omni_runner_handler):
"images": image,
"modalities": ["text"],
}
response = omni_runner_handler.send_request(request_config)
response = omni_runner_handler.send_omni_request(request_config)
assert response.success, f"Request failed: {response.error_message}"
assert response.text_content and len(response.text_content.strip()) > 0
@@ -124,8 +123,6 @@ def test_image_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_video_to_text(omni_runner, omni_runner_handler):
@@ -137,7 +134,7 @@ def test_video_to_text(omni_runner, omni_runner_handler):
"videos": video,
"modalities": ["text"],
}
response = omni_runner_handler.send_request(request_config)
response = omni_runner_handler.send_omni_request(request_config)
assert response.success, f"Request failed: {response.error_message}"
assert response.text_content and len(response.text_content.strip()) > 0
@@ -147,8 +144,6 @@ def test_video_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_mix_to_audio(omni_runner, omni_runner_handler):
@@ -166,5 +161,5 @@ def test_mix_to_audio(omni_runner, omni_runner_handler):
"audios": (audio, 16000),
"modalities": ["audio"],
}
response = omni_runner_handler.send_request(request_config)
response = omni_runner_handler.send_omni_request(request_config)
assert response.success, f"Request failed: {response.error_message}"
@@ -78,7 +78,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
}
# Test single completion
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@pytest.mark.advanced_model
@@ -98,4 +98,4 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None:
request_config = {"prompts": get_question("text_only"), "modalities": ["text"]}
# Test single completion
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@@ -61,4 +61,4 @@ def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
request_config = {"prompts": get_question(), "videos": video, "modalities": ["audio"]}
# Test single completion
omni_runner_handler.send_request(request_config)
omni_runner_handler.send_omni_request(request_config)
@@ -22,6 +22,15 @@ from tests.helpers.media import (
)
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
pytestmark = [
pytest.mark.full_model,
pytest.mark.omni,
]
_SKIP_ISSUE_3195 = pytest.mark.skip(
reason="https://github.com/vllm-project/vllm-omni/issues/3195",
)
QUANTIZED_MODEL = "Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound"
BASELINE_MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
@@ -69,8 +78,7 @@ quant_params = [(QUANTIZED_MODEL, stage_config)]
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@_SKIP_ISSUE_3195
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_text_to_text(omni_runner, omni_runner_handler):
@@ -89,8 +97,7 @@ def test_text_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@_SKIP_ISSUE_3195
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_audio_to_text(omni_runner, omni_runner_handler):
@@ -114,8 +121,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@_SKIP_ISSUE_3195
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_image_to_text(omni_runner, omni_runner_handler):
@@ -137,8 +143,7 @@ def test_image_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@_SKIP_ISSUE_3195
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_video_to_text(omni_runner, omni_runner_handler):
@@ -160,8 +165,7 @@ def test_video_to_text(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@_SKIP_ISSUE_3195
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_video_to_audio(omni_runner, omni_runner_handler):
@@ -182,8 +186,7 @@ def test_video_to_audio(omni_runner, omni_runner_handler):
# ------------------------------------------------------------------
@pytest.mark.advanced_model
@pytest.mark.omni
@_SKIP_ISSUE_3195
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
def test_mix_to_audio(omni_runner, omni_runner_handler):
@@ -68,7 +68,7 @@ def get_prompt():
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
@@ -64,7 +64,7 @@ def get_prompt():
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
@@ -1,626 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for diffusion batching via AsyncOmni.
This test fires multiple concurrent ``AsyncOmni.generate()`` calls for a
diffusion model and validates that every caller receives its correct
individual result. When the underlying diffusion stage is configured with
``batch_size > 1`` (via stage config or ``StageDiffusionClient``), the
requests will be batched internally.
Even without explicit batching config this test is useful for verifying
that concurrent async requests are handled correctly.
Usage (standalone):
python tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py \
--model <model_name_or_path> \
--num-prompts 8
Or via pytest:
pytest tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -s
"""
from __future__ import annotations
import argparse
import asyncio
import sys
import time
import uuid
from pathlib import Path
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
# ------------------------------------------------------------------
models = ["tiny-random/Qwen-Image"]
# ------------------------------------------------------------------
# Prompt fixtures
# ------------------------------------------------------------------
WARMUP_PROMPTS: list[dict[str, str]] = [
{"prompt": "a sunflower in a glass vase", "negative_prompt": "blurry"},
{"prompt": "a rocket launching into space", "negative_prompt": "low detail"},
{"prompt": "a small cottage in the snowy mountains", "negative_prompt": "foggy"},
{"prompt": "a colorful parrot sitting on a tree branch", "negative_prompt": "low contrast"},
]
TEST_PROMPTS: list[dict[str, str]] = [
{"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"},
{"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": "cinematic, realistic"},
{"prompt": "a futuristic city skyline at sunset", "negative_prompt": "blurry, foggy"},
{"prompt": "a bowl of fresh strawberries", "negative_prompt": "low detail"},
{"prompt": "a medieval knight standing in the rain", "negative_prompt": "modern clothing"},
{"prompt": "a cat wearing sunglasses lounging in a garden", "negative_prompt": "dark lighting"},
{"prompt": "a spaceship flying above a volcano", "negative_prompt": "low contrast"},
{"prompt": "a watercolor painting of a mountain lake", "negative_prompt": "photo, realistic"},
]
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _default_sampling_params(**overrides) -> OmniDiffusionSamplingParams:
defaults = dict(
num_inference_steps=2,
width=256,
height=256,
guidance_scale=0.0,
)
defaults.update(overrides)
return OmniDiffusionSamplingParams(**defaults)
def _default_sync_sampling_params(**overrides) -> OmniDiffusionSamplingParams:
"""Create sampling params for the synchronous Omni.generate() API."""
defaults = dict(
num_inference_steps=2,
width=256,
height=256,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
)
defaults.update(overrides)
return OmniDiffusionSamplingParams(**defaults)
async def _collect_generate(omni: AsyncOmni, prompt, request_id, sampling_params_list) -> OmniRequestOutput:
"""Consume the AsyncOmni.generate() async generator and return the last output."""
last_output: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
):
last_output = output
if last_output is None:
raise RuntimeError(f"No output received for request {request_id}")
return last_output
def _extract_images(output: OmniRequestOutput) -> list:
"""Extract images from an OmniRequestOutput, handling both direct
and nested request_output structures."""
if output.images:
return output.images
# When the output comes from the orchestrator pipeline, images may be
# nested inside request_output.
inner = getattr(output, "request_output", None)
if inner is not None and hasattr(inner, "images") and inner.images:
return inner.images
return []
# ------------------------------------------------------------------
# Warm-up (async)
# ------------------------------------------------------------------
async def warmup(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
"""Warm-up: send prompts in parallel to pre-load the model."""
print(f"🔥 Warming up with {len(prompts)} prompts ...")
sp = _default_sampling_params(num_inference_steps=2)
start = time.perf_counter()
tasks = [
_collect_generate(
omni,
prompt=p,
request_id=f"warmup-{i}-{uuid.uuid4().hex[:8]}",
sampling_params_list=[sp],
)
for i, p in enumerate(prompts)
]
await asyncio.gather(*tasks)
elapsed = time.perf_counter() - start
print(f" Warm-up done in {elapsed:.2f}s\n")
# ------------------------------------------------------------------
# Single (sequential) benchmark
# ------------------------------------------------------------------
async def run_single(omni: AsyncOmni, prompts: list[dict[str, str]]) -> float:
"""Run prompts one-by-one sequentially."""
print(f"🧩 Running SINGLE (sequential) mode – {len(prompts)} prompts ...")
sp = _default_sampling_params()
total_start = time.perf_counter()
for i, prompt in enumerate(prompts):
start = time.perf_counter()
result = await _collect_generate(
omni,
prompt=prompt,
request_id=f"single-{i}-{uuid.uuid4().hex[:8]}",
sampling_params_list=[sp],
)
elapsed = time.perf_counter() - start
images = _extract_images(result)
print(f" prompt {i}: {elapsed:.2f}s ({len(images)} images)")
total = time.perf_counter() - total_start
print(f" ✅ Total single-mode: {total:.2f}s\n")
return total
# ------------------------------------------------------------------
# Batch (parallel) benchmark — concurrent individual requests
# ------------------------------------------------------------------
async def run_batch(
omni: AsyncOmni,
prompts: list[dict[str, str]],
label: str = "batch",
) -> float:
"""Send all prompts concurrently via asyncio.gather (one request per prompt)."""
print(f"⚙️ Running {label.upper()} mode – {len(prompts)} prompts concurrently ...")
sp = _default_sampling_params()
start = time.perf_counter()
tasks = [
_collect_generate(
omni,
prompt=p,
request_id=f"{label}-{i}-{uuid.uuid4().hex[:8]}",
sampling_params_list=[sp],
)
for i, p in enumerate(prompts)
]
results = await asyncio.gather(*tasks)
elapsed = time.perf_counter() - start
for i, result in enumerate(results):
images = _extract_images(result)
print(f" prompt {i}: {len(images)} images, request_id={result.request_id}")
print(f" ✅ Total {label} mode: {elapsed:.2f}s\n")
return elapsed
# ------------------------------------------------------------------
# Explicit batch — single generate() call with list of prompts
# ------------------------------------------------------------------
async def run_batch_explicit(
omni: AsyncOmni,
prompts: list[dict[str, str]],
label: str = "batch_explicit",
) -> float:
"""Send all prompts as a single batch via generate(prompt=[...]).
Passes a *list* of prompts so that all are processed in **one**
``DiffusionEngine.step()`` call. The orchestrator detects
``isinstance(prompt, list)`` and routes to the batch path.
A single ``OmniRequestOutput`` is yielded containing ALL generated
images combined.
"""
print(f"⚡ Running {label.upper()} mode – {len(prompts)} prompts in ONE engine call ...")
sp = _default_sampling_params()
request_id = f"{label}-{uuid.uuid4().hex[:8]}"
start = time.perf_counter()
result: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompts,
request_id=request_id,
sampling_params_list=[sp],
):
result = output
elapsed = time.perf_counter() - start
if result is not None:
images = _extract_images(result)
print(f" Got {len(images)} images total from batch, request_id={result.request_id}")
else:
print(" ⚠️ No output received from batch generate()")
print(f" ✅ Total {label} mode: {elapsed:.2f}s\n")
return elapsed
# ------------------------------------------------------------------
# Async validation helpers
# ------------------------------------------------------------------
async def validate_batch_explicit(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
"""Validate generate(prompt=[...]) returns a single result with all images."""
print(f"🔍 Validating batch generate() correctness with {len(prompts)} prompts ...")
sp = _default_sampling_params()
request_id = f"validate-batch-{uuid.uuid4().hex[:8]}"
result: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompts,
request_id=request_id,
sampling_params_list=[sp],
):
result = output
assert result is not None, "No output received from batch generate()"
images = _extract_images(result)
# Batch mode returns ONE output with ALL images combined
assert len(images) == len(prompts), f"Expected {len(prompts)} images (one per prompt), got {len(images)}"
assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}"
print(f" ✅ Batch returned {len(images)} images with correct request_id.\n")
async def validate_concurrent(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
"""Validate that every concurrent request receives a distinct result
with its own request_id."""
print(f"🔍 Validating concurrent correctness with {len(prompts)} prompts ...")
sp = _default_sampling_params()
request_ids = [f"validate-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))]
tasks = [
_collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp])
for p, rid in zip(prompts, request_ids)
]
results = await asyncio.gather(*tasks)
assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}"
returned_ids = [r.request_id for r in results]
for rid in request_ids:
assert rid in returned_ids, f"Missing request_id {rid} in results"
print(" ✅ All request_ids matched, results count correct.\n")
# ------------------------------------------------------------------
# Single vs Parallel comparison (CLI only)
# ------------------------------------------------------------------
async def compare_single_vs_parallel(
model: str,
prompts: list[dict[str, str]],
batch_size: int = 1,
) -> None:
"""Run the same prompts sequentially then in parallel and print a comparison."""
omni = AsyncOmni(model=model, diffusion_batch_size=batch_size)
try:
await warmup(omni, WARMUP_PROMPTS)
single_time = await run_single(omni, prompts)
parallel_time = await run_batch(omni, prompts, label="parallel")
explicit_time = await run_batch_explicit(omni, prompts, label="batch_explicit")
finally:
omni.shutdown()
speedup_parallel = single_time / parallel_time if parallel_time > 0 else float("inf")
speedup_explicit = single_time / explicit_time if explicit_time > 0 else float("inf")
print("=" * 60)
print(f"📊 Summary ({len(prompts)} prompts)")
print(f" Sequential : {single_time:.2f}s")
print(f" Parallel (gather) : {parallel_time:.2f}s ({speedup_parallel:.2f}x)")
print(f" Explicit batch : {explicit_time:.2f}s ({speedup_explicit:.2f}x)")
print("=" * 60)
# ------------------------------------------------------------------
# CLI main entrypoint
# ------------------------------------------------------------------
async def main(model: str, num_prompts: int, mode: str, batch_size: int = 1) -> None:
prompts = (TEST_PROMPTS * ((num_prompts // len(TEST_PROMPTS)) + 1))[:num_prompts]
if mode == "compare":
await compare_single_vs_parallel(model, prompts, batch_size=batch_size)
return
omni = AsyncOmni(model=model, diffusion_batch_size=batch_size)
try:
await warmup(omni, WARMUP_PROMPTS)
if mode == "validate":
await validate_concurrent(omni, prompts)
elif mode == "validate_batch":
await validate_batch_explicit(omni, prompts)
elif mode == "batch":
await run_batch(omni, prompts, label="measurement")
elif mode == "batch_explicit":
await run_batch_explicit(omni, prompts)
elif mode == "single":
await run_single(omni, prompts)
else:
raise ValueError(f"Unknown mode: {mode}")
finally:
omni.shutdown()
# ==================================================================
# pytest test cases
# ==================================================================
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_sync_sequential(model_name: str):
"""Test that synchronous Omni can generate images for multiple prompts
submitted sequentially (one at a time) and each returns a valid image."""
try:
with OmniRunner(model_name) as runner:
m = runner.omni
sp = _default_sync_sampling_params()
prompts = TEST_PROMPTS[:4]
for i, prompt in enumerate(prompts):
outputs = m.generate(prompt, sp)
first_output = outputs[0]
assert first_output.final_output_type == "image", (
f"Expected 'image', got '{first_output.final_output_type}'"
)
# Images are surfaced both at top-level and inside request_output
images = _extract_images(first_output)
assert len(images) >= 1, f"Expected at least 1 image for prompt {i}, got {len(images)}"
assert images[0].width == 256
assert images[0].height == 256
print(f" prompt {i}: OK ({len(images)} images)")
except Exception as e:
print(f"Test failed with error: {e}")
raise
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_sync_multi_prompt(model_name: str):
"""Test that synchronous Omni correctly handles a list of multiple
prompts submitted at once and returns one result per prompt.
Note: Omni.generate() iterates the list and submits each prompt
individually with its own request_id. This tests concurrent request
handling at the diffusion stage, not the explicit list-batch path
(which is only available via AsyncOmni).
"""
try:
with OmniRunner(model_name) as runner:
m = runner.omni
sp = _default_sync_sampling_params()
prompts = TEST_PROMPTS[:4]
outputs = m.generate(prompts, sp)
assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}"
for i, output in enumerate(outputs):
assert output.final_output_type == "image", (
f"Output {i} final_output_type expected 'image', got '{output.final_output_type}'"
)
images = _extract_images(output)
assert images and len(images) >= 1, f"Expected at least 1 image for prompt {i}"
assert images[0].width == 256
assert images[0].height == 256
print(f" prompt {i}: OK ({len(images)} images, request_id={output.request_id})")
# Verify all request_ids are distinct
request_ids = [o.request_id for o in outputs]
assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids found: {request_ids}"
except Exception as e:
print(f"Test failed with error: {e}")
raise
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_async_concurrent(model_name: str):
"""Test that AsyncOmni correctly handles multiple concurrent requests
fired via asyncio.gather. Each request_id must appear in the results."""
async def _inner():
omni = AsyncOmni(model=model_name, diffusion_batch_size=1)
try:
prompts = TEST_PROMPTS[:4]
sp = _default_sampling_params()
request_ids = [f"async-concurrent-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))]
tasks = [
_collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp])
for p, rid in zip(prompts, request_ids)
]
results = await asyncio.gather(*tasks)
assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}"
returned_ids = [r.request_id for r in results]
for rid in request_ids:
assert rid in returned_ids, f"Missing request_id {rid} in results"
for i, result in enumerate(results):
images = _extract_images(result)
assert len(images) >= 1, f"No images for prompt {i}"
assert images[0].width == 256
assert images[0].height == 256
print(f" prompt {i}: OK ({len(images)} images, request_id={result.request_id})")
finally:
omni.shutdown()
asyncio.run(_inner())
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_async_explicit_batch(model_name: str):
"""Test that AsyncOmni batch mode (generate(prompt=[...])) dispatches
all prompts in a single engine call and returns a single combined result.
The list-prompt path routes through the orchestrator's
``add_batch_request_async`` → ``AsyncOmni.generate_batch``
and yields ONE ``OmniRequestOutput`` with ALL images combined.
"""
async def _inner():
omni = AsyncOmni(model=model_name, diffusion_batch_size=4)
try:
prompts = TEST_PROMPTS[:4]
sp = _default_sampling_params()
request_id = f"explicit-batch-{uuid.uuid4().hex[:8]}"
# Batch mode: pass list of prompts → single request_id
result: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompts,
request_id=request_id,
sampling_params_list=[sp],
):
result = output
assert result is not None, "No output received from batch generate()"
images = _extract_images(result)
# One image per prompt, all in a single output
assert len(images) == len(prompts), f"Expected {len(prompts)} images in combined output, got {len(images)}"
assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}"
for i, img in enumerate(images):
assert img.width == 256, f"Image {i} width mismatch"
assert img.height == 256, f"Image {i} height mismatch"
print(f" ✅ Batch returned {len(images)} images, request_id={result.request_id}")
finally:
omni.shutdown()
asyncio.run(_inner())
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_num_outputs(model_name: str):
"""Test that the diffusion model respects num_outputs_per_prompt and
generates the correct number of images per request."""
try:
with OmniRunner(model_name) as runner:
m = runner.omni
num_outputs = 2
sp = _default_sync_sampling_params(num_outputs_per_prompt=num_outputs)
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
sp,
)
first_output = outputs[0]
assert first_output.final_output_type == "image"
images = _extract_images(first_output)
assert images is not None and len(images) == num_outputs, (
f"Expected {num_outputs} images, got {len(images) if images else 0}"
)
for img in images:
assert img.width == 256
assert img.height == 256
except Exception as e:
print(f"Test failed with error: {e}")
raise
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_distinct_results(model_name: str):
"""Test that different prompts produce distinct images when batched,
ensuring the batching logic does not mix up results across requests."""
try:
with OmniRunner(model_name) as runner:
m = runner.omni
sp = _default_sync_sampling_params()
prompts = [
{"prompt": "a bright red apple on a white table", "negative_prompt": "blurry"},
{"prompt": "a blue ocean with white waves crashing", "negative_prompt": "blurry"},
]
outputs = m.generate(prompts, sp)
assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}"
# Verify each output has a unique request_id
request_ids = [o.request_id for o in outputs]
assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids: {request_ids}"
# Verify each output has images
for i, output in enumerate(outputs):
images = _extract_images(output)
assert images and len(images) >= 1, f"No images for prompt {i}"
assert images[0].width == 256
assert images[0].height == 256
except Exception as e:
print(f"Test failed with error: {e}")
raise
# ------------------------------------------------------------------
# CLI
# ------------------------------------------------------------------
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="E2E diffusion concurrent benchmark / validation")
parser.add_argument("--model", type=str, required=True, help="Model name or path")
parser.add_argument("--num-prompts", type=int, default=8, help="Number of prompts to run")
parser.add_argument("--batch-size", type=int, default=1, help="Diffusion batch size (1 = no batching)")
parser.add_argument(
"--mode",
choices=["batch", "batch_explicit", "single", "compare", "validate", "validate_batch"],
default="compare",
help=(
"Run mode: 'batch' (parallel gather), 'batch_explicit' (list-prompt batch API), "
"'single' (sequential), 'compare' (all three), "
"'validate' (concurrent correctness), 'validate_batch' (list-prompt correctness)"
),
)
args = parser.parse_args()
asyncio.run(main(args.model, args.num_prompts, args.mode, batch_size=args.batch_size))
@@ -1,414 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
System test for Sequence Parallel (SP) backends: Ulysses and Ring attention.
Tests verify that SP inference produces correct outputs compared to baseline.
"""
import gc
import os
import sys
import time
from pathlib import Path
from typing import NamedTuple
import numpy as np
import pytest
import torch
import torch.distributed as dist
from PIL import Image
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
# Test configuration
MODELS = ["riverclouds/qwen_image_random"]
PROMPT = "a photo of a cat sitting on a laptop keyboard"
DEFAULT_HEIGHT = 256
DEFAULT_WIDTH = 256
DEFAULT_SEED = 42
DEFAULT_STEPS = 4
DIFF_MEAN_THRESHOLD = 2e-2
DIFF_MAX_THRESHOLD = 2e-1
class InferenceResult(NamedTuple):
"""Result of an inference run."""
images: list[Image.Image]
elapsed_ms: float
def _cleanup_distributed():
"""Clean up distributed environment and GPU resources."""
if dist.is_initialized():
dist.destroy_process_group()
for key in ["MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"]:
os.environ.pop(key, None)
gc.collect()
if current_omni_platform.is_available():
current_omni_platform.empty_cache()
current_omni_platform.synchronize()
time.sleep(5)
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
ta = torch.from_numpy(np.asarray(a.convert("RGB"), dtype=np.float32) / 255.0)
tb = torch.from_numpy(np.asarray(b.convert("RGB"), dtype=np.float32) / 255.0)
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
abs_diff = torch.abs(ta - tb)
return abs_diff.mean().item(), abs_diff.max().item()
def _run_inference(
model_name: str,
dtype: torch.dtype,
attn_backend: str,
ulysses_degree: int = 1,
ring_degree: int = 1,
height: int = DEFAULT_HEIGHT,
width: int = DEFAULT_WIDTH,
seed: int = DEFAULT_SEED,
warmup: bool = True,
) -> InferenceResult:
"""Run inference with specified configuration.
Args:
warmup: If True, run one warmup iteration before the timed run.
"""
parallel_config = DiffusionParallelConfig(ulysses_degree=ulysses_degree, ring_degree=ring_degree)
try:
with OmniRunner(
model_name,
parallel_config=parallel_config,
dtype=dtype,
attention_backend=attn_backend,
) as runner:
omni = runner.omni
# Warmup run (not timed)
if warmup:
_ = omni.generate(
PROMPT,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=DEFAULT_STEPS,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed + 1000),
num_outputs_per_prompt=1,
),
)
# Timed run
start = time.time()
outputs = omni.generate(
PROMPT,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=DEFAULT_STEPS,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
elapsed_ms = (time.time() - start) * 1000
return InferenceResult(
images=outputs[0].request_output.images,
elapsed_ms=elapsed_ms,
)
finally:
_cleanup_distributed()
# =============================================================================
# Correctness & Performance Tests
# =============================================================================
# SP configurations: (ulysses_degree, ring_degree, height, width, warmup, is_perf_test)
# - warmup: whether to run warmup for this SP config
# - is_perf_test: whether this is a performance test (show speedup metrics)
SP_CONFIGS_L2 = [
# Hybrid - correctness only
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
]
SP_CONFIGS_L3 = [
# Ulysses-2 - performance test
(2, 1, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True),
(1, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True), # Ring-2 - performance test
# Hybrid - correctness only
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
(4, 1, 272, 272, False, False), # Ulysses-4 - shape and correctness
]
def _get_sp_mode(ulysses_degree: int, ring_degree: int) -> str:
"""Get SP mode name for logging."""
if ulysses_degree > 1 and ring_degree == 1:
return f"ulysses-{ulysses_degree}"
elif ring_degree > 1 and ulysses_degree == 1:
return f"ring-{ring_degree}"
else:
return f"hybrid-{ulysses_degree}x{ring_degree}"
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 2, "rocm": 2})
@pytest.mark.parametrize("model_name", MODELS)
def test_sp_correctness(model_name: str):
"""Test that SP inference produces correct outputs and measure performance.
Runs baseline once per unique (height, width), then tests all SP configs.
Note: Run with `pytest -v -s` to see detailed output.
"""
device_count = current_omni_platform.get_device_count()
# Cache baseline results by (height, width)
# Key: (height, width), Value: (result, warmup_used)
baseline_cache: dict[tuple[int, int], InferenceResult] = {}
# Collect results for summary
results: list[dict] = []
print("\n" + "=" * 70)
print(f"Sequence Parallel Test - Model: {model_name}")
print(f"Available GPUs: {device_count}")
print("=" * 70)
for ulysses_degree, ring_degree, height, width, sp_warmup, is_perf_test in SP_CONFIGS_L2:
sp_size = ulysses_degree * ring_degree
sp_mode = _get_sp_mode(ulysses_degree, ring_degree)
if device_count < sp_size:
print(f"\n[{sp_mode}] SKIPPED (requires {sp_size} GPUs)")
continue
# Determine baseline warmup: only for default size (performance tests)
cache_key = (height, width)
baseline_warmup = height == DEFAULT_HEIGHT and width == DEFAULT_WIDTH
# Get or compute baseline for this (height, width)
if cache_key not in baseline_cache:
print(f"\n--- Running baseline {height}x{width} (warmup={baseline_warmup}) ---")
baseline = _run_inference(
model_name,
torch.bfloat16,
"sdpa",
height=height,
width=width,
warmup=baseline_warmup,
)
assert len(baseline.images) == 1
baseline_cache[cache_key] = baseline
print(f"[baseline] {height}x{width}: {baseline.elapsed_ms:.0f}ms")
else:
baseline = baseline_cache[cache_key]
# Run SP
print(f"\n--- Running {sp_mode} (warmup={sp_warmup}) ---")
sp_result = _run_inference(
model_name,
torch.bfloat16,
"sdpa",
ulysses_degree=ulysses_degree,
ring_degree=ring_degree,
height=height,
width=width,
warmup=sp_warmup,
)
assert len(sp_result.images) == 1
# Compare outputs (correctness)
mean_diff, max_diff = _diff_metrics(baseline.images[0], sp_result.images[0])
# Build result entry
result = {
"mode": sp_mode,
"sp_size": sp_size,
"height": height,
"width": width,
"baseline_ms": baseline.elapsed_ms,
"sp_ms": sp_result.elapsed_ms,
"mean_diff": mean_diff,
"max_diff": max_diff,
"is_perf_test": is_perf_test,
}
results.append(result)
# Output based on test type
if is_perf_test:
speedup = baseline.elapsed_ms / sp_result.elapsed_ms if sp_result.elapsed_ms > 0 else 0
result["speedup"] = speedup
print(
f"[{sp_mode}] {sp_size} GPUs | "
f"baseline: {baseline.elapsed_ms:.0f}ms, sp: {sp_result.elapsed_ms:.0f}ms, "
f"speedup: {speedup:.2f}x"
)
else:
print(f"[{sp_mode}] {sp_size} GPUs | sp: {sp_result.elapsed_ms:.0f}ms (correctness only)")
print(f"[{sp_mode}] diff: mean={mean_diff:.6e}, max={max_diff:.6e}")
# Assert correctness
assert mean_diff <= DIFF_MEAN_THRESHOLD and max_diff <= DIFF_MAX_THRESHOLD, (
f"[{sp_mode}] SP output differs from baseline: mean={mean_diff:.6e}, max={max_diff:.6e}"
)
# Summary
print("\n" + "=" * 70)
print("SUMMARY")
print("=" * 70)
print(f"{'Mode':<15} {'GPUs':<6} {'Size':<10} {'Baseline':<12} {'SP':<12} {'Speedup':<10} {'Status'}")
print("-" * 70)
for r in results:
speedup_str = f"{r['speedup']:.2f}x" if r.get("speedup") else "N/A"
baseline_str = f"{r['baseline_ms']:.0f}ms" if r["is_perf_test"] else "N/A"
status = "PASS" if r["mean_diff"] <= DIFF_MEAN_THRESHOLD else "FAIL"
print(
f"{r['mode']:<15} {r['sp_size']:<6} {r['height']}x{r['width']:<5} "
f"{baseline_str:<12} {r['sp_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
)
print("=" * 70)
# TODO: After PR#1272 is merged, add markers
# @pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
@pytest.mark.parametrize("model_name", MODELS)
def test_sp_correctness_advanced(model_name: str):
"""Test that SP inference produces correct outputs and measure performance.
Runs baseline once per unique (height, width), then tests all SP configs.
Note: Run with `pytest -v -s` to see detailed output.
"""
device_count = current_omni_platform.get_device_count()
# Cache baseline results by (height, width)
# Key: (height, width), Value: (result, warmup_used)
baseline_cache: dict[tuple[int, int], InferenceResult] = {}
# Collect results for summary
results: list[dict] = []
print("\n" + "=" * 70)
print(f"Sequence Parallel Test - Model: {model_name}")
print(f"Available GPUs: {device_count}")
print("=" * 70)
for ulysses_degree, ring_degree, height, width, sp_warmup, is_perf_test in SP_CONFIGS_L3:
sp_size = ulysses_degree * ring_degree
sp_mode = _get_sp_mode(ulysses_degree, ring_degree)
if device_count < sp_size:
print(f"\n[{sp_mode}] SKIPPED (requires {sp_size} GPUs)")
continue
# Determine baseline warmup: only for default size (performance tests)
cache_key = (height, width)
baseline_warmup = height == DEFAULT_HEIGHT and width == DEFAULT_WIDTH
# Get or compute baseline for this (height, width)
if cache_key not in baseline_cache:
print(f"\n--- Running baseline {height}x{width} (warmup={baseline_warmup}) ---")
baseline = _run_inference(
model_name,
torch.bfloat16,
"sdpa",
height=height,
width=width,
warmup=baseline_warmup,
)
assert len(baseline.images) == 1
baseline_cache[cache_key] = baseline
print(f"[baseline] {height}x{width}: {baseline.elapsed_ms:.0f}ms")
else:
baseline = baseline_cache[cache_key]
# Run SP
print(f"\n--- Running {sp_mode} (warmup={sp_warmup}) ---")
sp_result = _run_inference(
model_name,
torch.bfloat16,
"sdpa",
ulysses_degree=ulysses_degree,
ring_degree=ring_degree,
height=height,
width=width,
warmup=sp_warmup,
)
assert len(sp_result.images) == 1
# Compare outputs (correctness)
mean_diff, max_diff = _diff_metrics(baseline.images[0], sp_result.images[0])
# Build result entry
result = {
"mode": sp_mode,
"sp_size": sp_size,
"height": height,
"width": width,
"baseline_ms": baseline.elapsed_ms,
"sp_ms": sp_result.elapsed_ms,
"mean_diff": mean_diff,
"max_diff": max_diff,
"is_perf_test": is_perf_test,
}
results.append(result)
# Output based on test type
if is_perf_test:
speedup = baseline.elapsed_ms / sp_result.elapsed_ms if sp_result.elapsed_ms > 0 else 0
result["speedup"] = speedup
print(
f"[{sp_mode}] {sp_size} GPUs | "
f"baseline: {baseline.elapsed_ms:.0f}ms, sp: {sp_result.elapsed_ms:.0f}ms, "
f"speedup: {speedup:.2f}x"
)
else:
print(f"[{sp_mode}] {sp_size} GPUs | sp: {sp_result.elapsed_ms:.0f}ms (correctness only)")
print(f"[{sp_mode}] diff: mean={mean_diff:.6e}, max={max_diff:.6e}")
# Assert correctness
assert mean_diff <= DIFF_MEAN_THRESHOLD and max_diff <= DIFF_MAX_THRESHOLD, (
f"[{sp_mode}] SP output differs from baseline: mean={mean_diff:.6e}, max={max_diff:.6e}"
)
# Summary
print("\n" + "=" * 70)
print("SUMMARY")
print("=" * 70)
print(f"{'Mode':<15} {'GPUs':<6} {'Size':<10} {'Baseline':<12} {'SP':<12} {'Speedup':<10} {'Status'}")
print("-" * 70)
for r in results:
speedup_str = f"{r['speedup']:.2f}x" if r.get("speedup") else "N/A"
baseline_str = f"{r['baseline_ms']:.0f}ms" if r["is_perf_test"] else "N/A"
status = "PASS" if r["mean_diff"] <= DIFF_MEAN_THRESHOLD else "FAIL"
print(
f"{r['mode']:<15} {r['sp_size']:<6} {r['height']}x{r['width']:<5} "
f"{baseline_str:<12} {r['sp_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
)
print("=" * 70)
@@ -1,75 +0,0 @@
import pytest
import torch
from tests.helpers.mark import hardware_test
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# Match unprefixed HF id even when MODEL_PREFIX is set (omni_runner resolves full path).
_QWEN_IMAGE_RANDOM_ID = "riverclouds/qwen_image_random"
def _is_qwen_image_random(model_path: str) -> bool:
return model_path.rstrip("/").endswith(_QWEN_IMAGE_RANDOM_ID)
models = ["Tongyi-MAI/Z-Image-Turbo", "riverclouds/qwen_image_random"]
# Modelscope can't find riverclouds/qwen_image_random
# TODO: When NPU support is ready, remove this branch.
if current_omni_platform.is_npu():
models = ["Tongyi-MAI/Z-Image-Turbo", "Qwen/Qwen-Image"]
# omni_runner expects (model, stage_configs_path); single-stage diffusion has no YAML.
test_params = [(m, None) for m in models]
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2})
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
def test_diffusion_model(omni_runner, run_level):
resolved = omni_runner.model_name
if run_level == "core_model" and not _is_qwen_image_random(resolved):
pytest.skip()
if run_level == "advanced_model" and _is_qwen_image_random(resolved):
pytest.skip()
# high resolution may cause OOM on L4
height = 256
width = 256
sampling = OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=2,
)
# OmniRunner.generate() is typed for list[TextPrompt]; diffusion uses Omni.generate(str, ...).
outputs = omni_runner.omni.generate(
"a photo of a cat sitting on a laptop keyboard",
sampling,
)
# Extract images from request_output['images']
first_output = outputs[0]
assert first_output.final_output_type == "image"
if not hasattr(first_output, "request_output") or not first_output.request_output:
raise ValueError("No request_output found in OmniRequestOutput")
req_out = first_output.request_output
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
raise ValueError("Invalid request_output structure or missing 'images' key")
images = req_out.images
assert len(images) == 2
# check image size
assert images[0].width == width
assert images[0].height == height
images[0].save("image_output.png")
@@ -1,56 +0,0 @@
import os
import pytest
import torch
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
models = ["Wan-AI/Wan2.2-T2V-A14B-Diffusers"]
@pytest.mark.parametrize("model_name", models)
def test_video_diffusion_model(model_name: str):
with OmniRunner(
model_name,
boundary_ratio=0.875,
flow_shift=5.0,
) as runner:
# Use minimal settings for testing
# num_frames must satisfy: num_frames % vae_scale_factor_temporal == 1
# For Wan2.2, vae_scale_factor_temporal=4, so valid values are 5, 9, 13, 17, ...
height = 480
width = 640
num_frames = 5
outputs = runner.omni.generate(
prompts="A cat sitting on a table",
sampling_params_list=OmniDiffusionSamplingParams(
height=height,
width=width,
num_frames=num_frames,
num_inference_steps=2,
guidance_scale=1.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
),
)
first_output = outputs[0]
assert first_output.final_output_type == "image"
if not hasattr(first_output, "request_output") or not first_output.request_output:
raise ValueError("No request_output found in OmniRequestOutput")
req_out = first_output.request_output
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
raise ValueError("Invalid request_output structure or missing 'images' key")
frames = req_out.images[0]
assert frames is not None
assert hasattr(frames, "shape")
# frames shape: (batch, num_frames, height, width, channels)
assert frames.shape[1] == num_frames
assert frames.shape[2] == height
assert frames.shape[3] == width
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
System test for TeaCache backend.
This test verifies that TeaCache acceleration works correctly with diffusion models.
It uses minimal settings to keep test time short for CI.
"""
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# Use random weights model for testing
models = ["riverclouds/qwen_image_random"]
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.cache
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_teacache(model_name: str):
"""Test TeaCache backend with diffusion model."""
# Configure TeaCache with default settings for fast testing
cache_config = {
"rel_l1_thresh": 0.2, # Default threshold
}
with OmniRunner(
model_name,
cache_backend="tea_cache",
cache_config=cache_config,
) as runner:
# Use minimal settings for fast testing
height = 256
width = 256
num_inference_steps = 4 # Minimal steps for fast test
outputs = runner.omni.generate(
"a photo of a cat sitting on a laptop keyboard",
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
),
)
# Extract images from request_output['images']
first_output = outputs[0]
assert first_output.final_output_type == "image"
if not hasattr(first_output, "request_output") or not first_output.request_output:
raise ValueError("No request_output found in OmniRequestOutput")
req_out = first_output.request_output
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
raise ValueError("Invalid request_output structure or missing 'images' key")
images = req_out.images
# Verify generation succeeded
assert images is not None
assert len(images) == 1
# Check image size
assert images[0].width == width
assert images[0].height == height
@@ -1,152 +0,0 @@
import os
import sys
from pathlib import Path
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
import time
from tests.helpers.runtime import OmniRunner
from vllm_omni.platforms import current_omni_platform
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
all_model_configs = [
{
"model_name": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
"out_width": 1280,
"out_height": 720,
"out_frames": 16,
},
{
"model_name": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
"out_width": 1280,
"out_height": 720,
"out_frames": 16,
},
{
"model_name": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
"out_width": 1280,
"out_height": 720,
"out_frames": 16,
},
{
"model_name": "Qwen/Qwen-Image",
"out_width": 3072,
"out_height": 3072,
"out_frames": 1,
},
# {
# "model_name": "stepfun-ai/NextStep-1.1",
# "out_width": 3072,
# "out_height": 3072,
# "out_frames": 1,
# },
]
def is_nextstep_model(model_name: str) -> bool:
"""Check if the model is a NextStep model by reading its config."""
from vllm.transformers_utils.config import get_hf_file_to_dict
try:
cfg = get_hf_file_to_dict("config.json", model_name)
if cfg and cfg.get("model_type") == "nextstep":
return True
except Exception:
pass
return False
def model_run(model_configs, tp, out_height, out_width, out_frames, using_tile, vae_patch_parallel_size=1):
parallel_config = DiffusionParallelConfig(
tensor_parallel_size=tp,
vae_patch_parallel_size=vae_patch_parallel_size,
)
omni_kwargs = {
"vae_use_tiling": using_tile,
"parallel_config": parallel_config,
}
use_nextstep = is_nextstep_model(model_configs["model_name"])
if use_nextstep:
# NextStep-1.1 requires explicit pipeline class
omni_kwargs["model_class_name"] = "NextStep11Pipeline"
with OmniRunner(model_configs["model_name"], **omni_kwargs) as runner:
m = runner.omni
image = Image.new("RGB", (out_width, out_height), (0, 0, 0))
start = time.perf_counter()
outputs = m.generate(
{
"prompt": "A cat sitting on a table",
"multi_modal_data": {"image": image},
},
sampling_params_list=OmniDiffusionSamplingParams(
height=model_configs["out_height"],
width=model_configs["out_width"],
num_frames=model_configs["out_frames"],
num_inference_steps=2,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
),
)
end = time.perf_counter()
first_output = outputs[0]
req_out = first_output.request_output
frames = req_out.images[0]
if isinstance(frames, torch.Tensor):
frames = frames.detach().cpu().numpy()
elif isinstance(frames, Image.Image):
frames = np.array(frames)
# frames shape: (batch, num_frames, height, width, channels)
cost = (end - start) * 1000
return frames, cost
cleanup_dist_env_and_memory()
@pytest.mark.parametrize("model_configs", all_model_configs)
def test_vae_parallel_model(model_configs: str):
parallel_result, parallel_time = model_run(
model_configs=model_configs,
tp=2,
out_width=1280,
out_height=704,
out_frames=5,
using_tile=True,
vae_patch_parallel_size=2,
)
non_parallel_result, non_parallel_time = model_run(
model_configs=model_configs,
tp=2,
out_width=1280,
out_height=704,
out_frames=5,
using_tile=True,
vae_patch_parallel_size=1,
)
result_diff = np.abs(non_parallel_result - parallel_result)
mean_threshold = 3e-2
max_threshold = 3e-2 # they should be totally same
print(
f"{model_configs['model_name']} TP = 2 (tile + parallel vs tile): "
f"mean_abs_diff={result_diff.mean():.6e}, max_abs_diff={result_diff.max():.6e}; "
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
f"parallel generate take time: {parallel_time:.2f} ms, non-parallel take time: {non_parallel_time:.2f} ms"
)
# assert non_parallel_time > parallel_time
assert result_diff.mean() < mean_threshold
assert result_diff.max() < max_threshold
+6 -3
View File
@@ -41,7 +41,8 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor:
@pytest.mark.core_model
@pytest.mark.omni
@pytest.mark.advanced_model
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm2_zero_shot_001(voxcpm2_engine):
"""Test zero-shot TTS produces valid audio output."""
@@ -54,7 +55,8 @@ def test_voxcpm2_zero_shot_001(voxcpm2_engine):
@pytest.mark.core_model
@pytest.mark.omni
@pytest.mark.advanced_model
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm2_voice_clone_002(voxcpm2_engine):
"""Test voice cloning with a reference audio file.
@@ -95,7 +97,8 @@ def test_voxcpm2_voice_clone_002(voxcpm2_engine):
@pytest.mark.core_model
@pytest.mark.omni
@pytest.mark.advanced_model
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm2_prefill_decode_mixed_batch_003(voxcpm2_engine):
"""Regression: prefill+decode mixed batch must not crash (PR #2903)."""
@@ -3,44 +3,28 @@
"""E2E test for VoxCPM offline inference."""
import json
import os
from pathlib import Path
from typing import Any
import numpy as np
import pytest
import torch
import tests.helpers.runtime as omni_runtime
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.model_executor.models.voxcpm.voxcpm_runtime_utils import (
prepare_voxcpm_hf_config_dir,
resolve_voxcpm_model_dir,
)
from tests.helpers.stage_config import get_deploy_config_path
VOXCPM_MODEL = os.environ.get("VOXCPM_MODEL", "OpenBMB/VoxCPM1.5")
STAGE_CONFIG = str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
)
VOXCPM_MODEL = "OpenBMB/VoxCPM1.5"
STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml")
SAMPLE_RATE = 24000
# (model, stage_config_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
_OMNI_RUNNER_PARAM = (VOXCPM_MODEL, STAGE_CONFIG)
@pytest.fixture(autouse=True)
def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
"""Limit the NPU cleanup workaround to this VoxCPM test module only."""
original_cleanup = omni_runtime.cleanup_dist_env_and_memory
def _safe_cleanup() -> None:
try:
original_cleanup()
except RuntimeError as exc:
if "Allocator for npu is not a DeviceAllocator" in str(exc):
return
raise
monkeypatch.setattr(omni_runtime, "cleanup_dist_env_and_memory", _safe_cleanup)
pytestmark = [
pytest.mark.full_model,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def _build_prompt(text: str) -> dict[str, Any]:
@@ -94,50 +78,9 @@ def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
@pytest.fixture
def voxcpm_model_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> str:
model_dir = resolve_voxcpm_model_dir(VOXCPM_MODEL)
hf_config_env = os.environ.get("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH")
if hf_config_env:
hf_config_dir = Path(hf_config_env).expanduser()
else:
hf_config_dir = tmp_path / "voxcpm_hf_config"
if not (hf_config_dir / "config.json").exists():
prepare_voxcpm_hf_config_dir(model_dir, hf_config_dir)
monkeypatch.setenv("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH", str(hf_config_dir))
return str(model_dir)
def test_prepare_voxcpm_hf_config_dir(tmp_path: Path):
model_dir = tmp_path / "model"
model_dir.mkdir()
(model_dir / "config.json").write_text(json.dumps({"hidden_size": 1024}), encoding="utf-8")
(model_dir / "generation_config.json").write_text(json.dumps({"do_sample": False}), encoding="utf-8")
hf_config_dir = prepare_voxcpm_hf_config_dir(model_dir, tmp_path / "voxcpm_hf_config")
prepared_config = json.loads((hf_config_dir / "config.json").read_text(encoding="utf-8"))
assert prepared_config["model_type"] == "voxcpm"
assert prepared_config["architectures"] == ["VoxCPMForConditionalGeneration"]
assert (hf_config_dir / "generation_config.json").exists()
def test_resolve_voxcpm_model_dir_local_path(tmp_path: Path):
model_dir = tmp_path / "OpenBMB" / "VoxCPM1.5"
model_dir.mkdir(parents=True)
assert resolve_voxcpm_model_dir(str(model_dir)) == model_dir
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm_zero_shot_001(voxcpm_model_path: str):
with OmniRunner(voxcpm_model_path, stage_configs_path=STAGE_CONFIG) as runner:
outputs = list(runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
def test_voxcpm_zero_shot_001(omni_runner: OmniRunner) -> None:
outputs = list(omni_runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
assert outputs, "No outputs returned"
+54 -69
View File
@@ -31,7 +31,7 @@ from vllm import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.entrypoints.async_omni import AsyncOmni
MODEL = "mistralai/Voxtral-4B-TTS-2603"
@@ -42,6 +42,14 @@ MIN_AUDIO_SAMPLES = 1000
VOICE = "casual_female"
TEST_TEXT = "Hello, how are you?"
# (model, stage_config_path, extra_omni_kwargs) for indirect parametrize on
# ``omni_runner_function`` (function-scoped: must exit before ``AsyncOmni`` test).
_OMNI_RUNNER_PARAM = (
MODEL,
STAGE_CONFIG,
{"enforce_eager": True},
)
def _compose_request(model_name: str, text: str, voice: str) -> dict:
"""Build the TTS input dict using mistral tokenizer."""
@@ -58,83 +66,60 @@ def _compose_request(model_name: str, text: str, voice: str) -> dict:
}
def _resolve_stage_config(run_level: str) -> str:
"""Resolve stage config: strip load_format for advanced_model (real weights)."""
if run_level == "advanced_model":
return modify_stage_config(
STAGE_CONFIG,
deletes={
"stages": {
0: ["load_format"],
1: ["load_format"],
}
},
)
return STAGE_CONFIG
@pytest.mark.advanced_model
@pytest.mark.tts
@pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True)
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None:
"""Offline sync path; function-scoped runner so AsyncOmni test does not overlap a live Omni."""
omni = omni_runner_function.omni
inputs = _compose_request(MODEL, TEST_TEXT, VOICE)
sampling_params = SamplingParams(max_tokens=2500)
sampling_params_list = [sampling_params, sampling_params]
outputs = list(omni.generate([inputs], sampling_params_list))
assert len(outputs) > 0, "No outputs generated"
# Find audio output from the final stage
audio_data = None
for o in outputs:
mm = getattr(o, "multimodal_output", None)
if mm and "audio" in mm:
audio_data = mm["audio"]
break
assert audio_data is not None, "No audio output found in any stage output"
# Concatenate audio chunks if returned as a list of tensors
if isinstance(audio_data, list):
audio_tensor = torch.cat(audio_data)
elif isinstance(audio_data, torch.Tensor):
audio_tensor = audio_data
else:
audio_tensor = torch.tensor(audio_data)
audio_array = audio_tensor.float().cpu().numpy()
assert len(audio_array) > MIN_AUDIO_SAMPLES, (
f"Audio too short: {len(audio_array)} samples, expected > {MIN_AUDIO_SAMPLES}"
)
# Verify audio isn't all zeros / silence
assert np.max(np.abs(audio_array)) > 0.01, "Audio appears to be silence"
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_voxtral_tts_offline_basic(run_level):
"""Test basic Voxtral TTS offline inference with a voice preset."""
stage_config = _resolve_stage_config(run_level)
with OmniRunner(
MODEL,
stage_configs_path=stage_config,
enforce_eager=True,
) as runner:
omni = runner.omni
inputs = _compose_request(MODEL, TEST_TEXT, VOICE)
sampling_params = SamplingParams(max_tokens=2500)
sampling_params_list = [sampling_params, sampling_params]
outputs = list(omni.generate([inputs], sampling_params_list))
assert len(outputs) > 0, "No outputs generated"
# Find audio output from the final stage
audio_data = None
for o in outputs:
mm = getattr(o, "multimodal_output", None)
if mm and "audio" in mm:
audio_data = mm["audio"]
break
assert audio_data is not None, "No audio output found in any stage output"
# Concatenate audio chunks if returned as a list of tensors
if isinstance(audio_data, list):
audio_tensor = torch.cat(audio_data)
elif isinstance(audio_data, torch.Tensor):
audio_tensor = audio_data
else:
audio_tensor = torch.tensor(audio_data)
audio_array = audio_tensor.float().cpu().numpy()
assert len(audio_array) > MIN_AUDIO_SAMPLES, (
f"Audio too short: {len(audio_array)} samples, expected > {MIN_AUDIO_SAMPLES}"
)
# Verify audio isn't all zeros / silence
assert np.max(np.abs(audio_array)) > 0.01, "Audio appears to be silence"
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_voxtral_tts_offline_streaming(run_level):
def test_voxtral_tts_offline_streaming():
"""Test AsyncOmni streaming inference for Voxtral TTS."""
async def _run():
stage_config = _resolve_stage_config(run_level)
async_omni = AsyncOmni(
model=MODEL,
stage_configs_path=stage_config,
stage_configs_path=STAGE_CONFIG,
stage_init_timeout=300,
enforce_eager=True,
)
@@ -0,0 +1,32 @@
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunnerHandler
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 1})
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
def test_text_to_video_001(omni_runner_handler: OmniRunnerHandler):
sampling = OmniDiffusionSamplingParams(
height=512,
width=512,
num_frames=8,
fps=8,
num_inference_steps=2,
guidance_scale=4.0,
seed=42,
)
request_config = {
"model": MODEL,
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"sampling_params": sampling,
}
omni_runner_handler.send_diffusion_request(request_config)
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunnerHandler
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "Tongyi-MAI/Z-Image-Turbo"
PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard."
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2})
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
def test_zimage(omni_runner_handler: OmniRunnerHandler):
# high resolution may cause OOM on L4
sampling = OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=2,
guidance_scale=0.0,
seed=42,
num_outputs_per_prompt=2,
)
request_config = {
"model": MODEL,
"prompt": PROMPT,
"sampling_params": sampling,
}
omni_runner_handler.send_diffusion_request(request_config)
@@ -1,275 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Z-Image end-to-end tests for diffusion parallelism.
This file currently covers:
- DiT tensor parallelism (TP=2) vs TP=1.
- VAE patch parallelism (vae_patch_parallel_size=2) vs baseline on TP=2.
Note: CUDA-only (>=2 GPUs). We use `enforce_eager=False` (default) to enable
`torch.compile`.
"""
import os
import time
from pathlib import Path
import numpy as np
import pytest
import torch
from PIL import Image
from tests.helpers.env import DeviceMemoryMonitor
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
PROMPT = "a photo of a cat sitting on a laptop keyboard"
def _get_zimage_model() -> str:
# Allow overriding the model for local/offline environments.
# Can be either a HuggingFace repo id or a local path.
return os.environ.get("VLLM_TEST_ZIMAGE_MODEL", "Tongyi-MAI/Z-Image-Turbo")
def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor:
"""Convert PIL image to float32 RGB tensor in [0, 1] with shape [H, W, 3]."""
arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0
return torch.from_numpy(arr)
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
"""Return (mean_abs_diff, p99_abs_diff) over RGB pixels in [0, 1]."""
ta = _pil_to_float_rgb_tensor(a)
tb = _pil_to_float_rgb_tensor(b)
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
abs_diff = torch.abs(ta - tb)
p99_abs_diff = torch.quantile(abs_diff.flatten(), 0.99).item()
return abs_diff.mean().item(), p99_abs_diff
def _extract_single_image(outputs) -> Image.Image:
first_output = outputs[0]
assert first_output.final_output_type == "image"
if not hasattr(first_output, "request_output") or not first_output.request_output:
raise ValueError("No request_output found in OmniRequestOutput")
req_out = first_output.request_output
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
raise ValueError("Invalid request_output structure or missing 'images' key")
images = req_out.images
if images is None or len(images) != 1:
raise ValueError(f"Expected 1 image, got {0 if images is None else len(images)}")
return images[0]
def _run_zimage_generate(
*,
tp_size: int,
height: int,
width: int,
num_inference_steps: int,
seed: int,
enforce_eager: bool,
vae_use_tiling: bool = False,
vae_patch_parallel_size: int = 1,
num_requests: int = 4,
) -> tuple[Image.Image, float, float]:
if num_requests < 2:
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
current_omni_platform.empty_cache()
device_index = current_omni_platform.current_device()
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
monitor.start()
try:
# Each run needs a distinct DiffusionParallelConfig; use OmniRunner per call (not the
# parametrized omni_runner fixture, which is fixed per module).
with OmniRunner(
_get_zimage_model(),
parallel_config=DiffusionParallelConfig(
tensor_parallel_size=tp_size,
vae_patch_parallel_size=vae_patch_parallel_size,
),
enforce_eager=enforce_eager,
vae_use_tiling=vae_use_tiling,
) as runner:
# NOTE: Omni closes itself when a generate() call is exhausted.
# To avoid measuring teardown time (process shutdown, memory cleanup),
# we measure the latency to produce *subsequent* outputs within a single
# generator run.
#
# This also serves as a warmup: the first output may include extra
# compilation/caching overhead, while later outputs are closer to
# steady-state inference.
gen = runner.omni.generate(
[PROMPT] * num_requests,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
seed=seed,
num_outputs_per_prompt=1,
),
py_generator=True,
)
warmup_output = next(gen)
t_prev = time.perf_counter()
per_request_times_s: list[float] = []
last_output = warmup_output
for _ in range(num_requests - 1):
last_output = next(gen)
t_now = time.perf_counter()
per_request_times_s.append(t_now - t_prev)
t_prev = t_now
# Ensure the generator is fully consumed so it can clean up.
for _ in gen:
pass
median_time_s = float(np.median(per_request_times_s))
peak_memory_mb = monitor.peak_used_mb
return _extract_single_image([last_output]), median_time_s, peak_memory_mb
finally:
monitor.stop()
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
def test_zimage_tensor_parallel_tp2(tmp_path: Path):
if current_omni_platform.is_npu():
pytest.skip("Z-Image TP e2e test is only supported on CUDA and ROCm for now.")
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
pytest.skip("Z-Image TP=2 requires >= 2 devices.")
enforce_eager = False
height = 512
width = 512
num_inference_steps = 2
seed = 42
tp1_img, tp1_time_s, tp1_peak_mem = _run_zimage_generate(
tp_size=1,
height=height,
width=width,
num_inference_steps=num_inference_steps,
seed=seed,
enforce_eager=enforce_eager,
)
tp2_img, tp2_time_s, tp2_peak_mem = _run_zimage_generate(
tp_size=2,
height=height,
width=width,
num_inference_steps=num_inference_steps,
seed=seed,
enforce_eager=enforce_eager,
)
tp1_path = tmp_path / "zimage_tp1.png"
tp2_path = tmp_path / "zimage_tp2.png"
tp1_img.save(tp1_path)
tp2_img.save(tp2_path)
assert tp1_img.width == width and tp1_img.height == height
assert tp2_img.width == width and tp2_img.height == height
mean_abs_diff, p99_abs_diff = _diff_metrics(tp1_img, tp2_img)
mean_threshold = 3e-2
p99_threshold = 2.5e-1
print(
"Z-Image TP image diff stats (TP=1 vs TP=2): "
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
f"tp1_img={tp1_path}, tp2_img={tp2_path}"
)
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
)
print(f"Z-Image TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}")
# ROCm is not optimized TP2 can be slower than TP1
if not current_omni_platform.is_rocm():
assert tp2_time_s < tp1_time_s, f"Expected TP=2 to be faster than TP=1 (tp1={tp1_time_s}, tp2={tp2_time_s})"
print(f"Z-Image TP peak memory (MB): tp1_peak_mem={tp1_peak_mem:.2f}, tp2_peak_mem={tp2_peak_mem:.2f}")
assert tp2_peak_mem < tp1_peak_mem, (
f"Expected TP=2 to use less peak memory than TP=1 (tp1={tp1_peak_mem}, tp2={tp2_peak_mem})"
)
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
if current_omni_platform.is_npu():
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA and ROCm for now.")
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 devices.")
enforce_eager = False
# Use a larger image to ensure there are multiple VAE tiles.
height = 1152
width = 1152
num_inference_steps = 2
seed = 42
baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
tp_size=2,
height=height,
width=width,
num_inference_steps=num_inference_steps,
seed=seed,
enforce_eager=enforce_eager,
vae_use_tiling=True,
vae_patch_parallel_size=1,
num_requests=2,
)
pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
tp_size=2,
height=height,
width=width,
num_inference_steps=num_inference_steps,
seed=seed,
enforce_eager=enforce_eager,
vae_use_tiling=True,
vae_patch_parallel_size=2,
num_requests=2,
)
baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
baseline_img.save(baseline_path)
pp2_img.save(pp2_path)
mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img)
mean_threshold = 5e-3
p99_threshold = 1e-1
print(
"Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
f"pp1_img={baseline_path}, pp2_img={pp2_path}"
)
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
)
@@ -35,7 +35,7 @@ def _audiox_server_cases(model: str):
]
@pytest.mark.core_model
@pytest.mark.full_model
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _audiox_server_cases(AUDIOX_TEST_MODEL), indirect=True)
def test_audiox_t2a_online(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
@@ -19,6 +19,8 @@ from tests.helpers.media import load_test_audio_data_url
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
# Official CosyVoice zero-shot prompt audio and its transcript. Vendored under
@@ -64,8 +66,6 @@ tts_async_chunk_server_params = [
]
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_voice_clone_zh_001(omni_server, openai_client) -> None:
@@ -88,8 +88,6 @@ def test_voice_clone_zh_001(omni_server, openai_client) -> None:
openai_client.send_audio_speech_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_async_chunk_server_params, indirect=True)
def test_voice_clone_zh_002(omni_server, openai_client) -> None:
@@ -112,8 +110,6 @@ def test_voice_clone_zh_002(omni_server, openai_client) -> None:
openai_client.send_audio_speech_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_voice_clone_en_001(omni_server, openai_client) -> None:
@@ -7,13 +7,13 @@ Example online tests for Dynin-Omni model.
import base64
import os
from io import BytesIO
from pathlib import Path
import pytest
from vllm.assets.image import ImageAsset
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
@@ -21,7 +21,7 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "snu-aidas/Dynin-Omni"
STAGE_CONFIG = str(Path(__file__).parent.parent / "stage_configs" / "dynin_omni_ci.yaml")
STAGE_CONFIG = get_deploy_config_path("dynin_omni_ci.yaml")
T2I_PROMPT = "A high quality detailed living room interior photo."
T2S_PROMPT = "Please read this sentence naturally: Hello from online serving."
@@ -12,7 +12,12 @@ Coverage:
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
@@ -96,9 +101,10 @@ def _get_diffusion_feature_cases(model: str):
indirect=True,
)
def test_flux2_klein(omni_server: OmniServer, openai_client: OpenAIClientHandler):
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
request_config = {
"model": omni_server.model,
"messages": [{"role": "user", "content": POSITIVE_PROMPT}],
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
@@ -0,0 +1,62 @@
"""
Tests for Flux2 Klein; currently Dev is implemented separately,
but ideally these models will fold together in the future.
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
POSITIVE_PROMPT = "A cat sitting on a windowsill"
NEGATIVE_PROMPT = "blurry, low quality"
# For now, we only test a good configuration for this model to keep the CI
# light, but in the near future may explore using random models to check
# cross-feature compatibility more generally.
def _get_diffusion_feature_cases(model: str):
return [
# FP8 / Hybrid sequence parallelism
pytest.param(
OmniServerParams(
model=model,
),
id="default",
marks=FOUR_CARD_FEATURE_MARKS,
),
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases(
model="black-forest-labs/FLUX.2-klein-4B",
),
indirect=True,
)
def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@@ -0,0 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests of common diffusion feature combinations in online serving mode
for HunyuanVideo-1.5-T2V (480p).
Coverage (H100, since model cannot fit L4):
- CacheDiT + Layerwise CPU offloading (1 GPU)
- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs)
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
)
PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion."
NEGATIVE_PROMPT = "low quality, blurry, distorted"
MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v"
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return L4 diffusion feature cases for HunyuanVideo-1.5.
Designed for 2x H100 environment per issue #1832.
"""
return [
# (1 GPU) CacheDiT + Layerwise CPU offloading
pytest.param(
OmniServerParams(
model=model,
),
id="default",
marks=SINGLE_CARD_MARKS,
),
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases(MODEL),
indirect=True,
)
def test_text_to_video_001(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
):
"""L4 diffusion feature coverage for HunyuanVideo-1.5-T2V on H100."""
form_data = {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": 480,
"width": 640,
"num_frames": 5,
"num_inference_steps": 2,
"guidance_scale": 6.0,
"seed": 42,
}
request_config = {
"model": omni_server.model,
"form_data": form_data,
}
openai_client.send_video_diffusion_request(request_config)
@@ -1,278 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E online serving test for Qwen-Image-Edit-2509 multi-image input.
"""
import base64
import os
import signal
import socket
import subprocess
import sys
import threading
import time
from io import BytesIO
from typing import Any
import openai
import pytest
import requests
from PIL import Image
from vllm.assets.image import ImageAsset
from vllm.utils.network_utils import get_open_port
from tests.helpers.mark import hardware_test
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
# Increase timeout for downloading assets from S3 (default 5s is too short for CI)
os.environ.setdefault("VLLM_IMAGE_FETCH_TIMEOUT", "60")
models = ["Qwen/Qwen-Image-Edit-2509"]
test_params = models
t2i_models = ["Tongyi-MAI/Z-Image-Turbo"]
class OmniServer:
"""Omniserver for vLLM-Omni tests."""
def __init__(
self,
model: str,
serve_args: list[str],
*,
env_dict: dict[str, str] | None = None,
) -> None:
self.model = model
self.serve_args = serve_args
self.env_dict = env_dict
self.proc: subprocess.Popen | None = None
self.host = "127.0.0.1"
self.port = get_open_port()
def _start_server(self) -> None:
"""Start the vLLM-Omni server subprocess."""
env = os.environ.copy()
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
if self.env_dict is not None:
env.update(self.env_dict)
cmd = [
sys.executable,
"-m",
"vllm_omni.entrypoints.cli.main",
"serve",
self.model,
"--omni",
"--host",
self.host,
"--port",
str(self.port),
] + self.serve_args
print(f"Launching OmniServer with: {' '.join(cmd)}")
self.proc = subprocess.Popen(
cmd,
env=env,
cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # Set working directory to vllm-omni root
start_new_session=True,
)
# Wait for server to be ready
max_wait = 600 # 10 minutes
start_time = time.time()
while time.time() - start_time < max_wait:
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.settimeout(1)
result = sock.connect_ex((self.host, self.port))
if result == 0:
print(f"Server ready on {self.host}:{self.port}")
return
except Exception:
pass
time.sleep(2)
raise RuntimeError(f"Server failed to start within {max_wait} seconds")
def __enter__(self):
self._start_server()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.proc:
try:
os.killpg(self.proc.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
self.proc.wait(timeout=30)
except subprocess.TimeoutExpired:
try:
os.killpg(self.proc.pid, signal.SIGKILL)
except ProcessLookupError:
pass
self.proc.wait()
@pytest.fixture
def omni_server(request):
"""Start vLLM-Omni server as a subprocess with actual model weights."""
model = request.param
with OmniServer(model, ["--num-gpus", "1"]) as server:
yield server
@pytest.fixture
def client(omni_server):
"""OpenAI client for the running vLLM-Omni server."""
return openai.OpenAI(
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
api_key="EMPTY",
)
@pytest.fixture(scope="session")
def base64_encoded_images() -> list[str]:
"""Base64 encoded PNG images for testing."""
images = [
ImageAsset("cherry_blossom").pil_image.convert("RGB"),
ImageAsset("stop_sign").pil_image.convert("RGB"),
]
encoded: list[str] = []
for img in images:
with BytesIO() as buffer:
img.save(buffer, format="PNG")
encoded.append(base64.b64encode(buffer.getvalue()).decode("utf-8"))
return encoded
def dummy_messages_from_image_data(
image_data_urls: list[str],
content_text: str = "Combine these two images into one scene.",
):
"""Create messages with image data URLs for OpenAI API."""
content = [{"type": "text", "text": content_text}]
for image_url in image_data_urls:
content.append({"type": "image_url", "image_url": {"url": image_url}})
return [{"role": "user", "content": content}]
def _extract_image_data_url(message_content) -> str:
assert isinstance(message_content, list) and len(message_content) >= 1
content_part = message_content[0]
if isinstance(content_part, dict):
image_url = content_part.get("image_url", {}).get("url", "")
else:
image_url_obj = getattr(content_part, "image_url", None)
if isinstance(image_url_obj, dict):
image_url = image_url_obj.get("url", "")
else:
image_url = getattr(image_url_obj, "url", "")
assert isinstance(image_url, str) and image_url
return image_url
def _decode_data_url_to_image_bytes(data_url: str) -> bytes:
assert data_url.startswith("data:image")
_, b64_data = data_url.split(",", 1)
return base64.b64decode(b64_data)
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_i2i_multi_image_input_qwen_image_edit_2509(
omni_server,
base64_encoded_images: list[str],
) -> None:
"""Test multi-image input editing via OpenAI API with concurrent requests."""
image_data_urls = [f"data:image/png;base64,{img}" for img in base64_encoded_images]
messages = dummy_messages_from_image_data(image_data_urls)
barrier = threading.Barrier(2)
results: list[tuple[int, int]] = []
def _call_chat(width: int, height: int) -> None:
client = openai.OpenAI(
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
api_key="EMPTY",
)
barrier.wait()
chat_completion = client.chat.completions.create(
model=omni_server.model,
messages=messages,
extra_body={
"height": height,
"width": width,
"num_inference_steps": 2,
"guidance_scale": 0.0,
"seed": 42,
},
)
assert len(chat_completion.choices) == 1
choice = chat_completion.choices[0]
assert choice.finish_reason == "stop"
assert choice.message.role == "assistant"
image_data_url = _extract_image_data_url(choice.message.content)
image_bytes = _decode_data_url_to_image_bytes(image_data_url)
img = Image.open(BytesIO(image_bytes))
img.load()
results.append(img.size)
threads = [
threading.Thread(target=_call_chat, args=(1248, 832)),
threading.Thread(target=_call_chat, args=(1024, 768)),
]
for t in threads:
t.start()
for t in threads:
t.join()
# TODO @ZJY
# assert (1248, 832) in results
# assert (1024, 768) in results
@pytest.mark.parametrize("omni_server", t2i_models, indirect=True)
def test_t2i_concurrent_requests_different_sizes(omni_server) -> None:
"""Test /v1/images/generations concurrent requests with different sizes."""
base_url = f"http://{omni_server.host}:{omni_server.port}"
url = f"{base_url}/v1/images/generations"
barrier = threading.Barrier(2)
results: list[tuple[int, int]] = []
def _call_generate(size: str) -> None:
payload: dict[str, Any] = {
"prompt": "cute cat playing with a ball",
"n": 1,
"size": size,
"response_format": "b64_json",
"num_inference_steps": 2,
}
barrier.wait()
response = requests.post(url, json=payload, timeout=120)
assert response.status_code == 200
data = response.json()
image_b64 = data["data"][0]["b64_json"]
image_bytes = base64.b64decode(image_b64)
img = Image.open(BytesIO(image_bytes))
img.load()
results.append(img.size)
threads = [
threading.Thread(target=_call_generate, args=("512x512",)),
threading.Thread(target=_call_generate, args=("768x512",)),
]
for t in threads:
t.start()
for t in threads:
t.join()
assert (512, 512) in results
assert (768, 512) in results
@@ -1,208 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E online serving test for /v1/images/generations with per-request LoRA.
This validates:
- The API server accepts a per-request `lora` object in the Images API payload.
- LoRA can be switched per request (adapter A -> adapter B -> no LoRA).
- Output correctness is asserted using a small image slice with tolerance.
"""
import base64
import json
import os
from io import BytesIO
from pathlib import Path
import numpy as np
import pytest
import requests
import torch
from PIL import Image
from safetensors.torch import save_file
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
MODEL = "Tongyi-MAI/Z-Image-Turbo"
DIFFUSION_INIT_TIMEOUT_S = 900
PROMPT = "a photo of a cat sitting on a laptop keyboard"
SIZE = "256x256"
SEED = 42
@pytest.fixture(scope="module")
def omni_server():
with OmniServer(
MODEL,
[
"--num-gpus",
"1",
"--stage-init-timeout",
str(DIFFUSION_INIT_TIMEOUT_S),
"--init-timeout",
str(DIFFUSION_INIT_TIMEOUT_S),
],
) as server:
yield server
def _write_zimage_lora(adapter_dir: Path, *, q_scale: float = 0.0, k_scale: float = 0.0, v_scale: float = 0.0):
adapter_dir.mkdir(parents=True, exist_ok=True)
# Z-Image transformer uses dim=3840 by default.
dim = 3840
module_name = "transformer.layers.0.attention.to_qkv"
rank = 1
lora_a = torch.zeros((rank, dim), dtype=torch.float32)
lora_a[0, 0] = 1.0
# QKVParallelLinear packs (Q, K, V) => out dim is 3 * dim (tp=1).
lora_b = torch.zeros((3 * dim, rank), dtype=torch.float32)
if q_scale:
lora_b[:dim, 0] = q_scale
if k_scale:
lora_b[dim : 2 * dim, 0] = k_scale
if v_scale:
lora_b[2 * dim :, 0] = v_scale
save_file(
{
f"base_model.model.{module_name}.lora_A.weight": lora_a,
f"base_model.model.{module_name}.lora_B.weight": lora_b,
},
str(adapter_dir / "adapter_model.safetensors"),
)
(adapter_dir / "adapter_config.json").write_text(
json.dumps(
{
"r": rank,
"lora_alpha": rank,
"target_modules": [module_name],
}
),
encoding="utf-8",
)
def _post_images(server: OmniServer, payload: dict) -> Image.Image:
url = f"http://{server.host}:{server.port}/v1/images/generations"
resp = requests.post(url, json=payload, headers={"Authorization": "Bearer EMPTY"}, timeout=900)
resp.raise_for_status()
data = resp.json()
b64 = data["data"][0]["b64_json"]
img_bytes = base64.b64decode(b64)
img = Image.open(BytesIO(img_bytes))
img.load()
return img.convert("RGB")
def _image_blue_tail_slice(img: Image.Image) -> np.ndarray:
arr = np.asarray(img, dtype=np.uint8)
assert arr.ndim == 3 and arr.shape[-1] == 3
tail = arr[-3:, -3:, -1].astype(np.float32)
assert tail.shape == (3, 3)
return tail
def _slice_diff_stats(actual: np.ndarray, expected: np.ndarray) -> tuple[float, float]:
diff = np.abs(actual - expected)
return float(diff.max()), float(diff.mean())
def _assert_slice_close(
actual: np.ndarray,
expected: np.ndarray,
*,
label: str,
base_max: float,
base_mean: float,
) -> None:
assert actual.shape == (3, 3)
assert expected.shape == (3, 3)
max_diff, mean_diff = _slice_diff_stats(actual, expected)
# NOTE: Different attention backends / torch.compile can introduce small
# floating-point drift that shows up as a few LSBs in uint8 pixels. Keep
# the reset check tolerant but bounded to avoid flaky CI.
max_thresh = max(10.0, base_max + 4.0)
mean_thresh = max(6.0, base_mean + 4.0)
assert max_diff <= max_thresh and mean_diff <= mean_thresh, (
f"{label} slice mismatch (max={max_diff:.1f} > {max_thresh:.1f} or "
f"mean={mean_diff:.1f} > {mean_thresh:.1f}): {actual.tolist()}"
)
def _assert_slice_diff(actual: np.ndarray, baseline: np.ndarray, *, label: str) -> None:
assert actual.shape == (3, 3)
assert baseline.shape == (3, 3)
diff = np.abs(actual - baseline).mean()
assert diff > 0.1, f"{label} slice diff too small: {diff} ({actual.tolist()} vs {baseline.tolist()})"
def _basic_payload() -> dict:
return {
"prompt": PROMPT,
"n": 1,
"size": SIZE,
"num_inference_steps": 2,
"guidance_scale": 0.0,
"seed": SEED,
}
@pytest.mark.advanced_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
def test_images_generations_per_request_lora_switching(omni_server: OmniServer, tmp_path: Path) -> None:
# Base generation.
base_img = _post_images(omni_server, _basic_payload())
base_slice = _image_blue_tail_slice(base_img)
base_ref_img = _post_images(omni_server, _basic_payload())
base_ref_slice = _image_blue_tail_slice(base_ref_img)
base_ref_max, base_ref_mean = _slice_diff_stats(base_ref_slice, base_slice)
# Adapter A: apply delta to V slice only.
lora_a_dir = tmp_path / "zimage_lora_a"
_write_zimage_lora(lora_a_dir, v_scale=8.0)
payload_a = _basic_payload()
payload_a["lora"] = {"name": "a", "path": str(lora_a_dir), "scale": 64.0}
img_a = _post_images(omni_server, payload_a)
a_slice = _image_blue_tail_slice(img_a)
_assert_slice_diff(a_slice, base_slice, label="lora_a_vs_base")
a_vs_base = float(np.abs(a_slice - base_slice).mean())
# Adapter B: apply delta to K slice only (should differ from adapter A).
lora_b_dir = tmp_path / "zimage_lora_b"
_write_zimage_lora(lora_b_dir, k_scale=4.0)
payload_b = _basic_payload()
payload_b["lora"] = {"name": "b", "path": str(lora_b_dir), "scale": 64.0}
img_b = _post_images(omni_server, payload_b)
b_slice = _image_blue_tail_slice(img_b)
_assert_slice_diff(b_slice, base_slice, label="lora_b_vs_base")
_assert_slice_diff(b_slice, a_slice, label="lora_b_vs_lora_a")
b_vs_base = float(np.abs(b_slice - base_slice).mean())
b_vs_a = float(np.abs(b_slice - a_slice).mean())
# Ensure switching back to no-LoRA restores the base output.
base_img_2 = _post_images(omni_server, _basic_payload())
base_slice_2 = _image_blue_tail_slice(base_img_2)
_, base_reset_mean = _slice_diff_stats(base_slice_2, base_slice)
_assert_slice_close(
base_slice_2,
base_slice,
label="base_after_reset",
base_max=base_ref_max,
base_mean=base_ref_mean,
)
# Ensure LoRA effects are clearly above the baseline drift.
min_delta = max(base_reset_mean + 1.0, 1.5)
assert a_vs_base > min_delta, f"lora_a_vs_base drift too small: {a_vs_base} <= {min_delta}"
assert b_vs_base > min_delta, f"lora_b_vs_base drift too small: {b_vs_base} <= {min_delta}"
assert b_vs_a > min_delta, f"lora_b_vs_lora_a drift too small: {b_vs_a} <= {min_delta}"
@@ -0,0 +1,69 @@
"""
Recommended tests of diffusion features that are available in online serving mode
and are supported by the following model:
- LongCat-Image: text-to-image with single prompt input
Coverage:
- CPU offloading (model-level sequential offload via --enable-cpu-offload)
- Cache-DiT
- SP (Ulysses)
This validates:
- Successful image generation at the expected 768x1344 resolution with recommended feature combinations
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
TEXT_TO_IMAGE_PROMPT = (
"A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed."
)
NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return diffusion feature cases for LongCat-Image."""
return [
pytest.param(
OmniServerParams(
model=model,
),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
)
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("meituan-longcat/LongCat-Image"),
indirect=True,
)
def test_longcat_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test the recommended feature combinations for LongCat-Image."""
messages = dummy_messages_from_mix_data(content_text=TEXT_TO_IMAGE_PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 768,
"width": 1344,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"guidance_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@@ -0,0 +1,69 @@
"""
Recommended tests of diffusion features that are available in online serving mode
and are supported by the following model:
- LongCat-Image-Edit: image-to-image edit with a single image + single edit prompt input
Coverage:
- CPU offloading (model-level sequential offload via --enable-cpu-offload)
- Cache-DiT
- SP (Ulysses)
This validates:
- Successful image generation at the expected 1024x1024 resolution with recommended feature combinations
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting."
NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return diffusion feature cases for LongCat-Image-Edit."""
return [
pytest.param(
OmniServerParams(
model=model,
),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
)
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("meituan-longcat/LongCat-Image-Edit"),
indirect=True,
)
def test_longcat_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test the recommended feature combinations for LongCat-Image-Edit."""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 1024,
"width": 1024,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"guidance_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
+3 -2
View File
@@ -53,7 +53,7 @@ try:
OmniServerParams(
model=model,
stage_config_path=stage_config,
server_args=["--chat-template", CHAT_TEMPLATE_PATH, "--load-format", "dummy"],
server_args=["--chat-template", CHAT_TEMPLATE_PATH],
)
for model in models
for stage_config in stage_configs
@@ -68,7 +68,7 @@ except Exception as exc:
def get_prompt(prompt_type="text_only"):
prompts = {
"text_only": "What is the capital of China? Answer in 20 words.",
"audio": "What is recited in the audio?",
"audio": "What one English word is repeated in the audio?",
}
return prompts.get(prompt_type, prompts["text_only"])
@@ -104,6 +104,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
"model": omni_server.model,
"messages": messages,
"stream": True,
"sampling_params_list": [{"max_tokens": 64}, {"max_tokens": 64}],
"key_words": {
"audio": ["test"],
},
@@ -21,6 +21,12 @@ from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
pytestmark = [pytest.mark.omni, pytest.mark.full_model]
_SKIP_NEED_4_H100_NOT_CI = pytest.mark.skip(
reason="Requires 4x H100 GPUs; skipped in CI for now.",
)
models = ["Jonathan1909/Ming-flash-omni-2.0"]
# Use thinker-only topology to test text outputs
@@ -62,8 +68,7 @@ def get_max_batch_size(size_type="few"):
return batch_sizes.get(size_type, 5)
@pytest.mark.core_model
@pytest.mark.omni
@_SKIP_NEED_4_H100_NOT_CI
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_text_001(omni_server, openai_client) -> None:
@@ -89,8 +94,7 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@_SKIP_NEED_4_H100_NOT_CI
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_text_to_text_stream_001(omni_server, openai_client) -> None:
@@ -116,8 +120,7 @@ def test_text_to_text_stream_001(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
@pytest.mark.core_model
@pytest.mark.omni
@_SKIP_NEED_4_H100_NOT_CI
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_image_to_text_001(omni_server, openai_client) -> None:
@@ -144,8 +147,7 @@ def test_image_to_text_001(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@_SKIP_NEED_4_H100_NOT_CI
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_audio_to_text_001(omni_server, openai_client) -> None:
@@ -172,8 +174,7 @@ def test_audio_to_text_001(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config)
@pytest.mark.core_model
@pytest.mark.omni
@_SKIP_NEED_4_H100_NOT_CI
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_video_to_text_001(omni_server, openai_client) -> None:
@@ -200,9 +201,7 @@ def test_video_to_text_001(omni_server, openai_client) -> None:
openai_client.send_omni_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.omni
@_SKIP_NEED_4_H100_NOT_CI
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
def test_mix_to_text_001(omni_server, openai_client) -> None:
@@ -25,6 +25,13 @@ from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
# ``omni`` for all tests; 001/002 also get ``full_model`` via ``TestMossTtsNanoFull`` (003 is core+advanced only, no full_model).
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
_SKIP_ISSUE_3168 = pytest.mark.skip(
reason="https://github.com/vllm-project/vllm-omni/issues/3168",
)
MODEL = "OpenMOSS-Team/MOSS-TTS-Nano"
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS-Nano/main/assets/audio/zh_1.wav"
REF_AUDIO_TRANSCRIPT = "欢迎关注模思智能、上海创智学院与复旦大学自然语言处理实验室。"
@@ -79,8 +86,6 @@ tts_server_params = [
]
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> None:
@@ -103,8 +108,6 @@ def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> No
openai_client.send_audio_speech_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_002(omni_server, openai_client, ref_audio_data_url) -> None:
@@ -136,8 +139,6 @@ def test_text_to_audio_002(omni_server, openai_client, ref_audio_data_url) -> No
openai_client.send_audio_speech_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_003(omni_server, openai_client, ref_audio_data_url) -> None:
@@ -160,8 +161,6 @@ def test_text_to_audio_003(omni_server, openai_client, ref_audio_data_url) -> No
openai_client.send_audio_speech_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_004_ref_text_ignored(omni_server, openai_client, ref_audio_data_url) -> None:
-201
View File
@@ -1,201 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E Online tests for OmniVoice TTS model via /v1/audio/speech endpoint.
Tests verify that the OmniVoice model generates valid audio when
accessed through the standard OpenAI-compatible speech API.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import httpx
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import generate_synthetic_audio
from tests.helpers.runtime import OmniServerParams
try:
from transformers import HiggsAudioV2TokenizerModel # noqa: F401
_HAS_VOICE_CLONE = True
except ImportError:
_HAS_VOICE_CLONE = False
MODEL = "k2-fsa/OmniVoice"
STAGE_CONFIG = str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "omnivoice.yaml"
)
EXTRA_ARGS = [
"--trust-remote-code",
"--disable-log-stats",
]
TEST_PARAMS = [
OmniServerParams(
model=MODEL,
stage_config_path=STAGE_CONFIG,
server_args=EXTRA_ARGS,
)
]
MIN_AUDIO_BYTES = 5000
def _get_ref_audio_b64() -> str:
"""Generate synthetic speech for reference audio.
Returns:
Base64 data URL string (data:audio/wav;base64,...)
"""
audio_data = generate_synthetic_audio(duration=2, num_channels=1, sample_rate=24000)
return f"data:audio/wav;base64,{audio_data['base64']}"
def make_speech_request(
host: str,
port: int,
text: str,
timeout: float = 180.0,
) -> httpx.Response:
"""Make a request to the /v1/audio/speech endpoint for OmniVoice."""
url = f"http://{host}:{port}/v1/audio/speech"
payload = {"input": text}
with httpx.Client(timeout=timeout) as client:
return client.post(url, json=payload)
def verify_wav_audio(content: bytes) -> bool:
"""Verify that content is valid WAV audio data."""
if len(content) < 44:
return False
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestOmniVoiceTTS:
"""E2E tests for OmniVoice TTS model."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speech_auto_voice(self, omni_server) -> None:
"""Test auto voice TTS generation (text only, no reference audio)."""
response = make_speech_request(
host=omni_server.host,
port=omni_server.port,
text="Hello, this is a test of the OmniVoice text to speech system.",
)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
assert len(response.content) > MIN_AUDIO_BYTES, (
f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
)
def make_voice_clone_request(
host: str,
port: int,
text: str,
ref_audio_b64: str,
ref_text: str | None = None,
timeout: float = 180.0,
) -> httpx.Response:
"""Make a voice cloning request to the /v1/audio/speech endpoint.
Args:
host: Server host
port: Server port
text: Text to synthesize
ref_audio_b64: Base64-encoded reference audio data URL
ref_text: Optional transcript of reference audio
timeout: Request timeout in seconds
Returns:
httpx.Response object
"""
url = f"http://{host}:{port}/v1/audio/speech"
payload = {
"input": text,
"ref_audio": ref_audio_b64,
}
if ref_text:
payload["ref_text"] = ref_text
with httpx.Client(timeout=timeout) as client:
return client.post(url, json=payload)
@pytest.mark.skipif(not _HAS_VOICE_CLONE, reason="Voice cloning requires transformers>=5.3.0")
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestOmniVoiceVoiceCloning:
"""E2E tests for OmniVoice voice cloning functionality."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voice_clone_ref_audio_only(self, omni_server) -> None:
"""Test voice cloning with ref_audio only (x_vector mode)."""
ref_audio_b64 = _get_ref_audio_b64()
response = make_voice_clone_request(
host=omni_server.host,
port=omni_server.port,
text="Hello, this is a voice cloning test.",
ref_audio_b64=ref_audio_b64,
)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
assert len(response.content) > MIN_AUDIO_BYTES, (
f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voice_clone_ref_audio_and_text(self, omni_server) -> None:
"""Test voice cloning with ref_audio and ref_text (in-context mode)."""
ref_audio_b64 = _get_ref_audio_b64()
ref_text = "This is the reference transcript."
response = make_voice_clone_request(
host=omni_server.host,
port=omni_server.port,
text="Hello, this is a voice cloning test with in-context learning.",
ref_audio_b64=ref_audio_b64,
ref_text=ref_text,
)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
assert len(response.content) > MIN_AUDIO_BYTES, (
f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voice_clone_invalid_ref_audio_format(self, omni_server) -> None:
"""Test that invalid ref_audio format returns a clear error."""
response = make_voice_clone_request(
host=omni_server.host,
port=omni_server.port,
text="This should fail with invalid ref_audio.",
ref_audio_b64="not_a_valid_uri",
)
assert response.status_code in (400, 422), (
f"Expected 400/422 for invalid ref_audio format, got {response.status_code}"
)
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E Online tests for OmniVoice TTS model via /v1/audio/speech endpoint.
Tests verify that the OmniVoice model generates valid audio when
accessed through the standard OpenAI-compatible speech API.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import load_test_audio_data_url
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
try:
from transformers import HiggsAudioV2TokenizerModel # noqa: F401
_HAS_VOICE_CLONE = True
except ImportError:
_HAS_VOICE_CLONE = False
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
MODEL = "k2-fsa/OmniVoice"
STAGE_CONFIG = get_deploy_config_path("omnivoice.yaml")
EXTRA_ARGS = [
"--trust-remote-code",
"--disable-log-stats",
]
TEST_PARAMS = [
OmniServerParams(
model=MODEL,
stage_config_path=STAGE_CONFIG,
server_args=EXTRA_ARGS,
)
]
# Lower this in ``request_config`` via ``min_audio_bytes`` if a run produces legitimately short WAVs.
_DEFAULT_MIN_AUDIO_BYTES = 5000
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
def get_prompt(prompt_type="text"):
prompts = {
"text": "The weather is nice today, perfect for a walk in the park.",
}
return prompts.get(prompt_type, prompts["text"])
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestOmniVoiceTTS:
"""E2E tests for OmniVoice TTS model."""
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speech_auto_voice(self, omni_server, openai_client) -> None:
"""Test auto voice TTS generation (text only, no reference audio)."""
request_config = {
"model": omni_server.model,
"input": get_prompt("text"),
"response_format": "wav",
"timeout": 180.0,
"min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES,
}
openai_client.send_audio_speech_request(request_config)
@pytest.mark.skipif(not _HAS_VOICE_CLONE, reason="Voice cloning requires transformers>=5.3.0")
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestOmniVoiceVoiceCloning:
"""E2E tests for OmniVoice voice cloning functionality."""
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voice_clone_ref_audio_only(self, omni_server, openai_client) -> None:
"""Test voice cloning with ref_audio only (x_vector mode)."""
request_config = {
"model": omni_server.model,
"input": get_prompt("text"),
"ref_audio": REF_AUDIO_URL,
"response_format": "wav",
"timeout": 180.0,
"min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES,
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voice_clone_ref_audio_and_text(self, omni_server, openai_client) -> None:
"""Test voice cloning with ref_audio and ref_text (in-context mode)."""
request_config = {
"model": omni_server.model,
"input": get_prompt("text"),
"ref_audio": REF_AUDIO_URL,
"ref_text": REF_TEXT,
"response_format": "wav",
"timeout": 180.0,
"min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES,
}
openai_client.send_audio_speech_request(request_config)
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voice_clone_invalid_ref_audio_format(self, omni_server, openai_client) -> None:
"""Test that invalid ref_audio format returns a clear error."""
request_config = {
"model": omni_server.model,
"input": get_prompt("text"),
"ref_audio": "not_a_valid_uri",
"response_format": "wav",
"timeout": 180.0,
"status_code": (400, 422),
}
openai_client.send_audio_speech_request(request_config)
@@ -87,7 +87,7 @@ def _assert_audio_outputs(responses: list[OmniResponse], *, expect_text: bool) -
)
@pytest.mark.core_model
@pytest.mark.full_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
@@ -104,7 +104,7 @@ def test_text_only_batch_uses_multi_replica_talker(omni_server, openai_client) -
_assert_text_outputs(responses)
@pytest.mark.core_model
@pytest.mark.full_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
@@ -121,7 +121,7 @@ def test_text_to_audio_stream_batch_uses_multi_replica_vocoder(omni_server, open
_assert_audio_outputs(responses, expect_text=False)
@pytest.mark.core_model
@pytest.mark.full_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
@@ -20,6 +20,7 @@ from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 180.0
# Vendored under tests/assets/qwen3_tts/clone_2.wav so the server does not need
# to fetch the reference audio over HTTPS at request time (CI runners in some
@@ -56,7 +57,7 @@ tts_server_params = [
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_001(omni_server, openai_client) -> None:
@@ -73,6 +74,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
"model": omni_server.model,
"input": get_prompt(),
"stream": False,
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
"response_format": "wav",
"task_type": "Base",
"voice": "clone",
@@ -83,7 +85,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_002(omni_server, openai_client) -> None:
@@ -99,6 +101,7 @@ def test_text_to_audio_002(omni_server, openai_client) -> None:
"model": omni_server.model,
"input": get_prompt(),
"stream": True,
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
"response_format": "wav",
"task_type": "Base",
"voice": "clone",
@@ -16,7 +16,7 @@ from tests.helpers.media import load_test_audio_data_url
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -1,433 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E tests for the /v1/audio/speech/batch endpoint.
Validates bulk synthesis via the batch API with actual model inference.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import base64
import shutil
import struct
import tempfile
import time
from pathlib import Path
import httpx
import pytest
import yaml
from tests.helpers.mark import hardware_test
from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_text
from tests.helpers.runtime import OmniServer
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
STAGE_INIT_TIMEOUT_S = 120
def get_stage_config(name: str = "qwen3_tts.yaml") -> str:
"""Resolve a deploy config path under vllm_omni/deploy/."""
return get_deploy_config_path(name)
@pytest.fixture(scope="module")
def omni_server():
"""Start vLLM-Omni server with CustomVoice model."""
stage_config_path = get_stage_config()
with OmniServer(
MODEL,
[
"--stage-configs-path",
stage_config_path,
"--stage-init-timeout",
str(STAGE_INIT_TIMEOUT_S),
"--trust-remote-code",
"--enforce-eager",
"--disable-log-stats",
],
) as server:
yield server
def make_batch_request(
host: str,
port: int,
items: list[dict],
voice: str | None = "vivian",
language: str | None = "English",
response_format: str = "wav",
timeout: float = 300.0,
**kwargs,
) -> httpx.Response:
"""Make a request to the /v1/audio/speech/batch endpoint."""
url = f"http://{host}:{port}/v1/audio/speech/batch"
payload: dict = {
"items": items,
"response_format": response_format,
}
if voice is not None:
payload["voice"] = voice
if language is not None:
payload["language"] = language
payload.update(kwargs)
with httpx.Client(timeout=timeout) as client:
return client.post(url, json=payload)
def verify_wav_bytes(audio_bytes: bytes) -> bool:
"""Verify decoded audio bytes form a valid WAV."""
if len(audio_bytes) < 44:
return False
return audio_bytes[:4] == b"RIFF" and audio_bytes[8:12] == b"WAVE"
def assert_not_silence(pcm_bytes: bytes):
"""Assert PCM16 samples are not all identical."""
samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
unique = set(samples)
assert len(unique) > 1, f"All-silence: {len(samples)} samples, unique={unique}"
MIN_AUDIO_BYTES = 10000
class TestSpeechBatchE2E:
"""E2E tests for /v1/audio/speech/batch endpoint."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_basic_two_items(self, omni_server) -> None:
"""Batch with two items returns two successful base64-encoded results."""
items = [
{"input": "Hello, how are you today?"},
{"input": "The weather is nice outside."},
]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
assert data["total"] == 2
assert data["succeeded"] == 2
assert data["failed"] == 0
assert data["id"].startswith("speech-batch-")
assert len(data["results"]) == 2
for i, result in enumerate(data["results"]):
assert result["index"] == i
assert result["status"] == "success"
assert result["audio_data"] is not None
assert result["error"] is None
audio_bytes = base64.b64decode(result["audio_data"])
assert verify_wav_bytes(audio_bytes), f"Item {i}: invalid WAV"
assert len(audio_bytes) > MIN_AUDIO_BYTES, f"Item {i}: audio too small ({len(audio_bytes)} bytes)"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_single_item(self, omni_server) -> None:
"""Batch with a single item works correctly."""
items = [{"input": "Single item batch test."}]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
assert data["total"] == 1
assert data["succeeded"] == 1
assert data["failed"] == 0
assert len(data["results"]) == 1
assert data["results"][0]["status"] == "success"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_per_item_voice_override(self, omni_server) -> None:
"""Per-item voice overrides the batch-level default."""
items = [
{"input": "First item with default voice."},
{"input": "Second item with ryan voice.", "voice": "ryan"},
]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
voice="vivian",
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
assert data["total"] == 2
assert data["succeeded"] == 2
# Both items should produce valid audio
for result in data["results"]:
assert result["status"] == "success"
audio_bytes = base64.b64decode(result["audio_data"])
assert verify_wav_bytes(audio_bytes)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_multiple_languages(self, omni_server) -> None:
"""Batch items with different languages per item."""
items = [
{"input": "Hello, nice to meet you.", "language": "English"},
{"input": "你好,很高兴认识你。", "language": "Chinese"},
]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
assert data["succeeded"] == 2
for result in data["results"]:
assert result["status"] == "success"
audio_bytes = base64.b64decode(result["audio_data"])
assert len(audio_bytes) > MIN_AUDIO_BYTES
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_whisper_transcription(self, omni_server) -> None:
"""Whisper transcription of batch output matches input text."""
input_text = "Good morning, welcome to the speech synthesis test."
items = [{"input": input_text}]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
assert data["succeeded"] == 1
audio_bytes = base64.b64decode(data["results"][0]["audio_data"])
assert verify_wav_bytes(audio_bytes)
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
f.write(audio_bytes)
wav_path = f.name
try:
transcript = convert_audio_file_to_text(wav_path)
print(f"Whisper transcript: {transcript}")
assert len(transcript.strip()) > 0, "Empty transcript — likely silence"
similarity = cosine_similarity_text(transcript.lower(), input_text.lower())
print(f"Cosine similarity: {similarity:.3f}")
assert similarity > 0.7, f"Transcript mismatch: similarity={similarity:.2f}, transcript='{transcript}'"
finally:
os.unlink(wav_path)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_result_indices_ordered(self, omni_server) -> None:
"""Result indices match the input item order."""
items = [
{"input": "First sentence."},
{"input": "Second sentence."},
{"input": "Third sentence."},
]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
indices = [r["index"] for r in data["results"]]
assert indices == [0, 1, 2]
class TestSpeechBatchValidation:
"""Validation / error-handling tests for the batch endpoint."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_empty_items_rejected(self, omni_server) -> None:
"""Empty items list returns a 4xx error."""
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech/batch"
payload = {"items": [], "voice": "vivian"}
with httpx.Client(timeout=30.0) as client:
response = client.post(url, json=payload)
assert response.status_code in (400, 422)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch_exceeds_max_items(self, omni_server) -> None:
"""Batch exceeding 32 items returns 400 error."""
items = [{"input": f"Item {i}"} for i in range(33)]
response = make_batch_request(
host=omni_server.host,
port=omni_server.port,
items=items,
)
assert response.status_code == 400
def _make_batch2_stage_config() -> str:
"""Create a temporary stage config with max_batch_size=2 for both stages."""
src = Path(get_stage_config("qwen3_tts_batch.yaml"))
with open(src) as f:
cfg = yaml.safe_load(f)
for stage in cfg["stage_args"]:
stage["runtime"]["max_batch_size"] = 2
tmp_dir = tempfile.mkdtemp(prefix="tts_batch2_")
dst = Path(tmp_dir) / "qwen3_tts_batch2.yaml"
with open(dst, "w") as f:
yaml.dump(cfg, f)
return str(dst), tmp_dir
def make_single_request(
host: str,
port: int,
text: str,
voice: str = "vivian",
language: str = "English",
timeout: float = 300.0,
) -> httpx.Response:
"""Make a request to the single /v1/audio/speech endpoint."""
url = f"http://{host}:{port}/v1/audio/speech"
payload = {"input": text, "voice": voice, "language": language}
with httpx.Client(timeout=timeout) as client:
return client.post(url, json=payload)
@pytest.fixture(scope="module")
def omni_server_batch2():
"""Start vLLM-Omni server with max_batch_size=2 config."""
config_path, tmp_dir = _make_batch2_stage_config()
with OmniServer(
MODEL,
[
"--stage-configs-path",
config_path,
"--stage-init-timeout",
str(STAGE_INIT_TIMEOUT_S),
"--trust-remote-code",
"--enforce-eager",
"--disable-log-stats",
],
) as server:
yield server
shutil.rmtree(tmp_dir, ignore_errors=True)
class TestSpeechBatchSize2:
"""E2E tests with max_batch_size=2 to verify true batched inference."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch2_produces_valid_audio(self, omni_server_batch2) -> None:
"""Batch of 2 items with batched engine produces valid audio."""
items = [
{"input": "Hello, this is the first sentence."},
{"input": "And this is the second sentence."},
]
response = make_batch_request(
host=omni_server_batch2.host,
port=omni_server_batch2.port,
items=items,
)
assert response.status_code == 200, f"Request failed: {response.text}"
data = response.json()
assert data["total"] == 2
assert data["succeeded"] == 2
assert data["failed"] == 0
# Save audio files for inspection
output_dir = Path(tempfile.mkdtemp(prefix="tts_batch2_output_"))
for result in data["results"]:
assert result["status"] == "success"
audio_bytes = base64.b64decode(result["audio_data"])
assert verify_wav_bytes(audio_bytes), f"Item {result['index']}: invalid WAV"
assert len(audio_bytes) > MIN_AUDIO_BYTES
wav_path = output_dir / f"batch_item_{result['index']}.wav"
wav_path.write_bytes(audio_bytes)
print(f" Item {result['index']}: {len(audio_bytes)} bytes -> {wav_path}")
print(f"\nBatch audio saved to: {output_dir}")
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_batch2_vs_sequential_timing(self, omni_server_batch2) -> None:
"""Compare batch-of-2 vs 2 sequential single requests.
With max_batch_size=2 the engine can process both items
concurrently, so the batch call should be faster than two
sequential single calls (or at least comparable).
"""
text_a = "The quick brown fox jumps over the lazy dog."
text_b = "A journey of a thousand miles begins with a single step."
host = omni_server_batch2.host
port = omni_server_batch2.port
# --- Sequential: 2 single /v1/audio/speech calls ---
t0 = time.perf_counter()
resp_a = make_single_request(host, port, text_a)
resp_b = make_single_request(host, port, text_b)
sequential_time = time.perf_counter() - t0
assert resp_a.status_code == 200, f"Single A failed: {resp_a.text}"
assert resp_b.status_code == 200, f"Single B failed: {resp_b.text}"
# --- Batch: 1 /v1/audio/speech/batch call with 2 items ---
items = [{"input": text_a}, {"input": text_b}]
t0 = time.perf_counter()
resp_batch = make_batch_request(host, port, items)
batch_time = time.perf_counter() - t0
assert resp_batch.status_code == 200, f"Batch failed: {resp_batch.text}"
batch_data = resp_batch.json()
assert batch_data["succeeded"] == 2
# Verify both produced valid audio
for result in batch_data["results"]:
audio_bytes = base64.b64decode(result["audio_data"])
assert verify_wav_bytes(audio_bytes)
assert len(audio_bytes) > MIN_AUDIO_BYTES
print(f"\n Sequential (2 singles): {sequential_time:.2f}s")
print(f" Batch (2 items): {batch_time:.2f}s")
print(f" Speedup: {sequential_time / batch_time:.2f}x")
@@ -49,7 +49,7 @@ tts_server_params = [
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_001(omni_server, openai_client) -> None:
@@ -75,7 +75,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_text_to_audio_002(omni_server, openai_client) -> None:
@@ -15,7 +15,7 @@ from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
@@ -1,248 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E tests for speaker_embedding API parameter.
Tests Base models (0.6B-Base and 1.7B-Base) which support voice cloning
via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import struct
import httpx
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
from tests.helpers.stage_config import get_deploy_config_path
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
STAGE_INIT_TIMEOUT_S = 120
# A synthetic 1024-dim speaker embedding (all 0.1 — not a real voice, but
# exercises the full code path through the talker's _build_prompt_embeds).
DUMMY_EMBEDDING_1024 = [0.1] * 1024
DUMMY_EMBEDDING_2048 = [0.1] * 2048
SYN_TEXT = "Hello."
MIN_AUDIO_BYTES = 2000
# Limit generation to keep tests fast (dummy embeddings produce nonsensical
# output that may never hit a natural stop token).
MAX_NEW_TOKENS = 256
def get_stage_config() -> str:
return get_deploy_config_path("qwen3_tts.yaml")
def _server_args():
return [
"--stage-configs-path",
get_stage_config(),
"--stage-init-timeout",
str(STAGE_INIT_TIMEOUT_S),
"--trust-remote-code",
"--enforce-eager",
"--disable-log-stats",
]
def verify_wav_audio(content: bytes) -> bool:
if len(content) < 44:
return False
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
def assert_not_silence(pcm_bytes: bytes):
"""Assert PCM16 samples are not all identical (e.g. all-silence)."""
samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
unique = set(samples)
assert len(unique) > 1, f"All-silence detected: {len(samples)} samples, unique values: {unique}"
# ── 0.6B-Base model tests ──
@pytest.fixture(scope="module")
def base_server():
"""Start vLLM-Omni server with 0.6B-Base model."""
with OmniServer(MODEL_BASE, _server_args()) as server:
yield server
class TestSpeakerEmbeddingBase:
"""Speaker embedding tests against the 0.6B-Base model (supports Base task)."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_produces_audio(self, base_server) -> None:
"""speaker_embedding with Base task produces valid WAV audio."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "wav",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV"
assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_pcm_not_silence(self, base_server) -> None:
"""speaker_embedding PCM output contains real audio, not all-silence."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "pcm",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert len(response.content) > MIN_AUDIO_BYTES
assert_not_silence(response.content)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_streaming(self, base_server) -> None:
"""speaker_embedding works with streaming PCM output."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "pcm",
"stream": True,
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert "audio/pcm" in response.headers.get("content-type", "")
assert len(response.content) > MIN_AUDIO_BYTES
assert_not_silence(response.content)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_mutually_exclusive_with_ref_audio(self, base_server) -> None:
"""Sending both speaker_embedding and ref_audio returns 400."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"ref_audio": "https://example.com/audio.wav",
"response_format": "wav",
}
with httpx.Client(timeout=30.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
assert "mutually exclusive" in response.text
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_empty_rejected(self, base_server) -> None:
"""Empty speaker_embedding list returns 400."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": [],
"response_format": "wav",
}
with httpx.Client(timeout=30.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
assert "non-empty" in response.text
# ── 1.7B-Base model tests (2048-dim embeddings) ──
@pytest.fixture(scope="module")
def base_1_7b_server():
"""Start vLLM-Omni server with 1.7B-Base model."""
with OmniServer(MODEL_BASE_1_7B, _server_args()) as server:
yield server
class TestSpeakerEmbedding1_7B:
"""Speaker embedding tests against the 1.7B-Base model (2048-dim embeddings)."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_2048_dim_embedding_produces_audio(self, base_1_7b_server) -> None:
"""2048-dim speaker_embedding with 1.7B-Base model produces valid WAV audio."""
url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE_1_7B,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_2048,
"x_vector_only_mode": True,
"response_format": "wav",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV"
assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_1024_dim_on_1_7b_model_rejected(self, base_1_7b_server) -> None:
"""1024-dim speaker_embedding on a 1.7B model (expects 2048) should fail gracefully."""
url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE_1_7B,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "wav",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
# Wrong dimensions should produce an error, not silent garbage.
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
assert "speaker_embedding has 1024 dimensions" in response.text
assert "expected 2048" in response.text
@@ -1,145 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E online tests for Qwen3-TTS WebSocket streaming speech.
"""
import asyncio
import json
import os
import pytest
import websockets
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
from tests.helpers.stage_config import get_deploy_config_path
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
STAGE_INIT_TIMEOUT_S = 120
def get_stage_config() -> str:
return get_deploy_config_path("qwen3_tts.yaml")
@pytest.fixture(scope="module")
def omni_server():
stage_config_path = get_stage_config()
with OmniServer(
MODEL,
[
"--stage-configs-path",
stage_config_path,
"--stage-init-timeout",
str(STAGE_INIT_TIMEOUT_S),
"--trust-remote-code",
"--enforce-eager",
"--disable-log-stats",
],
env_dict={"VLLM_DISABLE_COMPILE_CACHE": "1"},
) as server:
yield server
async def _run_ws_session(host: str, port: int) -> dict:
uri = f"ws://{host}:{port}/v1/audio/speech/stream"
starts: list[dict] = []
dones: list[dict] = []
chunk_lengths: dict[int, list[int]] = {}
session_done: dict | None = None
async with websockets.connect(uri, max_size=None) as ws:
await ws.send(
json.dumps(
{
"type": "session.config",
"model": MODEL,
"voice": "vivian",
"language": "English",
"response_format": "pcm",
"stream_audio": True,
}
)
)
await ws.send(
json.dumps(
{
"type": "input.text",
"text": (
"Hello, this is a websocket streaming test for Qwen three TTS, "
"and this sentence is intentionally long enough to produce audio chunks. "
"This is the second sentence."
),
}
)
)
await ws.send(json.dumps({"type": "input.done"}))
while True:
message = await asyncio.wait_for(ws.recv(), timeout=180)
if isinstance(message, bytes):
if not starts:
raise AssertionError("Received audio bytes before audio.start")
sentence_index = starts[-1]["sentence_index"]
chunk_lengths.setdefault(sentence_index, []).append(len(message))
continue
payload = json.loads(message)
msg_type = payload.get("type")
if msg_type == "audio.start":
starts.append(payload)
chunk_lengths.setdefault(payload["sentence_index"], [])
elif msg_type == "audio.done":
dones.append(payload)
elif msg_type == "session.done":
session_done = payload
break
elif msg_type == "error":
raise AssertionError(f"WebSocket error: {payload['message']}")
else:
raise AssertionError(f"Unexpected WebSocket message: {payload}")
return {
"starts": starts,
"dones": dones,
"chunk_lengths": chunk_lengths,
"session_done": session_done,
}
class TestQwen3TTSWebSocket:
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=4)
def test_streaming_pcm_output(self, omni_server) -> None:
result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port))
starts = result["starts"]
dones = result["dones"]
chunk_lengths = result["chunk_lengths"]
session_done = result["session_done"]
assert session_done is not None
assert session_done["total_sentences"] == 2
assert len(starts) == 2
assert len(dones) == 2
for idx, start in enumerate(starts):
assert start["type"] == "audio.start"
assert start["sentence_index"] == idx
assert start["format"] == "pcm"
assert start["sample_rate"] == 24000
assert start["sentence_text"]
for done in dones:
sentence_index = done["sentence_index"]
total_bytes = done["total_bytes"]
assert done["error"] is False
assert total_bytes > 0
assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}"
assert sum(chunk_lengths[sentence_index]) == total_bytes
+105
View File
@@ -0,0 +1,105 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Online serving tests for ``Qwen/Qwen-Image`` (text-to-image).
- ``test_text_to_image_001``: single chat request, default server (``_get_default_case``).
- ``test_batch_001``: concurrent prompts via ``send_diffusion_request([cfg0, cfg1, ...])`` — one
dict per prompt; each entry carries its own ``messages`` / ``negative_prompt`` (see
``TEST_PROMPTS``).
- ``test_acceleration_feature_001``: ``advanced_model`` only — parametrized ``OmniServerParams``
with cache/offload/parallel flags (see ``_get_diffusion_feature_cases``).
Full feature grids for Qwen-Image remain in ``test_qwen_image_expansion.py`` (L4).
From ``tests/``::
pytest -s -v e2e/online_serving/test_qwen_image.py -m "core_model and diffusion" --run-level=core_model
pytest -s -v e2e/online_serving/test_qwen_image.py -m "advanced_model and diffusion" --run-level=advanced_model
"""
import os
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Qwen/Qwen-Image"
T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
NEGATIVE_PROMPT = "blurry, low quality"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
TEST_PROMPTS: list[dict[str, str]] = [
{"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"},
{"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": "cinematic, realistic"},
{"prompt": "a futuristic city skyline at sunset", "negative_prompt": "blurry, foggy"},
{"prompt": "a bowl of fresh strawberries", "negative_prompt": "low detail"},
{"prompt": "a medieval knight standing in the rain", "negative_prompt": "modern clothing"},
{"prompt": "a cat wearing sunglasses lounging in a garden", "negative_prompt": "dark lighting"},
{"prompt": "a spaceship flying above a volcano", "negative_prompt": "low contrast"},
{"prompt": "a watercolor painting of a mountain lake", "negative_prompt": "photo, realistic"},
]
def _get_default_case(model: str):
"""Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
return [
pytest.param(
OmniServerParams(model=model),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
),
]
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
"""Default Qwen-Image T2I smoke (single ``default`` server config)."""
messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
def test_batch_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
"""Concurrent T2I: one ``request_config`` dict per prompt (``send_diffusion_request`` list mode)."""
request_config = [
{
"model": omni_server.model,
"messages": dummy_messages_from_mix_data(content_text=prompt["prompt"]),
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": prompt["negative_prompt"],
"true_cfg_scale": 4.0,
"seed": 42,
},
}
for prompt in TEST_PROMPTS
]
openai_client.send_diffusion_request(request_config)
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2509`` — two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2509`` only, ``advanced_model`` — mixed input
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row per model.
From ``tests/``::
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "core_model and diffusion" --run-level=core_model
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "advanced_model and diffusion" --run-level=advanced_model
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
"Transform the second image into a Vincent van Gogh style painting. "
"Then juxtapose the two transformed images into a single artwork for visual contrast."
)
NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``)."""
return [
pytest.param(
OmniServerParams(
model=model,
),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
),
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
indirect=True,
)
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Single-reference edit smoke for ``Qwen/Qwen-Image-Edit`` (CFG when negative prompt + ``true_cfg_scale`` > 1)."""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
indirect=True,
)
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2509``."""
image_data_url_list = [f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" for _ in range(2)]
messages = dummy_messages_from_mix_data(image_data_url=image_data_url_list, content_text=MULTI_EDIT_PROMPT)
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
indirect=True,
)
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Multi-reference edit with distinct input resolutions; per-output ``width``/``height`` as lists (client splits into concurrent scalar-size calls)."""
image_data_url_list = [
f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}",
f"data:image/jpeg;base64,{generate_synthetic_image(384, 256)['base64']}",
]
messages = dummy_messages_from_mix_data(image_data_url=image_data_url_list, content_text=MULTI_EDIT_PROMPT)
# List height/width: client sends 512×512 and 768×512 in parallel, merges to two images; assertions use lists.
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": [512, 768],
"width": [512, 512],
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Online serving tests for ``Qwen/Qwen-Image-Layered`` (layer decomposition).
- ``test_feature``: default single-GPU smoke via ``_get_diffusion_feature_cases`` (one ``default`` case).
- ``test_layered_output_image_count``: guard for issue #1969 (multi-layer image count).
- ``test_empty_prompt``: guard for issue #1966 (empty prompt).
Broader parallel / cache feature matrices previously lived in ``test_qwen_image_layered_expansion.py``;
that module is now a shim that points here.
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.media import generate_synthetic_image
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
MODEL = "Qwen/Qwen-Image-Layered"
EDIT_PROMPT = "Decompose this image into layers."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
"Transform the second image into a Vincent van Gogh style painting. "
"Then juxtapose the two transformed images into a single artwork for visual contrast."
)
NEGATIVE_PROMPT = "blurry, low quality, distorted"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
return [
pytest.param(
OmniServerParams(model=model),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
),
]
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Default Qwen-Image-Layered smoke (single ``default`` server config)."""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"num_inference_steps": 2,
"negative_prompt": NEGATIVE_PROMPT,
"true_cfg_scale": 4.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
+61
View File
@@ -0,0 +1,61 @@
"""
Tests for Stable Diffusion 3.5 medium model.
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
POSITIVE_PROMPT = "A serene mountain landscape at sunset"
NEGATIVE_PROMPT = "blurry, low quality, distorted"
# For now, we only test one or two good configurations for this model to keep
# the CI light. The test cases focus on the most common feature combinations
# that provide good performance improvements.
def _get_diffusion_feature_cases(model: str):
return [
# Cache-DiT + CFG Parallel + Tensor Parallel
pytest.param(
OmniServerParams(
model=model,
),
id="default",
marks=FOUR_CARD_FEATURE_MARKS,
),
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases(
model="stabilityai/stable-diffusion-3.5-medium",
),
indirect=True,
)
def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 1024,
"width": 1024,
"num_inference_steps": 28,
"negative_prompt": NEGATIVE_PROMPT,
"guidance_scale": 4.5,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@@ -5,7 +5,12 @@ Tests for Stable Diffusion 3.5 medium model.
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import (
OmniServer,
OmniServerParams,
OpenAIClientHandler,
dummy_messages_from_mix_data,
)
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
@@ -45,9 +50,10 @@ def _get_diffusion_feature_cases(model: str):
indirect=True,
)
def test_sd3_medium(omni_server: OmniServer, openai_client: OpenAIClientHandler):
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
request_config = {
"model": omni_server.model,
"messages": [{"role": "user", "content": POSITIVE_PROMPT}],
"messages": messages,
"extra_body": {
"height": 1024,
"width": 1024,
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E online serving tests for the async video generation API.
These tests exercise the real `/v1/videos` job lifecycle against a running
diffusion model, including create, retrieve, list, download, and delete flows.
"""
import os
import time
import uuid
from pathlib import Path
from typing import Any
import pytest
import requests
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
VIDEO_POLL_INTERVAL_S = 2.0
VIDEO_TIMEOUT_S = 900.0
@pytest.fixture(scope="module")
def video_server():
"""Start a real video generation server for async video API tests."""
with OmniServer(
MODEL,
[
"--num-gpus",
"1",
"--boundary-ratio",
"0.875",
"--flow-shift",
"5.0",
"--disable-log-stats",
],
) as server:
yield server
def _video_api_url(server: OmniServer, suffix: str = "") -> str:
return f"http://{server.host}:{server.port}/v1/videos{suffix}"
def _multipart_fields(payload: dict[str, Any]) -> list[tuple[str, tuple[None, str]]]:
return [(key, (None, str(value))) for key, value in payload.items() if value is not None]
def _create_video_job(
server: OmniServer,
*,
prompt: str | None = None,
**overrides: Any,
) -> requests.Response:
payload: dict[str, Any] = {
"prompt": prompt or f"video api e2e {uuid.uuid4().hex[:8]}",
"width": 640,
"height": 480,
"num_frames": 5,
"fps": 8,
"num_inference_steps": 2,
"guidance_scale": 1.0,
"guidance_scale_2": 1.0,
"boundary_ratio": 0.875,
"flow_shift": 5.0,
"seed": 42,
}
payload.update(overrides)
return requests.post(
_video_api_url(server),
files=_multipart_fields(payload),
timeout=VIDEO_TIMEOUT_S,
)
def _retrieve_video_job(server: OmniServer, video_id: str) -> requests.Response:
return requests.get(_video_api_url(server, f"/{video_id}"), timeout=VIDEO_TIMEOUT_S)
def _wait_for_video_status(server: OmniServer, video_id: str, expected_status: str) -> dict[str, Any]:
deadline = time.time() + VIDEO_TIMEOUT_S
last_payload: dict[str, Any] | None = None
while time.time() < deadline:
response = _retrieve_video_job(server, video_id)
assert response.status_code == 200, response.text
last_payload = response.json()
status = last_payload["status"]
if status == expected_status:
return last_payload
if status == "failed":
raise AssertionError(f"Video job {video_id} failed unexpectedly: {last_payload}")
time.sleep(VIDEO_POLL_INTERVAL_S)
raise AssertionError(
f"Timed out waiting for video job {video_id} to reach status={expected_status}. Last payload: {last_payload}"
)
def _wait_for_video_missing(server: OmniServer, video_id: str) -> None:
deadline = time.time() + 30.0
last_status: int | None = None
while time.time() < deadline:
response = _retrieve_video_job(server, video_id)
last_status = response.status_code
if response.status_code == 404:
return
time.sleep(0.5)
raise AssertionError(f"Timed out waiting for video job {video_id} to disappear. Last status={last_status}")
def _delete_video_job(server: OmniServer, video_id: str) -> requests.Response:
return requests.delete(_video_api_url(server, f"/{video_id}"), timeout=VIDEO_TIMEOUT_S)
def _delete_video_job_with_retry(server: OmniServer, video_id: str) -> requests.Response:
deadline = time.time() + 30.0
last_response: requests.Response | None = None
while time.time() < deadline:
response = _delete_video_job(server, video_id)
last_response = response
if response.status_code != 409:
return response
time.sleep(1.0)
raise AssertionError(
f"Timed out waiting to delete video job {video_id}. "
f"Last response: {None if last_response is None else last_response.text}"
)
def _best_effort_delete(server: OmniServer, video_id: str) -> None:
try:
response = _delete_video_job_with_retry(server, video_id)
if response.status_code not in (200, 404):
print(f"Cleanup delete for {video_id} returned {response.status_code}: {response.text}")
except Exception as exc:
print(f"Cleanup delete for {video_id} failed: {exc}")
def _assert_mp4_payload(content: bytes) -> None:
assert len(content) > 32, f"Downloaded video payload is unexpectedly small: {len(content)} bytes"
assert content[4:8] == b"ftyp", "Downloaded payload does not look like an MP4 file."
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_create_list_retrieve_download_video(video_server: OmniServer, tmp_path: Path) -> None:
video_id: str | None = None
try:
create_response = _create_video_job(video_server, prompt="A small paper airplane gliding over a lake at dawn.")
assert create_response.status_code == 200, create_response.text
created = create_response.json()
video_id = created["id"]
assert created["object"] == "video"
assert created["status"] == "queued"
assert created["model"] == MODEL
completed = _wait_for_video_status(video_server, video_id, "completed")
assert completed["file_name"] is not None
assert completed["completed_at"] is not None
assert completed["progress"] == 100
retrieve_response = _retrieve_video_job(video_server, video_id)
assert retrieve_response.status_code == 200, retrieve_response.text
retrieved = retrieve_response.json()
assert retrieved["id"] == video_id
assert retrieved["status"] == "completed"
list_response = requests.get(_video_api_url(video_server), timeout=VIDEO_TIMEOUT_S)
assert list_response.status_code == 200, list_response.text
listed = list_response.json()
assert listed["object"] == "list"
assert any(item["id"] == video_id for item in listed["data"])
download_response = requests.get(
_video_api_url(video_server, f"/{video_id}/content"),
timeout=VIDEO_TIMEOUT_S,
)
assert download_response.status_code == 200, download_response.text
assert download_response.headers["content-type"].startswith("video/mp4")
assert completed["file_name"] in download_response.headers.get("content-disposition", "")
_assert_mp4_payload(download_response.content)
output_path = tmp_path / completed["file_name"]
output_path.write_bytes(download_response.content)
assert output_path.stat().st_size == len(download_response.content)
finally:
if video_id is not None:
_best_effort_delete(video_server, video_id)
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_delete_video_while_queued_or_in_progress(video_server: OmniServer) -> None:
create_response = _create_video_job(
video_server,
prompt="A toy boat sailing through fog.",
num_frames=13,
num_inference_steps=4,
)
assert create_response.status_code == 200, create_response.text
created = create_response.json()
video_id = created["id"]
assert created["status"] == "queued"
# Delete immediately so the job is still queued or just entering execution.
delete_response = _delete_video_job_with_retry(video_server, video_id)
assert delete_response.status_code == 200, delete_response.text
deleted = delete_response.json()
assert deleted["id"] == video_id
assert deleted["deleted"] is True
assert deleted["object"] == "video.deleted"
_wait_for_video_missing(video_server, video_id)
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
def test_delete_video_after_processing(video_server: OmniServer) -> None:
create_response = _create_video_job(
video_server,
prompt="A lantern drifting above a quiet forest.",
seed=43,
)
assert create_response.status_code == 200, create_response.text
video_id = create_response.json()["id"]
completed = _wait_for_video_status(video_server, video_id, "completed")
assert completed["file_name"] is not None
delete_response = _delete_video_job_with_retry(video_server, video_id)
assert delete_response.status_code == 200, delete_response.text
deleted = delete_response.json()
assert deleted["id"] == video_id
assert deleted["deleted"] is True
assert deleted["object"] == "video.deleted"
_wait_for_video_missing(video_server, video_id)
download_response = requests.get(
_video_api_url(video_server, f"/{video_id}/content"),
timeout=VIDEO_TIMEOUT_S,
)
assert download_response.status_code == 404, download_response.text
+52 -154
View File
@@ -12,7 +12,6 @@ import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import httpx
import pytest
from tests.helpers.mark import hardware_test
@@ -26,36 +25,15 @@ EXTRA_ARGS = ["--trust-remote-code", "--enforce-eager", "--disable-log-stats"]
TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, server_args=EXTRA_ARGS)]
def make_speech_request(
host: str,
port: int,
text: str,
voice: str = "casual_female",
language: str = "English",
timeout: float = 120.0,
) -> httpx.Response:
"""Make a request to the /v1/audio/speech endpoint."""
url = f"http://{host}:{port}/v1/audio/speech"
payload = {
"input": text,
"voice": voice,
"language": language,
}
with httpx.Client(timeout=timeout) as client:
return client.post(url, json=payload)
def verify_wav_audio(content: bytes) -> bool:
"""Verify that content is valid WAV audio data."""
# WAV files start with "RIFF" header
if len(content) < 44: # Minimum WAV header size
if len(content) < 44:
return False
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
# Minimum expected audio size for a short sentence (~1 second of 24kHz 16-bit mono WAV)
MIN_AUDIO_BYTES = 10000
_MIN_AUDIO_BYTES_BASIC = 10000
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
@@ -63,151 +41,71 @@ class TestVoxtralTTSFixedVoice:
"""E2E tests for Voxtral TTS model."""
@pytest.mark.core_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_speech_english_basic(self, omni_server) -> None:
def test_speech_english_basic(self, omni_server, openai_client) -> None:
"""Test basic English TTS generation."""
response = make_speech_request(
host=omni_server.host,
port=omni_server.port,
text="Hello, how are you?",
voice="casual_female",
language="English",
)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
assert len(response.content) > MIN_AUDIO_BYTES, (
f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Hello, how are you?",
"voice": "casual_female",
"language": "English",
"response_format": "wav",
"timeout": 120.0,
"min_audio_bytes": _MIN_AUDIO_BYTES_BASIC,
}
)
@pytest.mark.core_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_speech_english_streaming(self, omni_server) -> None:
"""Test basic streaming English TTS generation."""
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
payload = {
"input": "Hello, how are you?",
"voice": "casual_female",
"language": "English",
"stream": True,
"response_format": "pcm",
}
with httpx.Client(timeout=120.0) as client:
with client.stream("POST", url, json=payload) as response:
assert response.status_code == 200
assert response.headers.get("content-type") == "audio/pcm"
total = sum(len(c) for c in response.iter_bytes())
assert total > MIN_AUDIO_BYTES, (
f"Streamed audio too small ({total} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
def test_speech_english_streaming(self, omni_server, openai_client) -> None:
"""Test basic streaming English TTS generation (PCM via streaming API)."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Hello, how are you?",
"voice": "casual_female",
"language": "English",
"stream": True,
"response_format": "pcm",
"timeout": 120.0,
"min_audio_bytes": _MIN_AUDIO_BYTES_BASIC,
}
)
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_speech_different_voices(self, omni_server) -> None:
def test_speech_different_voices(self, omni_server, openai_client) -> None:
"""Test TTS with different voice presets."""
voices = ["casual_female", "neutral_male"]
for voice in voices:
response = make_speech_request(
host=omni_server.host,
port=omni_server.port,
text="Testing voice selection.",
voice=voice,
language="English",
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Testing voice selection.",
"voice": voice,
"language": "English",
"response_format": "wav",
"timeout": 120.0,
}
)
assert response.status_code == 200, f"Request failed for voice {voice}: {response.text}"
assert verify_wav_audio(response.content), f"Invalid WAV for voice {voice}"
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_speech_invalid_voice_rejected(self, omni_server) -> None:
"""Request with a non-existent voice should return an error response."""
response = make_speech_request(
host=omni_server.host,
port=omni_server.port,
text="This voice does not exist.",
voice="nonexistent_voice_xyz",
language="English",
def test_speech_invalid_voice_rejected(self, omni_server, openai_client) -> None:
"""Request with a non-existent voice should be rejected (HTTP 200 with JSON or 4xx, depending on server)."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "This voice does not exist.",
"voice": "nonexistent_voice_xyz",
"language": "English",
"response_format": "wav",
"timeout": 120.0,
"err_message": "Invalid speaker",
}
)
# vLLM returns ErrorResponse as JSON with HTTP 200;
# verify the body is a JSON error, not valid WAV audio.
data = response.json()
assert "error" in data or "message" in data, f"Expected error response for invalid voice, got: {data}"
assert not verify_wav_audio(response.content), "Should not return valid audio for invalid voice"
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_speech_binary_response_not_utf8_error(self, omni_server) -> None:
"""
Regression test: Verify binary audio is returned, not UTF-8 error.
This test ensures the multimodal_output property correctly retrieves
audio from completion outputs, preventing the "TTS model did not
produce audio output" error.
"""
response = make_speech_request(
host=omni_server.host,
port=omni_server.port,
text="This should return binary audio, not a JSON error.",
voice="casual_female",
language="English",
)
# Should NOT be a JSON error response
assert response.status_code == 200, f"Request failed: {response.text}"
# Verify it's binary audio, not JSON
try:
# If this succeeds and starts with {"error", it's a bug
text = response.content.decode("utf-8")
assert not text.startswith('{"error"'), f"Got error response instead of audio: {text}"
except UnicodeDecodeError:
# This is expected - binary audio can't be decoded as UTF-8
pass
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestVoxtralTTSAPIEndpoints:
"""Test API endpoint functionality."""
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_list_voices_endpoint(self, omni_server) -> None:
"""Test the /v1/audio/voices endpoint returns available voices."""
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/voices"
with httpx.Client(timeout=30.0) as client:
response = client.get(url)
assert response.status_code == 200
data = response.json()
assert "voices" in data
assert isinstance(data["voices"], list)
assert len(data["voices"]) > 0
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_models_endpoint(self, omni_server) -> None:
"""Test the /v1/models endpoint returns loaded model."""
url = f"http://{omni_server.host}:{omni_server.port}/v1/models"
with httpx.Client(timeout=30.0) as client:
response = client.get(url)
assert response.status_code == 200
data = response.json()
assert "data" in data
assert len(data["data"]) > 0
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Online serving smoke for ``Wan-AI/Wan2.2-T2V-A14B-Diffusers`` (text-to-video via ``/v1/videos``).
Uses a single ``default`` ``OmniServerParams`` row via ``_get_diffusion_feature_cases`` (no extra
``server_args``). Multi-variant / parallel coverage lives in ``test_wan22_expansion.py`` (L4).
From ``tests/``::
pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model
pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "advanced_model and diffusion" --run-level=advanced_model
"""
import os
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
return [
pytest.param(
OmniServerParams(model=model),
id="default",
marks=SINGLE_CARD_FEATURE_MARKS,
),
]
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
def test_text_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
"""Default Wan2.2 T2V smoke: async ``/v1/videos`` job completes and returns video bytes."""
request_config = {
"model": omni_server.model,
"form_data": {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,
"height": 512,
"width": 512,
"num_frames": 8,
"fps": 8,
"num_inference_steps": 2,
"guidance_scale": 4.0,
"seed": 42,
},
}
openai_client.send_video_diffusion_request(request_config)
+88
View File
@@ -0,0 +1,88 @@
"""
Tests of common diffusion feature combinations in online serving mode
for Z-Image.
Coverage is intentionally limited to the minimal 4xL4 cases that
exercise Z-Image's supported parallel feature combinations:
- CacheDiT + FP8 + Ring=2 + TP=2
- TeaCache + FP8 + Ulysses=2 + Ring=2
"""
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
MODEL = "Tongyi-MAI/Z-Image-Turbo"
PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard."
FOUR_CARD_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
def _get_diffusion_feature_cases():
"""Return the common Z-Image feature combinations that fit L4 CI."""
return [
pytest.param(
OmniServerParams(
model=MODEL,
),
id="default",
marks=FOUR_CARD_MARKS,
)
]
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases(),
indirect=True,
)
def test_basic_001(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
):
"""Exercise supported Z-Image diffusion features in minimal CI cases."""
messages = dummy_messages_from_mix_data(content_text=PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 2,
"guidance_scale": 0.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases(),
indirect=True,
)
def test_different_sizes_001(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
):
"""Exercise supported Z-Image diffusion features in minimal CI cases."""
messages = dummy_messages_from_mix_data(content_text=PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": [512, 512],
"width": [512, 768],
"num_inference_steps": 2,
"guidance_scale": 0.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
@@ -1,79 +0,0 @@
# stage config for running dynin_omni with a 3-stage architecture.
# this config is intended for e2e smoke tests.
stage_args:
- stage_id: 0
stage_type: llm
runtime:
process: true
devices: "0"
max_batch_size: 1
engine_args:
model_stage: token2text
model_arch: DyninOmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
engine_output_type: latent
trust_remote_code: true
gpu_memory_utilization: 0.5
enforce_eager: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 4096
is_comprehension: true
final_output: true
final_output_type: text
- stage_id: 1
stage_type: llm
runtime:
process: true
devices: "0"
max_batch_size: 1
engine_args:
model_stage: token2image
model_arch: DyninOmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
engine_output_type: latent
trust_remote_code: true
gpu_memory_utilization: 0.2
enforce_eager: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 4096
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image
final_output: true
final_output_type: image
- stage_id: 2
stage_type: llm
runtime:
process: true
devices: "0"
max_batch_size: 1
engine_args:
model_stage: token2audio
model_arch: DyninOmniForConditionalGeneration
worker_type: generation
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
engine_output_type: latent
trust_remote_code: true
gpu_memory_utilization: 0.2
enforce_eager: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 4096
engine_input_source: [1]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio
final_output: true
final_output_type: audio
runtime:
enabled: true
edges:
- from: 0
to: 1
- from: 1
to: 2