[Test] Unify L2/L3 test layout, Buildkite steps, and test helpers (#2556)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
b85833eeec
commit
6f2ad7b403
134 files changed
+4904
-4185
No files matched your search
@@ -282,7 +282,7 @@ def _run_with_custom_transformer(model: str, output_dir: Path) -> list[Image.Ima
|
||||
return frames
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skipif(
|
||||
@@ -381,7 +381,7 @@ def _run_vllm_omni_serving(model: str, output_dir: Path) -> list[Image.Image]:
|
||||
return frames
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.benchmark
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.skipif(
|
||||
|
||||
+12
-16
@@ -2,44 +2,40 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
# Prefer a tiny/random checkpoint for CI.
|
||||
# Override in CI if needed: AUDIOX_TEST_MODEL=<model-or-local-path>
|
||||
models = [os.environ.get("AUDIOX_TEST_MODEL", "zhangj1an/audiox_random")]
|
||||
|
||||
# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize
|
||||
_OMNI_RUNNER_PARAMS = [(m, None, {"model_class_name": "AudioXPipeline"}) for m in models]
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True, ids=models),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_audiox_model(model_name: str):
|
||||
m = Omni(model=model_name, model_class_name="AudioXPipeline")
|
||||
|
||||
def test_audiox_model(omni_runner: OmniRunner) -> None:
|
||||
# Keep runtime short for CI.
|
||||
seconds_total = 2.0
|
||||
# AudioXPipeline always emits 44.1 kHz stereo (advertised via class-level
|
||||
# ``audio_sample_rate``); the trimmed output should match this rate.
|
||||
sample_rate = 44100
|
||||
|
||||
outputs = m.generate(
|
||||
outputs = omni_runner.omni.generate(
|
||||
prompts={"prompt": "A dog barking in a quiet park."},
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
num_inference_steps=4,
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end tests for Bagel text2text and img2text (understanding) tasks.
|
||||
|
||||
These tests validate that the Bagel multistage pipeline correctly generates
|
||||
text output for understanding tasks, matching reference results.
|
||||
|
||||
Equivalent to running:
|
||||
python3 examples/offline_inference/bagel/end2end.py \
|
||||
--modality text2text \
|
||||
--prompts "Where is the capital of France?"
|
||||
|
||||
python3 examples/offline_inference/bagel/end2end.py \
|
||||
--modality img2text \
|
||||
--prompts "Please describe this image" \
|
||||
--image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml")
|
||||
|
||||
REFERENCE_TEXT_TEXT2TEXT = "The capital of France is Paris."
|
||||
REFERENCE_TEXT_IMG2TEXT = (
|
||||
"This is a photo of a wooden boardwalk or pathway that leads through "
|
||||
"tall green grass. The path appears to be in a natural setting, possibly "
|
||||
"a wetland or marsh area. The sky above is blue with some scattered "
|
||||
"clouds, suggesting it might be a sunny day. The overall scene looks "
|
||||
"peaceful and serene."
|
||||
)
|
||||
|
||||
|
||||
# (model, stage_config_path, extra_omni_kwargs) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
|
||||
_OMNI_RUNNER_PARAM = (MODEL_NAME, STAGE_CONFIG)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.advanced_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _extract_text(omni_outputs: list) -> str:
|
||||
"""Extract generated text from OmniRequestOutput list."""
|
||||
for req_output in omni_outputs:
|
||||
ro = getattr(req_output, "request_output", None)
|
||||
if ro and getattr(ro, "outputs", None):
|
||||
return "".join(getattr(o, "text", "") or "" for o in ro.outputs)
|
||||
return ""
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_text2text(run_level, omni_runner: OmniRunner) -> None:
|
||||
"""Test Bagel text2text produces correct text output."""
|
||||
omni = omni_runner.omni
|
||||
prompt = "<|im_start|>user\nWhere is the capital of France?<|im_end|>\n<|im_start|>assistant\n"
|
||||
params_list = omni.default_sampling_params_list
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[{"prompt": prompt, "modalities": ["text"]}],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0, "No outputs returned"
|
||||
text = _extract_text(omni_outputs)
|
||||
assert len(text) > 0, "Generated text is empty"
|
||||
|
||||
if run_level == "advanced_model":
|
||||
assert text == REFERENCE_TEXT_TEXT2TEXT, f"Text mismatch: expected {REFERENCE_TEXT_TEXT2TEXT!r}, got {text!r}"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_img2text(run_level, omni_runner: OmniRunner) -> None:
|
||||
"""Test Bagel img2text produces correct text output."""
|
||||
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
|
||||
omni = omni_runner.omni
|
||||
prompt = "<|im_start|>user\n<|image_pad|>\nPlease describe this image<|im_end|>\n<|im_start|>assistant\n"
|
||||
params_list = omni.default_sampling_params_list
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"image": input_image},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0, "No outputs returned"
|
||||
text = _extract_text(omni_outputs)
|
||||
assert len(text) > 0, "Generated text is empty"
|
||||
|
||||
if run_level in ["advanced_model", "full_model"]:
|
||||
assert "wooden boardwalk" in text.lower(), f"Text mismatch: expected 'wooden boardwalk' in {text!r}"
|
||||
+44
-51
@@ -2,7 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for BAGEL LoRA support (Stage 1 / DiT).
|
||||
Expanded end-to-end test for BAGEL in offline mode.
|
||||
|
||||
This test file primarily covers end-to-end tests for LoRA support (Stage 1 / DiT).
|
||||
|
||||
Validates that LoRA adapters are correctly loaded, applied with controllable
|
||||
scale, and cleanly deactivated. Uses a synthetic rank-1 adapter targeting the
|
||||
@@ -32,7 +34,7 @@ from safetensors.torch import save_file
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
@@ -41,26 +43,15 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
BAGEL_STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml")
|
||||
DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>"
|
||||
|
||||
# (model, stage_configs_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
|
||||
_OMNI_RUNNER_PARAM = (MODEL, BAGEL_STAGE_CONFIG)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers (reused from test_bagel_text2img.py patterns)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
||||
if run_level == "advanced_model":
|
||||
return modify_stage_config(
|
||||
config_path,
|
||||
deletes={
|
||||
"stages": {
|
||||
0: ["load_format"],
|
||||
1: ["load_format"],
|
||||
}
|
||||
},
|
||||
)
|
||||
return config_path
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 10) -> list[OmniSamplingParams]:
|
||||
params_list = omni.default_sampling_params_list
|
||||
if len(params_list) > 1:
|
||||
@@ -146,50 +137,52 @@ def _make_file_lora_request(adapter_dir: Path) -> LoRARequest:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_lora_scale_and_deactivation(run_level, tmp_path):
|
||||
def test_bagel_lora_scale_and_deactivation(omni_runner: OmniRunner, tmp_path) -> None:
|
||||
"""Validate LoRA effect, bounded perturbation, and clean deactivation."""
|
||||
config_path = _resolve_deploy_config(BAGEL_STAGE_CONFIG, run_level)
|
||||
with OmniRunner(MODEL, stage_configs_path=config_path) as runner:
|
||||
omni = runner.omni
|
||||
lora_request = _make_file_lora_request(tmp_path / "bagel_lora")
|
||||
omni = omni_runner.omni
|
||||
lora_request = _make_file_lora_request(tmp_path / "bagel_lora")
|
||||
|
||||
# 1) Baseline (no LoRA)
|
||||
baseline = _generate_bagel_image(omni)
|
||||
# 1) Baseline (no LoRA)
|
||||
baseline = _generate_bagel_image(omni)
|
||||
|
||||
# 2) LoRA with scale=1.0
|
||||
img_1x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=1.0)
|
||||
# 2) LoRA with scale=1.0
|
||||
img_1x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=1.0)
|
||||
|
||||
# 3) LoRA with scale=2.0
|
||||
img_2x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=2.0)
|
||||
# 3) LoRA with scale=2.0
|
||||
img_2x = _generate_bagel_image_with_lora(omni, lora_request, lora_scale=2.0)
|
||||
|
||||
# 4) No LoRA again (deactivation)
|
||||
restored = _generate_bagel_image(omni)
|
||||
# 4) No LoRA again (deactivation)
|
||||
restored = _generate_bagel_image(omni)
|
||||
|
||||
baseline_arr = np.array(baseline, dtype=np.int16)
|
||||
img_1x_arr = np.array(img_1x, dtype=np.int16)
|
||||
img_2x_arr = np.array(img_2x, dtype=np.int16)
|
||||
restored_arr = np.array(restored, dtype=np.int16)
|
||||
baseline_arr = np.array(baseline, dtype=np.int16)
|
||||
img_1x_arr = np.array(img_1x, dtype=np.int16)
|
||||
img_2x_arr = np.array(img_2x, dtype=np.int16)
|
||||
restored_arr = np.array(restored, dtype=np.int16)
|
||||
|
||||
diff_1x = np.abs(baseline_arr - img_1x_arr).mean()
|
||||
diff_2x = np.abs(baseline_arr - img_2x_arr).mean()
|
||||
diff_restored = np.abs(baseline_arr - restored_arr).mean()
|
||||
diff_1x = np.abs(baseline_arr - img_1x_arr).mean()
|
||||
diff_2x = np.abs(baseline_arr - img_2x_arr).mean()
|
||||
diff_restored = np.abs(baseline_arr - restored_arr).mean()
|
||||
|
||||
# (a) Adapter has visible effect at both scales
|
||||
assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}"
|
||||
assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}"
|
||||
# (a) Adapter has visible effect at both scales
|
||||
assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}"
|
||||
assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}"
|
||||
|
||||
# (b) Different scales produce different outputs
|
||||
assert not np.isclose(diff_1x, diff_2x, atol=1.0), (
|
||||
f"LoRA scale has no effect: diff_1x={diff_1x:.2f}, diff_2x={diff_2x:.2f}"
|
||||
)
|
||||
# (b) Different scales produce different outputs
|
||||
assert not np.isclose(diff_1x, diff_2x, atol=1.0), (
|
||||
f"LoRA scale has no effect: diff_1x={diff_1x:.2f}, diff_2x={diff_2x:.2f}"
|
||||
)
|
||||
|
||||
# (c) Output is not corrupted (scale=2.0 can produce ~2x the diff of scale=1.0)
|
||||
assert diff_1x < 80, f"LoRA output looks corrupted: diff_1x={diff_1x}"
|
||||
assert diff_2x < 120, f"LoRA output looks corrupted: diff_2x={diff_2x}"
|
||||
# (c) Output is not corrupted (scale=2.0 can produce ~2x the diff of scale=1.0)
|
||||
assert diff_1x < 80, f"LoRA output looks corrupted: diff_1x={diff_1x}"
|
||||
assert diff_2x < 120, f"LoRA output looks corrupted: diff_2x={diff_2x}"
|
||||
|
||||
# (d) Deactivation fully restores base model
|
||||
assert diff_restored == 0.0, f"Base model not restored after LoRA deactivation: diff={diff_restored}"
|
||||
# (d) Deactivation fully restores base model
|
||||
assert diff_restored == 0.0, f"Base model not restored after LoRA deactivation: diff={diff_restored}"
|
||||
@@ -1,220 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for Bagel img2img generation.
|
||||
|
||||
This test validates that the Bagel model generates images from an input image
|
||||
and text prompt that match expected reference pixel values within a ±10 tolerance.
|
||||
|
||||
Equivalent to running:
|
||||
python3 examples/offline_inference/bagel/end2end.py \
|
||||
--prompts "Change the grass color to red" \
|
||||
--modality img2img --step 15 \
|
||||
--image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
"""
|
||||
|
||||
import socket
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
# Generated with seed=52, num_inference_steps=15,
|
||||
# prompt='Change the grass color to red',
|
||||
# input image: 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||
]
|
||||
|
||||
if current_omni_platform.is_rocm():
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (156, 172, 217)},
|
||||
{"position": (400, 50), "rgb": (105, 144, 217)},
|
||||
{"position": (700, 100), "rgb": (118, 159, 232)},
|
||||
{"position": (150, 400), "rgb": (180, 22, 52)},
|
||||
{"position": (512, 336), "rgb": (221, 211, 194)},
|
||||
{"position": (700, 400), "rgb": (192, 10, 46)},
|
||||
{"position": (100, 600), "rgb": (102, 12, 22)},
|
||||
{"position": (400, 600), "rgb": (161, 28, 47)},
|
||||
{"position": (700, 600), "rgb": (100, 87, 94)},
|
||||
{"position": (256, 256), "rgb": (181, 201, 221)},
|
||||
]
|
||||
|
||||
PIXEL_TOLERANCE = 10
|
||||
|
||||
DEFAULT_PROMPT = "<|fim_middle|><|im_start|>Change the grass color to red<|im_end|>"
|
||||
|
||||
EXPECTED_OUTPUT_SIZE = (1024, 672)
|
||||
|
||||
|
||||
def _load_input_image() -> Image.Image:
|
||||
"""Load the test input image via vllm's ImageAsset."""
|
||||
return ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Find and return a free ephemeral port by binding to port 0."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
s.listen(1)
|
||||
port = s.getsockname()[1]
|
||||
return port
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||
"""Configure sampling parameters for Bagel img2img generation.
|
||||
|
||||
Args:
|
||||
omni: The Omni instance to get default params from.
|
||||
num_inference_steps: Number of inference steps for the diffusion stage.
|
||||
|
||||
Returns:
|
||||
Configured sampling params list.
|
||||
"""
|
||||
params_list = omni.default_sampling_params_list
|
||||
if len(params_list) > 1:
|
||||
params_list[1].num_inference_steps = num_inference_steps # type: ignore
|
||||
params_list[1].extra_args = { # type: ignore
|
||||
"cfg_text_scale": 4.0,
|
||||
"cfg_img_scale": 1.5,
|
||||
}
|
||||
return params_list
|
||||
|
||||
|
||||
def _extract_generated_image(omni_outputs: list) -> Image.Image | None:
|
||||
"""Extract the generated image from Omni outputs.
|
||||
|
||||
Args:
|
||||
omni_outputs: List of outputs from omni.generate().
|
||||
|
||||
Returns:
|
||||
The first generated PIL Image, or None if no image found.
|
||||
"""
|
||||
for req_output in omni_outputs:
|
||||
if images := getattr(req_output, "images", None):
|
||||
return images[0]
|
||||
if hasattr(req_output, "request_output") and req_output.request_output:
|
||||
stage_out = req_output.request_output
|
||||
if hasattr(stage_out, "images") and stage_out.images:
|
||||
return stage_out.images[0]
|
||||
return None
|
||||
|
||||
|
||||
def _validate_pixels(
|
||||
image: Image.Image,
|
||||
reference_pixels: list[dict[str, Any]] = REFERENCE_PIXELS,
|
||||
tolerance: int = PIXEL_TOLERANCE,
|
||||
) -> None:
|
||||
"""Validate that image pixels match expected reference values.
|
||||
|
||||
Args:
|
||||
image: The PIL Image to validate.
|
||||
reference_pixels: List of dicts with 'position' (x, y) and 'rgb' (R, G, B).
|
||||
tolerance: Maximum allowed difference per color channel.
|
||||
|
||||
Raises:
|
||||
AssertionError: If any pixel differs beyond tolerance.
|
||||
"""
|
||||
for ref in reference_pixels:
|
||||
x, y = ref["position"]
|
||||
expected = ref["rgb"]
|
||||
actual = image.getpixel((x, y))[:3]
|
||||
assert all(abs(a - e) <= tolerance for a, e in zip(actual, expected)), (
|
||||
f"Pixel mismatch at ({x}, {y}): expected {expected}, got {actual}"
|
||||
)
|
||||
|
||||
|
||||
def _generate_bagel_img2img(
|
||||
omni: Omni,
|
||||
input_image: Image.Image,
|
||||
prompt: str = DEFAULT_PROMPT,
|
||||
) -> Image.Image:
|
||||
"""Generate an image using Bagel model with img2img pipeline.
|
||||
|
||||
Args:
|
||||
omni: The Omni instance to use for generation.
|
||||
input_image: The input PIL Image for img2img.
|
||||
prompt: The text prompt for image editing.
|
||||
|
||||
Returns:
|
||||
The generated PIL Image.
|
||||
|
||||
Raises:
|
||||
AssertionError: If no image is generated or size is incorrect.
|
||||
"""
|
||||
params_list = _configure_sampling_params(omni)
|
||||
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"img2img": input_image},
|
||||
"modalities": ["img2img"],
|
||||
}
|
||||
],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
generated_image = _extract_generated_image(omni_outputs)
|
||||
assert generated_image is not None, "No images generated"
|
||||
assert generated_image.size == EXPECTED_OUTPUT_SIZE, f"Expected {EXPECTED_OUTPUT_SIZE}, got {generated_image.size}"
|
||||
|
||||
return generated_image
|
||||
|
||||
|
||||
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
||||
"""Resolve deploy config based on run level.
|
||||
|
||||
For advanced_model (real weights), strip load_format: dummy so the model
|
||||
falls back to loading real weights from HuggingFace.
|
||||
"""
|
||||
if run_level == "advanced_model":
|
||||
return modify_stage_config(
|
||||
config_path,
|
||||
deletes={
|
||||
"stages": {
|
||||
0: ["load_format"],
|
||||
1: ["load_format"],
|
||||
}
|
||||
},
|
||||
)
|
||||
return config_path
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_img2img_shared_memory_connector(run_level):
|
||||
"""Test Bagel img2img with shared memory connector."""
|
||||
input_image = _load_input_image()
|
||||
config_path = _resolve_deploy_config(BAGEL_CI_DEPLOY, run_level)
|
||||
with OmniRunner(
|
||||
"ByteDance-Seed/BAGEL-7B-MoT",
|
||||
stage_configs_path=config_path,
|
||||
) as runner:
|
||||
generated_image = _generate_bagel_img2img(runner.omni, input_image)
|
||||
if run_level == "advanced_model":
|
||||
_validate_pixels(generated_image)
|
||||
@@ -1,369 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for Bagel text2img generation.
|
||||
|
||||
This test validates that the Bagel model generates images that match
|
||||
expected reference pixel values within a ±5 tolerance.
|
||||
|
||||
Equivalent to running:
|
||||
python3 examples/offline_inference/bagel/end2end.py \
|
||||
--prompts "A futuristic city skyline at twilight, cyberpunk style" \
|
||||
--modality text2img --step 15
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
BAGEL_CI_DEPLOY = get_deploy_config_path("ci/bagel.yaml")
|
||||
BAGEL_MOONCAKE_CI_DEPLOY = get_deploy_config_path("ci/bagel_mooncake.yaml")
|
||||
|
||||
# Reference pixel data extracted from the known-good output image
|
||||
# Each entry contains (x, y) position and expected (R, G, B) values
|
||||
# "Generated with seed=52, num_inference_steps=15,
|
||||
# prompt='A futuristic city skyline at twilight, cyberpunk style'"
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||
]
|
||||
|
||||
if current_omni_platform.is_rocm():
|
||||
REFERENCE_PIXELS = [
|
||||
{"position": (100, 100), "rgb": (115, 113, 94)},
|
||||
{"position": (400, 50), "rgb": (159, 160, 144)},
|
||||
{"position": (700, 100), "rgb": (164, 151, 123)},
|
||||
{"position": (150, 400), "rgb": (120, 121, 107)},
|
||||
{"position": (512, 512), "rgb": (165, 133, 127)},
|
||||
{"position": (700, 400), "rgb": (217, 130, 66)},
|
||||
{"position": (100, 700), "rgb": (191, 168, 152)},
|
||||
{"position": (400, 700), "rgb": (130, 96, 77)},
|
||||
{"position": (700, 700), "rgb": (247, 203, 140)},
|
||||
{"position": (256, 256), "rgb": (167, 156, 150)},
|
||||
]
|
||||
|
||||
# Maximum allowed difference per color channel
|
||||
PIXEL_TOLERANCE = 10
|
||||
|
||||
# Default test prompt
|
||||
DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>"
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Find and return a free ephemeral port by binding to port 0."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
s.listen(1)
|
||||
port = s.getsockname()[1]
|
||||
return port
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||
"""Configure sampling parameters for Bagel text2img generation.
|
||||
|
||||
Args:
|
||||
omni: The Omni instance to get default params from.
|
||||
num_inference_steps: Number of inference steps for the diffusion stage.
|
||||
|
||||
Returns:
|
||||
Configured sampling params list.
|
||||
"""
|
||||
params_list = omni.default_sampling_params_list
|
||||
if len(params_list) > 1:
|
||||
params_list[1].num_inference_steps = num_inference_steps # type: ignore
|
||||
params_list[1].extra_args = { # type: ignore
|
||||
"cfg_text_scale": 4.0,
|
||||
"cfg_img_scale": 1.5,
|
||||
}
|
||||
return params_list
|
||||
|
||||
|
||||
def _extract_generated_image(omni_outputs: list) -> Image.Image | None:
|
||||
"""Extract the generated image from Omni outputs.
|
||||
|
||||
Args:
|
||||
omni_outputs: List of outputs from omni.generate().
|
||||
|
||||
Returns:
|
||||
The first generated PIL Image, or None if no image found.
|
||||
"""
|
||||
for req_output in omni_outputs:
|
||||
if images := getattr(req_output, "images", None):
|
||||
return images[0]
|
||||
if hasattr(req_output, "request_output") and req_output.request_output:
|
||||
stage_out = req_output.request_output
|
||||
if hasattr(stage_out, "images") and stage_out.images:
|
||||
return stage_out.images[0]
|
||||
return None
|
||||
|
||||
|
||||
def _validate_pixels(
|
||||
image: Image.Image,
|
||||
reference_pixels: list[dict[str, Any]] = REFERENCE_PIXELS,
|
||||
tolerance: int = PIXEL_TOLERANCE,
|
||||
) -> None:
|
||||
"""Validate that image pixels match expected reference values.
|
||||
|
||||
Args:
|
||||
image: The PIL Image to validate.
|
||||
reference_pixels: List of dicts with 'position' (x, y) and 'rgb' (R, G, B).
|
||||
tolerance: Maximum allowed difference per color channel.
|
||||
|
||||
Raises:
|
||||
AssertionError: If any pixel differs beyond tolerance.
|
||||
"""
|
||||
for ref in reference_pixels:
|
||||
x, y = ref["position"]
|
||||
expected = ref["rgb"]
|
||||
actual = image.getpixel((x, y))[:3]
|
||||
assert all(abs(a - e) <= tolerance for a, e in zip(actual, expected)), (
|
||||
f"Pixel mismatch at ({x}, {y}): expected {expected}, got {actual}"
|
||||
)
|
||||
|
||||
|
||||
def _generate_bagel_image(omni: Omni, prompt: str = DEFAULT_PROMPT) -> Image.Image:
|
||||
"""Generate an image using Bagel model with configured parameters.
|
||||
|
||||
Args:
|
||||
omni: The Omni instance to use for generation.
|
||||
prompt: The text prompt for image generation.
|
||||
|
||||
Returns:
|
||||
The generated PIL Image.
|
||||
|
||||
Raises:
|
||||
AssertionError: If no image is generated or size is incorrect.
|
||||
"""
|
||||
params_list = _configure_sampling_params(omni)
|
||||
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[{"prompt": prompt, "modalities": ["image"]}],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
generated_image = _extract_generated_image(omni_outputs)
|
||||
assert generated_image is not None, "No images generated"
|
||||
assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}"
|
||||
|
||||
return generated_image
|
||||
|
||||
|
||||
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
||||
"""Resolve deploy config based on run level.
|
||||
|
||||
For advanced_model (real weights), strip load_format: dummy so the model
|
||||
falls back to loading real weights from HuggingFace.
|
||||
"""
|
||||
if run_level == "advanced_model":
|
||||
return modify_stage_config(
|
||||
config_path,
|
||||
deletes={
|
||||
"stages": {
|
||||
0: ["load_format"],
|
||||
1: ["load_format"],
|
||||
}
|
||||
},
|
||||
)
|
||||
return config_path
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_text2img_shared_memory_connector(run_level):
|
||||
"""Test Bagel text2img with shared memory connector."""
|
||||
config_path = _resolve_deploy_config(BAGEL_CI_DEPLOY, run_level)
|
||||
with OmniRunner(
|
||||
"ByteDance-Seed/BAGEL-7B-MoT",
|
||||
stage_configs_path=config_path,
|
||||
) as runner:
|
||||
generated_image = _generate_bagel_image(runner.omni)
|
||||
if run_level == "advanced_model":
|
||||
_validate_pixels(generated_image)
|
||||
|
||||
|
||||
def _wait_for_port(host: str, port: int, timeout: int = 30) -> bool:
|
||||
"""Wait for a port to become available.
|
||||
|
||||
Args:
|
||||
host: The host address.
|
||||
port: The port number.
|
||||
timeout: Maximum seconds to wait.
|
||||
|
||||
Returns:
|
||||
True if port becomes available, False otherwise.
|
||||
"""
|
||||
for _ in range(timeout):
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1):
|
||||
return True
|
||||
except (TimeoutError, ConnectionRefusedError):
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def _is_mooncake_master_available() -> bool:
|
||||
"""Check if mooncake_master binary is present and can actually execute."""
|
||||
import shutil
|
||||
|
||||
binary = shutil.which("mooncake_master")
|
||||
if binary is None:
|
||||
return False
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[binary, "--help"],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return result.returncode != 127
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return True
|
||||
|
||||
|
||||
def _cleanup_mooncake_processes(timeout_secs: int = 5) -> None:
|
||||
"""Clean up any existing mooncake_master processes.
|
||||
|
||||
Args:
|
||||
timeout_secs: Maximum seconds to wait for graceful termination.
|
||||
"""
|
||||
subprocess.run(
|
||||
["pkill", "-f", "mooncake_master"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout_secs:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", "mooncake_master"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
subprocess.run(
|
||||
["pkill", "-9", "-f", "mooncake_master"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
|
||||
"""Load Mooncake config from CI overlay and substitute placeholders.
|
||||
|
||||
Args:
|
||||
host: Mooncake host address.
|
||||
rpc_port: RPC port for Mooncake master.
|
||||
http_port: HTTP metadata server port.
|
||||
|
||||
Returns:
|
||||
Path to the temporary config file with substituted values.
|
||||
"""
|
||||
with open(BAGEL_MOONCAKE_CI_DEPLOY) as f:
|
||||
config_content = f.read()
|
||||
|
||||
config_content = config_content.replace("${MOONCAKE_HOST}", host)
|
||||
config_content = config_content.replace("${MOONCAKE_RPC_PORT}", str(rpc_port))
|
||||
config_content = config_content.replace("${MOONCAKE_HTTP_PORT}", str(http_port))
|
||||
|
||||
temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False)
|
||||
temp_file.write(config_content)
|
||||
temp_file.close()
|
||||
return temp_file.name
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_bagel_text2img_mooncake_connector(run_level):
|
||||
"""Test Bagel text2img with Mooncake connector for inter-stage communication."""
|
||||
if not _is_mooncake_master_available():
|
||||
raise RuntimeError(
|
||||
"mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)"
|
||||
)
|
||||
MOONCAKE_HOST = "127.0.0.1"
|
||||
MOONCAKE_RPC_PORT = _find_free_port()
|
||||
MOONCAKE_HTTP_PORT = _find_free_port()
|
||||
MOONCAKE_METRICS_PORT = _find_free_port()
|
||||
|
||||
mooncake_master_proc = None
|
||||
temp_config_file = None
|
||||
|
||||
try:
|
||||
_cleanup_mooncake_processes()
|
||||
|
||||
# Start mooncake_master
|
||||
mooncake_master_proc = subprocess.Popen(
|
||||
[
|
||||
"mooncake_master",
|
||||
f"--rpc_port={MOONCAKE_RPC_PORT}",
|
||||
"--enable_http_metadata_server=true",
|
||||
"--http_metadata_server_host=0.0.0.0",
|
||||
f"--http_metadata_server_port={MOONCAKE_HTTP_PORT}",
|
||||
f"--metrics_port={MOONCAKE_METRICS_PORT}",
|
||||
],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
preexec_fn=os.setsid,
|
||||
)
|
||||
|
||||
assert _wait_for_port(MOONCAKE_HOST, MOONCAKE_RPC_PORT), "mooncake_master failed to start"
|
||||
|
||||
# Create temp config and initialize Omni
|
||||
temp_config_file = _load_mooncake_config(
|
||||
host=MOONCAKE_HOST,
|
||||
rpc_port=MOONCAKE_RPC_PORT,
|
||||
http_port=MOONCAKE_HTTP_PORT,
|
||||
)
|
||||
|
||||
temp_config_file = _resolve_deploy_config(temp_config_file, run_level)
|
||||
with OmniRunner(
|
||||
"ByteDance-Seed/BAGEL-7B-MoT",
|
||||
stage_configs_path=temp_config_file,
|
||||
stage_init_timeout=300,
|
||||
) as runner:
|
||||
generated_image = _generate_bagel_image(runner.omni)
|
||||
if run_level == "advanced_model":
|
||||
_validate_pixels(generated_image)
|
||||
|
||||
finally:
|
||||
if temp_config_file:
|
||||
try:
|
||||
os.unlink(temp_config_file)
|
||||
except OSError:
|
||||
pass
|
||||
if mooncake_master_proc:
|
||||
try:
|
||||
os.killpg(os.getpgid(mooncake_master_proc.pid), signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end tests for Bagel text2text and img2text (understanding) tasks.
|
||||
|
||||
These tests validate that the Bagel multistage pipeline correctly generates
|
||||
text output for understanding tasks, matching reference results.
|
||||
|
||||
Equivalent to running:
|
||||
python3 examples/offline_inference/bagel/end2end.py \
|
||||
--modality text2text \
|
||||
--prompts "Where is the capital of France?"
|
||||
|
||||
python3 examples/offline_inference/bagel/end2end.py \
|
||||
--modality img2text \
|
||||
--prompts "Please describe this image" \
|
||||
--image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import pytest
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
STAGE_CONFIG = get_deploy_config_path("ci/bagel.yaml")
|
||||
|
||||
REFERENCE_TEXT_TEXT2TEXT = "The capital of France is Paris."
|
||||
|
||||
REFERENCE_TEXT_IMG2TEXT = (
|
||||
"This is a photo of a wooden boardwalk or pathway that leads through "
|
||||
"tall green grass. The path appears to be in a natural setting, possibly "
|
||||
"a wetland or marsh area. The sky above is blue with some scattered "
|
||||
"clouds, suggesting it might be a sunny day. The overall scene looks "
|
||||
"peaceful and serene."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_deploy_config(config_path: str, run_level: str) -> str:
|
||||
"""Strip load_format: dummy for advanced_model (real weights)."""
|
||||
if run_level == "advanced_model":
|
||||
return modify_stage_config(
|
||||
config_path,
|
||||
deletes={
|
||||
"stages": {
|
||||
0: ["load_format"],
|
||||
1: ["load_format"],
|
||||
}
|
||||
},
|
||||
)
|
||||
return config_path
|
||||
|
||||
|
||||
def _extract_text(omni_outputs: list) -> str:
|
||||
"""Extract generated text from OmniRequestOutput list."""
|
||||
for req_output in omni_outputs:
|
||||
ro = getattr(req_output, "request_output", None)
|
||||
if ro and getattr(ro, "outputs", None):
|
||||
return "".join(getattr(o, "text", "") or "" for o in ro.outputs)
|
||||
return ""
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_text2text(run_level):
|
||||
"""Test Bagel text2text produces correct text output."""
|
||||
config_path = _resolve_deploy_config(STAGE_CONFIG, run_level)
|
||||
with OmniRunner(
|
||||
MODEL_NAME,
|
||||
stage_configs_path=config_path,
|
||||
) as runner:
|
||||
omni = runner.omni
|
||||
prompt = "<|im_start|>user\nWhere is the capital of France?<|im_end|>\n<|im_start|>assistant\n"
|
||||
params_list = omni.default_sampling_params_list
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[{"prompt": prompt, "modalities": ["text"]}],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0, "No outputs returned"
|
||||
text = _extract_text(omni_outputs)
|
||||
assert len(text) > 0, "Generated text is empty"
|
||||
|
||||
if run_level == "advanced_model":
|
||||
assert text == REFERENCE_TEXT_TEXT2TEXT, (
|
||||
f"Text mismatch: expected {REFERENCE_TEXT_TEXT2TEXT!r}, got {text!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_bagel_img2text(run_level):
|
||||
"""Test Bagel img2text produces correct text output."""
|
||||
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
|
||||
config_path = _resolve_deploy_config(STAGE_CONFIG, run_level)
|
||||
with OmniRunner(
|
||||
MODEL_NAME,
|
||||
stage_configs_path=config_path,
|
||||
stage_init_timeout=300,
|
||||
) as runner:
|
||||
omni = runner.omni
|
||||
prompt = "<|im_start|>user\n<|image_pad|>\nPlease describe this image<|im_end|>\n<|im_start|>assistant\n"
|
||||
params_list = omni.default_sampling_params_list
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"image": input_image},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0, "No outputs returned"
|
||||
text = _extract_text(omni_outputs)
|
||||
assert len(text) > 0, "Generated text is empty"
|
||||
|
||||
if run_level == "advanced_model":
|
||||
assert text == REFERENCE_TEXT_IMG2TEXT, f"Text mismatch: expected {REFERENCE_TEXT_IMG2TEXT!r}, got {text!r}"
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
System test for cache-dit backend.
|
||||
|
||||
This test verifies that cache-dit acceleration works correctly with diffusion models.
|
||||
It uses minimal settings to keep test time short for CI.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Use random weights model for testing
|
||||
models = ["riverclouds/qwen_image_random"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cache
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_cache_dit(model_name: str):
|
||||
"""Test cache-dit backend with diffusion model."""
|
||||
# Configure cache-dit with minimal settings for fast testing
|
||||
cache_config = {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 2, # Minimal warmup for fast test
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
}
|
||||
with OmniRunner(
|
||||
model_name,
|
||||
cache_backend="cache_dit",
|
||||
cache_config=cache_config,
|
||||
) as runner:
|
||||
# Use minimal settings for fast testing
|
||||
height = 256
|
||||
width = 256
|
||||
num_inference_steps = 4 # Minimal steps for fast test
|
||||
|
||||
outputs = runner.omni.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1, # Single output for speed
|
||||
),
|
||||
)
|
||||
# Extract images from request_output['images']
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
if not hasattr(first_output, "request_output") or not first_output.request_output:
|
||||
raise ValueError("No request_output found in OmniRequestOutput")
|
||||
|
||||
req_out = first_output.request_output
|
||||
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
|
||||
raise ValueError("Invalid request_output structure or missing 'images' key")
|
||||
|
||||
images = req_out.images
|
||||
|
||||
# Verify generation succeeded
|
||||
assert images is not None
|
||||
assert len(images) == 1
|
||||
# Check image size
|
||||
assert images[0].width == width
|
||||
assert images[0].height == height
|
||||
+59
-65
@@ -6,8 +6,9 @@ Offline E2E smoke test for CosyVoice3 zero-shot reference inference.
|
||||
This test uses the official upstream zero-shot prompt text/audio pair and
|
||||
verifies a stable reference recipe:
|
||||
- config-derived top_p/top_k and token-length ratios
|
||||
- model EOS token as the stop token
|
||||
- model EOS token as the stop token for stage-0 (talker) sampling
|
||||
- a conservative repetition penalty to avoid degenerate loops
|
||||
- stage-1 (code2wav) completion reports a normal terminal ``finish_reason``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -23,18 +24,19 @@ from huggingface_hub import snapshot_download
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import test_asset_path
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
|
||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
|
||||
MODEL_DIR_ENV = "VLLM_OMNI_COSYVOICE3_MODEL_DIR"
|
||||
|
||||
# Vendored under tests/assets/cosyvoice3/ so the test does not depend on
|
||||
# raw.githubusercontent.com being reachable from CI runners.
|
||||
REFERENCE_PROMPT_WAV_PATH = test_asset_path("cosyvoice3/zero_shot_prompt.wav")
|
||||
REFERENCE_PROMPT_WAV_PATH = get_asset_path("cosyvoice3/zero_shot_prompt.wav")
|
||||
REFERENCE_PROMPT_TEXT = "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。"
|
||||
REFERENCE_SYNTH_TEXT = (
|
||||
"CosyVoice is undergoing a comprehensive upgrade, providing more accurate, "
|
||||
@@ -44,12 +46,6 @@ REFERENCE_STAGE0_TEMPERATURE = 1.0
|
||||
REFERENCE_STAGE0_REPETITION_PENALTY = 2.0
|
||||
|
||||
|
||||
ASYNC_CHUNK_MODES = [
|
||||
pytest.param(False, id="sync"),
|
||||
pytest.param(True, id="async_chunk"),
|
||||
]
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _load_reference_prompt_wav() -> tuple[np.ndarray, int]:
|
||||
audio, sr = sf.read(str(REFERENCE_PROMPT_WAV_PATH), dtype="float32", always_2d=False)
|
||||
@@ -120,18 +116,6 @@ def _concat_audio(audio_val) -> np.ndarray:
|
||||
return audio_np.reshape(-1)
|
||||
|
||||
|
||||
def _get_stage_engine_outputs(omni_runner: OmniRunner, stage_id: int):
|
||||
stage_list = getattr(omni_runner.omni, "stage_list", None)
|
||||
if stage_list is not None:
|
||||
return getattr(stage_list[stage_id], "engine_outputs", None) or []
|
||||
|
||||
stage_clients = getattr(getattr(omni_runner.omni, "engine", None), "stage_clients", None)
|
||||
if stage_clients is not None:
|
||||
return getattr(stage_clients[stage_id], "engine_outputs", None) or []
|
||||
|
||||
raise AttributeError("Unable to locate stage outputs on Omni runner")
|
||||
|
||||
|
||||
def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
@@ -143,57 +127,67 @@ def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[s
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
# (model, stage_configs_path, extra_omni_kwargs) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
|
||||
_cosy_deployment = get_deploy_config_path("cosyvoice3.yaml")
|
||||
_cosy_model_path = str(_resolve_model_dir())
|
||||
_OMNI_RUNNER_PARAMS = [
|
||||
pytest.param(
|
||||
(_cosy_model_path, _cosy_deployment, {"async_chunk": False}),
|
||||
id="sync",
|
||||
),
|
||||
pytest.param(
|
||||
(_cosy_model_path, _cosy_deployment, {"async_chunk": True}),
|
||||
id="async_chunk",
|
||||
),
|
||||
]
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", _OMNI_RUNNER_PARAMS, indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("async_chunk", ASYNC_CHUNK_MODES)
|
||||
def test_cosyvoice3_offline_reference_zero_shot(async_chunk: bool) -> None:
|
||||
def test_cosyvoice3_offline_reference_zero_shot(omni_runner: OmniRunner) -> None:
|
||||
"""CosyVoice3 zero-shot reference inference should stop cleanly and produce sane audio."""
|
||||
prompt_audio, prompt_sr = _load_reference_prompt_wav()
|
||||
model_dir = _resolve_model_dir()
|
||||
expected_stop_token = int(CosyVoice3Config().llm["eos_token_id"])
|
||||
|
||||
with OmniRunner(
|
||||
str(model_dir),
|
||||
seed=42,
|
||||
stage_configs_path=get_deploy_config_path("cosyvoice3.yaml"),
|
||||
async_chunk=async_chunk,
|
||||
stage_init_timeout=300,
|
||||
) as omni_runner:
|
||||
sampling_params_list = omni_runner.get_default_sampling_params_list()
|
||||
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
|
||||
sampling_params_list = omni_runner.get_default_sampling_params_list()
|
||||
sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
|
||||
|
||||
outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list)
|
||||
outputs = omni_runner.omni.generate(_build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list)
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, "No audio output found"
|
||||
assert outputs, "No outputs returned"
|
||||
audio_mm = outputs[0].multimodal_output
|
||||
assert "audio" in audio_mm, "No audio output found"
|
||||
|
||||
audio = _concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, "Generated audio is empty"
|
||||
audio = _concat_audio(audio_mm["audio"])
|
||||
assert audio.size > 0, "Generated audio is empty"
|
||||
|
||||
sr_val = audio_mm.get("sr", 24000)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
sr = int(sr_val)
|
||||
assert sr == 24000, f"Unexpected sample_rate={sr}"
|
||||
sr_val = audio_mm.get("sr", 24000)
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
sr = int(sr_val)
|
||||
assert sr == 24000, f"Unexpected sample_rate={sr}"
|
||||
|
||||
duration_s = audio.size / sr
|
||||
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
|
||||
duration_s = audio.size / sr
|
||||
assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
|
||||
|
||||
stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
|
||||
if stage0_outputs:
|
||||
completion = stage0_outputs[0].outputs[0]
|
||||
finish_reason = getattr(completion, "finish_reason", None)
|
||||
stop_reason = getattr(completion, "stop_reason", None)
|
||||
num_tokens = len(getattr(completion, "token_ids", []) or [])
|
||||
|
||||
assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
|
||||
assert int(stop_reason) == expected_stop_token, (
|
||||
f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
|
||||
)
|
||||
assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
|
||||
else:
|
||||
assert async_chunk, "Stage-0 produced no engine outputs"
|
||||
# Code2wav is ``final_output`` (stage 1); ``Omni.generate`` yields its ``request_output`` / completions.
|
||||
pipeline_out = OmniRequestOutput.unwrap_result(outputs[0])
|
||||
assert pipeline_out.stage_id == 1, f"expected final stage 1, got {pipeline_out.stage_id}"
|
||||
assert pipeline_out.final_output_type == "audio", (
|
||||
f"expected audio final_output_type, got {pipeline_out.final_output_type!r}"
|
||||
)
|
||||
ro = pipeline_out.request_output
|
||||
assert ro is not None, "stage 1 should include request_output"
|
||||
completions = getattr(ro, "outputs", None) or []
|
||||
assert completions, "stage 1 request_output should include at least one completion"
|
||||
completion = completions[0]
|
||||
finish_reason = getattr(completion, "finish_reason", None)
|
||||
assert finish_reason == "stop", f"Stage-1 finish_reason={finish_reason}, expected 'stop'"
|
||||
token_ids = getattr(completion, "token_ids", None) or []
|
||||
if token_ids:
|
||||
assert 1 <= len(token_ids) <= 4096, f"Stage-1 num_tokens={len(token_ids)}, unexpected range"
|
||||
@@ -1,122 +0,0 @@
|
||||
import gc
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
AUDIO_MODEL = {
|
||||
"stabilityai/stable-audio-open-1.0": {"cuda": 100, "rocm": None},
|
||||
}
|
||||
|
||||
IMAGE_VIDEO_MODELS = {
|
||||
"riverclouds/qwen_image_random": {"cuda": 2500, "rocm": 2100},
|
||||
}
|
||||
|
||||
MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS}
|
||||
|
||||
AUDIO_MODEL_PARAMS = {
|
||||
"runner_params": {},
|
||||
"sampler_params": {},
|
||||
}
|
||||
|
||||
IMAGE_VIDEO_MODELS_PARAMS = {
|
||||
"runner_params": {},
|
||||
"sampler_params": {
|
||||
"height": 256,
|
||||
"width": 256,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def inference(model_name: str, offload: bool = True):
|
||||
gc.collect()
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = current_omni_platform.current_device()
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
|
||||
if model_name in AUDIO_MODEL:
|
||||
params = AUDIO_MODEL_PARAMS
|
||||
else:
|
||||
params = IMAGE_VIDEO_MODELS_PARAMS
|
||||
|
||||
with OmniRunner(
|
||||
model_name,
|
||||
# TODO: we might want to add overlapped feature e2e tests
|
||||
# cache_backend="cache_dit",
|
||||
enable_cpu_offload=offload,
|
||||
**params["runner_params"],
|
||||
) as runner:
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
monitor.start()
|
||||
output = runner.omni.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
OmniDiffusionSamplingParams(
|
||||
num_inference_steps=9,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
**params["sampler_params"],
|
||||
),
|
||||
)
|
||||
peak = monitor.peak_used_mb
|
||||
monitor.stop()
|
||||
|
||||
gc.collect()
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
return peak, output
|
||||
|
||||
|
||||
def check_audio_determinism(audio1, audio2, atol=1e-2):
|
||||
device = current_omni_platform.device_type
|
||||
if isinstance(audio1, np.ndarray):
|
||||
audio1 = torch.from_numpy(audio1).to(device)
|
||||
if isinstance(audio2, np.ndarray):
|
||||
audio2 = torch.from_numpy(audio2).to(device)
|
||||
|
||||
if not torch.allclose(audio1, audio2, atol=atol):
|
||||
diff = torch.abs(audio1 - audio2)
|
||||
print(f"Max difference: {diff.max().item()}")
|
||||
print(f"Mean difference: {diff.mean().item()}")
|
||||
raise AssertionError(f"Audio outputs differ beyond tolerance atol={atol}")
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
|
||||
@pytest.mark.parametrize("model_name", list(MODELS.keys()))
|
||||
def test_cpu_offload_diffusion_model(model_name: str):
|
||||
try:
|
||||
offload_peak_memory, output_offload = inference(model_name, offload=True)
|
||||
cleanup_dist_env_and_memory()
|
||||
no_offload_peak_memory, output_no_offload = inference(model_name, offload=False)
|
||||
except Exception:
|
||||
pytest.fail("Inference failed")
|
||||
print(f"Offload peak memory: {offload_peak_memory} MB")
|
||||
print(f"No offload peak memory: {no_offload_peak_memory} MB")
|
||||
|
||||
if model_name == "stabilityai/stable-audio-open-1.0":
|
||||
audio_offload = output_offload[0].request_output.multimodal_output.get("audio")
|
||||
audio_no_offload = output_no_offload[0].request_output.multimodal_output.get("audio")
|
||||
check_audio_determinism(audio_offload, audio_no_offload, atol=1e-2)
|
||||
|
||||
# Set platform-specific VRAM saving thresholds to account
|
||||
# for varying runtime memory overhead and fragmentation between CUDA and ROCm.
|
||||
is_rocm = torch.version.hip is not None
|
||||
platform = "rocm" if is_rocm else "cuda"
|
||||
threshold = MODELS[model_name][platform]
|
||||
if threshold is None:
|
||||
pytest.skip(f"Threshold not defined for {platform} on {model_name}")
|
||||
|
||||
assert offload_peak_memory + threshold < no_offload_peak_memory, (
|
||||
f"Offload peak memory {offload_peak_memory} MB should be less than "
|
||||
f"no offload peak memory {no_offload_peak_memory} MB by {threshold} MB"
|
||||
)
|
||||
@@ -1,118 +0,0 @@
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.e2e.offline_inference.test_diffusion_cpu_offload import check_audio_determinism
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
AUDIO_MODEL = {
|
||||
"stabilityai/stable-audio-open-1.0": {"cuda": 1500, "rocm": 1500},
|
||||
}
|
||||
|
||||
IMAGE_VIDEO_MODELS = {
|
||||
"riverclouds/qwen_image_random": {"cuda": 4500, "rocm": None},
|
||||
# "Wan-AI/Wan2.2-T2V-A14B-Diffusers": {"cuda": 45000, "rocm": None},
|
||||
}
|
||||
|
||||
MODELS = {**AUDIO_MODEL, **IMAGE_VIDEO_MODELS}
|
||||
|
||||
AUDIO_MODEL_PARAMS = {
|
||||
"runner_params": {},
|
||||
"sampler_params": {},
|
||||
}
|
||||
|
||||
IMAGE_VIDEO_MODELS_PARAMS = {
|
||||
"runner_params": {"boundary_ratio": 0.875, "flow_shift": 5.0},
|
||||
"sampler_params": {"height": 480, "width": 640, "num_frames": 5},
|
||||
}
|
||||
|
||||
|
||||
def run_inference(
|
||||
model_name: str,
|
||||
layerwise_offload: bool = False,
|
||||
num_inference_steps: int = 3,
|
||||
) -> float:
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = current_omni_platform.current_device()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
|
||||
if model_name in AUDIO_MODEL:
|
||||
params = AUDIO_MODEL_PARAMS
|
||||
else:
|
||||
params = IMAGE_VIDEO_MODELS_PARAMS
|
||||
|
||||
with OmniRunner(
|
||||
model_name,
|
||||
enable_layerwise_offload=layerwise_offload,
|
||||
# TODO: we might want to add overlapped feature e2e tests
|
||||
# cache_backend="cache_dit",
|
||||
**params["runner_params"],
|
||||
) as runner:
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
|
||||
# Refer to tests/e2e/offline_inference/test_t2v_model.py
|
||||
# Use minimal settings for testing
|
||||
output = runner.omni.generate(
|
||||
"A cat sitting on a table",
|
||||
OmniDiffusionSamplingParams(
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
guidance_scale=1.0,
|
||||
num_inference_steps=num_inference_steps,
|
||||
**params["sampler_params"],
|
||||
),
|
||||
)
|
||||
|
||||
peak = monitor.peak_used_mb
|
||||
monitor.stop()
|
||||
|
||||
return peak, output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", list(MODELS.keys()))
|
||||
def test_layerwise_offload_diffusion_model(model_name: str):
|
||||
"""Test that layerwise offloading reduces GPU memory usage.
|
||||
|
||||
This test verifies that layerwise offloading significantly reduces peak
|
||||
GPU memory usage compared to loading the entire model on GPU. The layerwise
|
||||
offloader keeps only a single transformer block on GPU at a time, with
|
||||
prefetching for compute-memory overlap.
|
||||
"""
|
||||
try:
|
||||
# Run without layerwise offloading (baseline)
|
||||
no_offload_peak_memory, output_no_offload = run_inference(model_name, layerwise_offload=False)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
# Run with layerwise offloading (1 layer on device)
|
||||
layerwise_offload_peak_memory, output_offload = run_inference(model_name, layerwise_offload=True)
|
||||
cleanup_dist_env_and_memory()
|
||||
except Exception:
|
||||
pytest.fail("Inference failed")
|
||||
|
||||
print(f"Layerwise offload peak memory (1 GPU layer): {layerwise_offload_peak_memory} MB")
|
||||
print(f"No offload peak memory: {no_offload_peak_memory} MB")
|
||||
|
||||
if model_name == "stabilityai/stable-audio-open-1.0":
|
||||
audio_offload = output_offload[0].request_output.multimodal_output.get("audio")
|
||||
audio_no_offload = output_no_offload[0].request_output.multimodal_output.get("audio")
|
||||
# Match the sibling cpu-offload test's tolerance: layerwise offload moves
|
||||
# blocks across the PCIe bus on a side stream, which can perturb cuBLAS
|
||||
# algorithm selection and produce ~ULP-level drift larger than 1e-3.
|
||||
check_audio_determinism(audio_offload, audio_no_offload, atol=1e-2)
|
||||
|
||||
is_rocm = torch.version.hip is not None
|
||||
platform = "rocm" if is_rocm else "cuda"
|
||||
expected_saved_memory = MODELS[model_name][platform]
|
||||
|
||||
if expected_saved_memory is None:
|
||||
pytest.skip(f"Threshold not defined for {platform} on {model_name}")
|
||||
|
||||
# Verify that layerwise offloading significantly reduces memory usage
|
||||
# Passes only if the actual savings exceeds the expected savings
|
||||
assert layerwise_offload_peak_memory + expected_saved_memory < no_offload_peak_memory, (
|
||||
f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB "
|
||||
f"should be significantly less than no offload peak memory {no_offload_peak_memory} MB"
|
||||
)
|
||||
@@ -1,136 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
|
||||
# This test is specific to Z-Image LoRA behavior. Keep it focused on a single
|
||||
# model to reduce runtime and avoid extra downloads.
|
||||
models = ["Tongyi-MAI/Z-Image-Turbo"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_model(model_name: str, tmp_path: Path):
|
||||
def _extract_images(outputs: list[OmniRequestOutput]):
|
||||
if not outputs:
|
||||
raise ValueError("Empty outputs from Omni.generate()")
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
if not hasattr(first_output, "request_output") or not first_output.request_output:
|
||||
raise ValueError("No request_output found in OmniRequestOutput")
|
||||
|
||||
req_out = first_output.request_output
|
||||
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
|
||||
raise ValueError("Invalid request_output structure or missing 'images' key")
|
||||
return req_out.images
|
||||
|
||||
def _write_zimage_lora(adapter_dir: Path) -> str:
|
||||
adapter_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Z-Image transformer uses dim=3840 by default (see ZImageTransformer2DModel).
|
||||
dim = 3840
|
||||
module_name = "transformer.layers.0.attention.to_qkv"
|
||||
rank = 1
|
||||
lora_a = torch.zeros((rank, dim), dtype=torch.float32)
|
||||
lora_a[0, 0] = 1.0
|
||||
|
||||
# QKVParallelLinear packs (Q, K, V). With tp=1 and n_kv_heads==n_heads in Z-Image,
|
||||
# each slice is `dim`, so total out dim is `3 * dim`.
|
||||
lora_b = torch.zeros((3 * dim, rank), dtype=torch.float32)
|
||||
# Apply a visible delta to the Q slice only to keep the perturbation bounded.
|
||||
lora_b[:dim, 0] = 0.1
|
||||
|
||||
save_file(
|
||||
{
|
||||
f"base_model.model.{module_name}.lora_A.weight": lora_a,
|
||||
f"base_model.model.{module_name}.lora_B.weight": lora_b,
|
||||
},
|
||||
str(adapter_dir / "adapter_model.safetensors"),
|
||||
)
|
||||
(adapter_dir / "adapter_config.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"r": rank,
|
||||
"lora_alpha": rank,
|
||||
"target_modules": [module_name],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return str(adapter_dir)
|
||||
|
||||
with OmniRunner(model_name) as runner:
|
||||
m = runner.omni
|
||||
# high resolution may cause OOM on L4
|
||||
height = 256
|
||||
width = 256
|
||||
prompt = "a photo of a cat sitting on a laptop keyboard"
|
||||
|
||||
outputs = m.generate(
|
||||
prompt,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
images = _extract_images(outputs)
|
||||
|
||||
assert len(images) == 1
|
||||
# check image size
|
||||
assert images[0].width == width
|
||||
assert images[0].height == height
|
||||
|
||||
# Real LoRA E2E: generate again with a real on-disk PEFT adapter and
|
||||
# verify that output changes.
|
||||
if model_name == "Tongyi-MAI/Z-Image-Turbo":
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
|
||||
lora_dir = _write_zimage_lora(tmp_path / "zimage_lora")
|
||||
lora_request = LoRARequest(
|
||||
lora_name="test",
|
||||
lora_int_id=stable_lora_int_id(lora_dir),
|
||||
lora_path=lora_dir,
|
||||
)
|
||||
outputs_lora = m.generate(
|
||||
prompt,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
lora_request=lora_request,
|
||||
lora_scale=2.0,
|
||||
),
|
||||
)
|
||||
images_lora = _extract_images(outputs_lora)
|
||||
assert len(images_lora) == 1
|
||||
assert images_lora[0].width == width
|
||||
assert images_lora[0].height == height
|
||||
|
||||
import numpy as np
|
||||
|
||||
diff = np.abs(np.array(images[0], dtype=np.int16) - np.array(images_lora[0], dtype=np.int16)).mean()
|
||||
assert diff > 0.0
|
||||
+4
-3
@@ -4,7 +4,7 @@
|
||||
E2E offline smoke tests for Dynin-Omni.
|
||||
|
||||
- model: "snu-aidas/Dynin-Omni"
|
||||
- stage config: tests/e2e/stage_configs/dynin_omni_ci.yaml
|
||||
- stage config: vllm_omni/deploy/dynin_omni_ci.yaml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,13 +19,14 @@ import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_DEFAULT_DYNIN_CONFIG_PATH: Path | None = None
|
||||
_DEFAULT_STAGE_CONFIG_PATH = _REPO_ROOT / "tests" / "e2e" / "stage_configs" / "dynin_omni_ci.yaml"
|
||||
_DEFAULT_STAGE_CONFIG_PATH = Path(get_deploy_config_path("dynin_omni_ci.yaml"))
|
||||
|
||||
models = ["snu-aidas/Dynin-Omni"]
|
||||
stage_configs = [str(_DEFAULT_STAGE_CONFIG_PATH)]
|
||||
@@ -34,7 +35,7 @@ test_params = [(model, stage_config) for model in models for stage_config in sta
|
||||
DYNIN_CONFIG_PATH = str(_DEFAULT_DYNIN_CONFIG_PATH) if _DEFAULT_DYNIN_CONFIG_PATH is not None else None
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.omni,
|
||||
pytest.mark.parametrize("omni_runner", test_params, indirect=True),
|
||||
]
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for ErnieImage text-to-image generation.
|
||||
|
||||
Equivalent to running:
|
||||
vllm serve baidu/ERNIE-Image --omni
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "baidu/ERNIE-Image"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_ernie_image_text_to_image():
|
||||
with OmniRunner(
|
||||
MODEL,
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
),
|
||||
enable_cpu_offload=True,
|
||||
) as runner:
|
||||
omni_outputs = list(
|
||||
runner.omni.generate(
|
||||
prompts=["A photo of a cat sitting on a laptop"],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=4.0,
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0
|
||||
output = omni_outputs[0]
|
||||
images = None
|
||||
if output.images:
|
||||
images = output.images
|
||||
elif hasattr(output, "request_output") and output.request_output:
|
||||
for stage_out in output.request_output:
|
||||
if hasattr(stage_out, "images") and stage_out.images:
|
||||
images = stage_out.images
|
||||
break
|
||||
|
||||
assert images is not None
|
||||
assert len(images) > 0
|
||||
assert isinstance(images[0], Image.Image)
|
||||
assert images[0].size == (512, 512)
|
||||
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for ErnieImage text-to-image generation.
|
||||
|
||||
Equivalent to running:
|
||||
vllm serve baidu/ERNIE-Image --omni
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunnerHandler
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "baidu/ERNIE-Image"
|
||||
|
||||
# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL,
|
||||
None,
|
||||
{
|
||||
"parallel_config": DiffusionParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
),
|
||||
"enable_cpu_offload": True,
|
||||
},
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_ernie_image_text_to_image(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
request_config = {
|
||||
"model": omni_runner_handler.runner.model_name,
|
||||
"prompt": "A photo of a cat sitting on a laptop",
|
||||
"sampling_params": OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=4.0,
|
||||
seed=42,
|
||||
),
|
||||
}
|
||||
omni_runner_handler.send_diffusion_request(request_config)
|
||||
@@ -1,247 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
System test for Expert Parallel (EP).
|
||||
|
||||
Tests verify that EP inference produces correct outputs compared to baseline.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import os
|
||||
import time
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Test configuration
|
||||
MODELS = ["tencent/HunyuanImage-3.0"]
|
||||
PROMPT = "A brown and white dog is running on the grass"
|
||||
DEFAULT_HEIGHT = 1024
|
||||
DEFAULT_WIDTH = 1024
|
||||
DEFAULT_SEED = 1234
|
||||
DEFAULT_STEPS = 50
|
||||
DEFAULT_GUIDANCE_SCALE = 0.0
|
||||
COS_SIM_MEAN_THRESHOLD = 1e-2
|
||||
COS_SIM_MAX_THRESHOLD = 1e-2
|
||||
MSE_THRESHOLD = 1e-2
|
||||
DIFF_MEAN_THRESHOLD = 5e-2
|
||||
DIFF_MAX_THRESHOLD = 1
|
||||
TOTAL_CARD_NUM = 8
|
||||
|
||||
|
||||
class InferenceResult(NamedTuple):
|
||||
"""Result of an inference run."""
|
||||
|
||||
images: list[Image.Image]
|
||||
elapsed_ms: float
|
||||
|
||||
|
||||
def _cleanup_distributed():
|
||||
"""Clean up distributed environment and GPU resources."""
|
||||
if dist.is_initialized():
|
||||
dist.destroy_process_group()
|
||||
|
||||
for key in ["MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"]:
|
||||
os.environ.pop(key, None)
|
||||
|
||||
gc.collect()
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float, float, float, float]:
|
||||
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
|
||||
ta = torch.from_numpy(np.asarray(a.convert("RGB"), dtype=np.float32) / 255.0)
|
||||
tb = torch.from_numpy(np.asarray(b.convert("RGB"), dtype=np.float32) / 255.0)
|
||||
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
|
||||
abs_diff = torch.abs(ta - tb)
|
||||
|
||||
vec_a = ta.reshape(-1)
|
||||
vec_b = tb.reshape(-1)
|
||||
cos_sim = torch.nn.functional.cosine_similarity(vec_a, vec_b, dim=0, eps=1e-8)
|
||||
mse = torch.nn.functional.mse_loss(ta, tb)
|
||||
return abs_diff.mean().item(), abs_diff.max().item(), cos_sim.mean().item(), cos_sim.max().item(), mse.item()
|
||||
|
||||
|
||||
def _run_inference(
|
||||
model_name: str,
|
||||
tensor_parallel_size: int,
|
||||
enable_expert_parallel: bool,
|
||||
guidance_scale: float = DEFAULT_GUIDANCE_SCALE,
|
||||
height: int = DEFAULT_HEIGHT,
|
||||
width: int = DEFAULT_WIDTH,
|
||||
seed: int = DEFAULT_SEED,
|
||||
warmup: bool = True,
|
||||
) -> InferenceResult:
|
||||
"""Run inference with specified configuration.
|
||||
|
||||
Args:
|
||||
warmup: If True, run one warmup iteration before the timed run.
|
||||
"""
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enable_expert_parallel=enable_expert_parallel,
|
||||
)
|
||||
try:
|
||||
with OmniRunner(model_name, parallel_config=parallel_config) as runner:
|
||||
omni = runner.omni
|
||||
# Warmup run (not timed)
|
||||
if warmup:
|
||||
_ = omni.generate(
|
||||
PROMPT,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=DEFAULT_STEPS,
|
||||
guidance_scale=guidance_scale,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
# Timed run
|
||||
start = time.time()
|
||||
outputs = omni.generate(
|
||||
PROMPT,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=DEFAULT_STEPS,
|
||||
guidance_scale=guidance_scale,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
elapsed_ms = (time.time() - start) * 1000
|
||||
|
||||
return InferenceResult(
|
||||
images=outputs[0].images,
|
||||
elapsed_ms=elapsed_ms,
|
||||
)
|
||||
finally:
|
||||
_cleanup_distributed()
|
||||
|
||||
|
||||
# EP configurations: (tensor_parallel_size, enable_ep, height, width, is_perf_test)
|
||||
EP_TEST_CONFIG = [
|
||||
(TOTAL_CARD_NUM, True, DEFAULT_HEIGHT, DEFAULT_WIDTH, True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": TOTAL_CARD_NUM})
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_ep(model_name):
|
||||
baseline_cache: dict[str, InferenceResult] = {}
|
||||
results: list[dict] = []
|
||||
# baseline_key = PROMPT + "_" + str(DEFAULT_HEIGHT) + "*" + str(DEFAULT_WIDTH) + "_" + str(DEFAULT_SEED) + "_" + str(DEFAULT_STEPS)
|
||||
# baseline_cache[baseline_key] = _run_inference(
|
||||
# model_name=model_name,
|
||||
# tensor_parallel_size=TOTAL_CARD_NUM,
|
||||
# enable_expert_parallel=False,
|
||||
# warmup=True,
|
||||
# )
|
||||
device_count = current_omni_platform.get_device_count()
|
||||
|
||||
print("\n" + "=" * 90)
|
||||
print(f"Sequence Parallel Test - Model: {model_name}")
|
||||
print(f"Available GPUs: {device_count}")
|
||||
print("=" * 90)
|
||||
|
||||
for tensor_parallel_size, enable_ep, height, width, is_perf_test in EP_TEST_CONFIG:
|
||||
cache_key = PROMPT + "_" + str(height) + "*" + str(width) + "_" + str(DEFAULT_SEED) + "_" + str(DEFAULT_STEPS)
|
||||
if cache_key not in baseline_cache:
|
||||
print(
|
||||
f"\n--- Running baseline{{ prompt: {PROMPT}, resolution: {height}x{width}, seed: {DEFAULT_SEED}, steps: {DEFAULT_STEPS} }}---"
|
||||
)
|
||||
baseline = _run_inference(
|
||||
model_name=model_name,
|
||||
tensor_parallel_size=TOTAL_CARD_NUM,
|
||||
enable_expert_parallel=False,
|
||||
warmup=True,
|
||||
)
|
||||
assert len(baseline.images) == 1
|
||||
baseline_cache[cache_key] = baseline
|
||||
print(f"[baseline] {height}x{width}: {baseline.elapsed_ms:.0f}ms")
|
||||
else:
|
||||
baseline = baseline_cache[cache_key]
|
||||
|
||||
print(f"\n--- Running test: {{ enable_ep: {enable_ep} }} ---")
|
||||
infer_result = _run_inference(
|
||||
model_name=model_name,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enable_expert_parallel=enable_ep,
|
||||
warmup=is_perf_test,
|
||||
)
|
||||
assert len(infer_result.images) == 1
|
||||
mean_diff, max_diff, cos_sim_mean, cos_sim_max, mse = _diff_metrics(baseline.images[0], infer_result.images[0])
|
||||
print(f"{cos_sim_mean}, {cos_sim_max}, {mse}")
|
||||
|
||||
# Build result entry
|
||||
result = {
|
||||
"prompt": PROMPT,
|
||||
"enable_ep": enable_ep,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"baseline_ms": baseline.elapsed_ms,
|
||||
"ep_ms": infer_result.elapsed_ms,
|
||||
"mean_diff": mean_diff,
|
||||
"max_diff": max_diff,
|
||||
"cos_sim_mean": cos_sim_mean,
|
||||
"cos_sim_max": cos_sim_max,
|
||||
"mse": mse,
|
||||
"is_perf_test": is_perf_test,
|
||||
}
|
||||
results.append(result)
|
||||
|
||||
# Output based on test type
|
||||
if is_perf_test:
|
||||
speedup = baseline.elapsed_ms / infer_result.elapsed_ms if infer_result.elapsed_ms > 0 else 0
|
||||
result["speedup"] = speedup
|
||||
print(
|
||||
f"[enable_ep: {enable_ep}] {tensor_parallel_size} GPUs | "
|
||||
f"baseline: {baseline.elapsed_ms:.0f}ms, ep: {infer_result.elapsed_ms:.0f}ms, "
|
||||
f"speedup: {speedup:.2f}x"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"[enable_ep: {enable_ep}] {tensor_parallel_size} GPUs | ep: {infer_result.elapsed_ms:.0f}ms (correctness only)"
|
||||
)
|
||||
|
||||
print(
|
||||
f"[enable_ep: {enable_ep}] diff: [mean={mean_diff:.6e}, max={max_diff:.6e}], cos_sim: [mean={cos_sim_mean:.6e}, max={cos_sim_max:.6e}], mse: {mse:.6e}"
|
||||
)
|
||||
assert mean_diff <= DIFF_MEAN_THRESHOLD and max_diff <= DIFF_MAX_THRESHOLD, (
|
||||
f"[enable_ep: {enable_ep}] output differs from baseline: mean={mean_diff:.6e}, max={max_diff:.6e}"
|
||||
)
|
||||
|
||||
print("\n" + "=" * 90)
|
||||
print("SUMMARY")
|
||||
print("=" * 90)
|
||||
print(f"{'Mode':<15} {'GPUs':<6} {'Size':<10} {'Baseline':<12} {'EP':<12} {'Speedup':<10} {'Status'}")
|
||||
print("-" * 90)
|
||||
for r in results:
|
||||
speedup_str = f"{r['speedup']:.2f}x" if r.get("speedup") else "N/A"
|
||||
baseline_str = f"{r['baseline_ms']:.0f}ms" if r["is_perf_test"] else "N/A"
|
||||
status = "PASS" if r["mean_diff"] <= DIFF_MEAN_THRESHOLD else "FAIL"
|
||||
print(
|
||||
f"{r['prompt']:<15.10} {r['enable_ep']:<6} {r['height']}x{r['width']:<5} "
|
||||
f"{baseline_str:<12} {r['ep_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
|
||||
)
|
||||
print("=" * 90)
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for Flux1 Schnell."""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.1-schnell"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux_schnell_text_to_image():
|
||||
"""Test FLUX.1-schnell text-to-image generation."""
|
||||
omni = Omni(model=MODEL)
|
||||
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=["A photo of a cat sitting on a laptop"],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0
|
||||
images = omni_outputs[0].images
|
||||
assert len(images) == 1
|
||||
assert isinstance(images[0], Image.Image)
|
||||
assert images[0].size == (512, 512)
|
||||
+39
-23
@@ -5,11 +5,16 @@
|
||||
These tests require:
|
||||
- A CUDA GPU
|
||||
- The quantized model checkpoint (vllm-project-org/FLUX.1-dev-AutoRound-w4a16)
|
||||
|
||||
``test_flux_autoround_w4a16_generates_image`` uses the shared ``omni_runner`` fixture
|
||||
(quantized model). ``test_flux_autoround_w4a16_memory_savings`` must load two different
|
||||
checkpoints in sequence, so it keeps explicit ``OmniRunner`` context managers.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import os as _os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
@@ -21,6 +26,10 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = pytest.mark.skip(
|
||||
reason="Current Hugging Face account is not yet authorized to access this model.",
|
||||
)
|
||||
|
||||
QUANTIZED_MODEL = "vllm-project-org/FLUX.1-dev-AutoRound-w4a16"
|
||||
BASELINE_MODEL = "black-forest-labs/FLUX.1-dev"
|
||||
|
||||
@@ -32,6 +41,27 @@ HEIGHT = 256
|
||||
WIDTH = 256
|
||||
NUM_STEPS = 2 # minimal for smoke-test
|
||||
|
||||
# Indirect param: (model_id, stage_config_path, extra OmniRunner / Omni kwargs)
|
||||
_OMNI_RUNNER_PARAM = (QUANTIZED_MODEL, None, {"enforce_eager": True})
|
||||
|
||||
|
||||
def _sampling_params() -> OmniDiffusionSamplingParams:
|
||||
return OmniDiffusionSamplingParams(
|
||||
height=HEIGHT,
|
||||
width=WIDTH,
|
||||
num_inference_steps=NUM_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
)
|
||||
|
||||
|
||||
def _first_request_images(outputs) -> list:
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
req_out = first_output.request_output
|
||||
assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
|
||||
return req_out.images
|
||||
|
||||
|
||||
def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
|
||||
"""Load a FLUX model, generate one image, return (images, peak_memory_mb)."""
|
||||
@@ -44,25 +74,12 @@ def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
|
||||
|
||||
with OmniRunner(model_name, enforce_eager=True, **extra_kwargs) as runner:
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
outputs = runner.omni.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
OmniDiffusionSamplingParams(
|
||||
height=HEIGHT,
|
||||
width=WIDTH,
|
||||
num_inference_steps=NUM_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
),
|
||||
)
|
||||
outputs = runner.omni.generate("a photo of a cat sitting on a laptop keyboard", _sampling_params())
|
||||
|
||||
peak = monitor.peak_used_mb
|
||||
monitor.stop()
|
||||
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
req_out = first_output.request_output
|
||||
assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
|
||||
images = req_out.images
|
||||
images = _first_request_images(outputs)
|
||||
|
||||
gc.collect()
|
||||
current_omni_platform.empty_cache()
|
||||
@@ -70,26 +87,25 @@ def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
|
||||
return images, peak
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_flux_autoround_w4a16_generates_image():
|
||||
@pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)
|
||||
def test_flux_autoround_w4a16_generates_image(omni_runner: OmniRunner):
|
||||
"""Load the W4A16 quantized FLUX model and verify it produces a valid image."""
|
||||
images, _ = _generate_image(QUANTIZED_MODEL)
|
||||
outputs = omni_runner.omni.generate("a photo of a cat sitting on a laptop keyboard", _sampling_params())
|
||||
images = _first_request_images(outputs)
|
||||
|
||||
assert len(images) >= 1, "Expected at least one generated image"
|
||||
img = images[0]
|
||||
assert img.width == WIDTH, f"Expected width {WIDTH}, got {img.width}"
|
||||
assert img.height == HEIGHT, f"Expected height {HEIGHT}, got {img.height}"
|
||||
|
||||
# Sanity: image should not be blank (all-zero)
|
||||
import numpy as np
|
||||
|
||||
arr = np.array(img)
|
||||
assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_flux_autoround_w4a16_memory_savings():
|
||||
@@ -98,7 +114,7 @@ def test_flux_autoround_w4a16_memory_savings():
|
||||
The W4A16 model should use meaningfully less memory than the
|
||||
BF16/FP16 baseline since weights are 4-bit instead of 16-bit.
|
||||
"""
|
||||
quant_images, quant_peak = _generate_image(QUANTIZED_MODEL)
|
||||
_, quant_peak = _generate_image(QUANTIZED_MODEL)
|
||||
cleanup_dist_env_and_memory()
|
||||
_, baseline_peak = _generate_image(BASELINE_MODEL)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Tests for FluxKontext model pipeline.
|
||||
|
||||
FluxKontext is a text-to-image and image-to-image diffusion model that supports:
|
||||
- Text-to-image generation
|
||||
- Image editing with text guidance
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.helpers.runtime import OmniRunnerHandler
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.1-Kontext-dev"
|
||||
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL,
|
||||
None,
|
||||
{
|
||||
"parallel_config": DiffusionParallelConfig(tensor_parallel_size=2),
|
||||
"enable_cpu_offload": False,
|
||||
},
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _sampling_512() -> OmniDiffusionSamplingParams:
|
||||
return OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
|
||||
def test_flux_kontext_text_to_image(omni_runner_handler: OmniRunnerHandler):
|
||||
"""Test FluxKontext text-to-image generation with real model."""
|
||||
request_config = {
|
||||
"model": MODEL,
|
||||
"prompt": "A photo of a cat sitting on a laptop",
|
||||
"sampling_params": _sampling_512(),
|
||||
}
|
||||
omni_runner_handler.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
def test_flux_kontext_image_edit(omni_runner_handler: OmniRunnerHandler):
|
||||
"""Test FluxKontext image-to-image editing with real model."""
|
||||
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
|
||||
request_config = {
|
||||
"prompt": "Transform this image into a Vincent van Gogh style painting",
|
||||
"multi_modal_data": {"img2img": input_image},
|
||||
"modalities": ["img2img"],
|
||||
"sampling_params": _sampling_512(),
|
||||
}
|
||||
omni_runner_handler.send_diffusion_request(request_config)
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for Flux1 Schnell."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.runtime import OmniRunnerHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.1-schnell"
|
||||
|
||||
_OMNI_RUNNER_PARAM = (MODEL, None)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def test_flux_schnell_text_to_image(omni_runner_handler: OmniRunnerHandler):
|
||||
"""Test FLUX.1-schnell text-to-image generation."""
|
||||
request_config = {
|
||||
"model": MODEL,
|
||||
"prompt": "A photo of a cat sitting on a laptop",
|
||||
"sampling_params": OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
),
|
||||
}
|
||||
omni_runner_handler.send_diffusion_request(request_config)
|
||||
@@ -4,31 +4,26 @@
|
||||
"""
|
||||
End-to-end test for Flux2 Klein inpainting.
|
||||
|
||||
Inpainting uses ``omni_runner_handler.send_diffusion_request`` with
|
||||
``multi_modal_data`` containing ``image`` and ``mask_image``; see
|
||||
:meth:`OmniRunnerHandler.send_diffusion_request` in ``tests.helpers.runtime``.
|
||||
"""
|
||||
|
||||
# ruff: noqa: E402
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import DiffusionResponse, OmniRunnerHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.2-klein-4B"
|
||||
|
||||
_OMNI_RUNNER_PARAM = (MODEL, None)
|
||||
|
||||
pytestmark = [pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)]
|
||||
|
||||
_HEIGHT = 512
|
||||
_WIDTH = 512
|
||||
_NUM_INFERENCE_STEPS = 4
|
||||
@@ -49,50 +44,56 @@ def _create_test_inputs(color: tuple = (100, 150, 200)):
|
||||
return _create_test_image(_WIDTH, _HEIGHT, color), _create_test_mask(_WIDTH, _HEIGHT)
|
||||
|
||||
|
||||
def _extract_images_from_output(outputs: list) -> list[Image.Image]:
|
||||
images = []
|
||||
for req_output in outputs:
|
||||
if hasattr(req_output, "images") and req_output.images:
|
||||
images.extend(req_output.images)
|
||||
elif hasattr(req_output, "request_output") and req_output.request_output:
|
||||
stage_out = req_output.request_output
|
||||
if isinstance(stage_out, OmniRequestOutput) and hasattr(stage_out, "images"):
|
||||
images.extend(stage_out.images)
|
||||
elif isinstance(stage_out, list):
|
||||
for s in stage_out:
|
||||
if hasattr(s, "images") and s.images:
|
||||
images.extend(s.images)
|
||||
return images
|
||||
def _images_from_response(response: DiffusionResponse) -> list[Image.Image]:
|
||||
if isinstance(response.images[0], list):
|
||||
return [f for fr in response.images for f in fr]
|
||||
return list(response.images)
|
||||
|
||||
|
||||
def _send_inpaint_with_generator(
|
||||
omni_runner_handler: OmniRunnerHandler, prompt: str, input_image, mask_image, generator: torch.Generator
|
||||
) -> DiffusionResponse:
|
||||
return omni_runner_handler.send_diffusion_request(
|
||||
{
|
||||
"model": MODEL,
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
"sampling_params": OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=generator,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# Regression test for https://github.com/vllm-project/vllm-omni/issues/3097
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_can_accept_text_inputs():
|
||||
model = Omni(model=MODEL)
|
||||
outputs = model.generate(
|
||||
"a cup of coffee on the table",
|
||||
OmniDiffusionSamplingParams(num_inference_steps=2, seed=42),
|
||||
def test_flux2_klein_can_accept_text_inputs(omni_runner_handler: OmniRunnerHandler):
|
||||
omni_runner_handler.send_diffusion_request(
|
||||
{
|
||||
"model": MODEL,
|
||||
"prompt": "a cup of coffee on the table",
|
||||
"sampling_params": OmniDiffusionSamplingParams(num_inference_steps=2, seed=42),
|
||||
}
|
||||
)
|
||||
assert len(outputs[0].images) == 1
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_inpaint_basic():
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=MODEL)
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
|
||||
outputs = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "Fill in the masked area with a beautiful garden",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_basic(omni_runner_handler: OmniRunnerHandler):
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
omni_runner_handler.send_diffusion_request(
|
||||
{
|
||||
"model": MODEL,
|
||||
"prompt": "Fill in the masked area with a beautiful garden",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
"sampling_params": OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
@@ -100,128 +101,45 @@ def test_flux2_klein_inpaint_basic():
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
images = _extract_images_from_output(list(outputs))
|
||||
assert len(images) == 1
|
||||
assert images[0].size == (_WIDTH, _HEIGHT)
|
||||
finally:
|
||||
if m is not None and hasattr(m, "close"):
|
||||
m.close()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_inpaint_deterministic():
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=MODEL)
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
seed = 12345
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandler):
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
seed = 12345
|
||||
|
||||
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
|
||||
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
|
||||
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
|
||||
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(seed)
|
||||
|
||||
outputs1 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A red flower in a field",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen1,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
r1 = _send_inpaint_with_generator(omni_runner_handler, "A red flower in a field", input_image, mask_image, gen1)
|
||||
r2 = _send_inpaint_with_generator(omni_runner_handler, "A red flower in a field", input_image, mask_image, gen2)
|
||||
|
||||
outputs2 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A red flower in a field",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen2,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
images1 = _images_from_response(r1)
|
||||
images2 = _images_from_response(r2)
|
||||
|
||||
images1 = _extract_images_from_output(list(outputs1))
|
||||
images2 = _extract_images_from_output(list(outputs2))
|
||||
|
||||
assert len(images1) == 1
|
||||
assert len(images2) == 1
|
||||
|
||||
assert list(images1[0].getdata()) == list(images2[0].getdata()), (
|
||||
"Same input with same seed should produce identical output. "
|
||||
"This is critical for offline/online consistency."
|
||||
)
|
||||
finally:
|
||||
if m is not None and hasattr(m, "close"):
|
||||
m.close()
|
||||
assert list(images1[0].getdata()) == list(images2[0].getdata()), (
|
||||
"Same input with same seed should produce identical output. This is critical for offline/online consistency."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux2_klein_inpaint_different_seeds_different_output():
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=MODEL)
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_different_seeds_different_output(omni_runner_handler: OmniRunnerHandler):
|
||||
input_image, mask_image = _create_test_inputs()
|
||||
|
||||
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(42)
|
||||
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(99999)
|
||||
gen1 = torch.Generator(current_omni_platform.device_type).manual_seed(42)
|
||||
gen2 = torch.Generator(current_omni_platform.device_type).manual_seed(99999)
|
||||
|
||||
outputs1 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A beautiful landscape",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen1,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
r1 = _send_inpaint_with_generator(omni_runner_handler, "A beautiful landscape", input_image, mask_image, gen1)
|
||||
r2 = _send_inpaint_with_generator(omni_runner_handler, "A beautiful landscape", input_image, mask_image, gen2)
|
||||
|
||||
outputs2 = m.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A beautiful landscape",
|
||||
"multi_modal_data": {"image": input_image, "mask_image": mask_image},
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=_HEIGHT,
|
||||
width=_WIDTH,
|
||||
num_inference_steps=_NUM_INFERENCE_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=gen2,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
images1 = _images_from_response(r1)
|
||||
images2 = _images_from_response(r2)
|
||||
|
||||
images1 = _extract_images_from_output(list(outputs1))
|
||||
images2 = _extract_images_from_output(list(outputs2))
|
||||
|
||||
assert len(images1) == 1
|
||||
assert len(images2) == 1
|
||||
|
||||
different_pixel_count = sum(1 for p1, p2 in zip(images1[0].getdata(), images2[0].getdata()) if p1 != p2)
|
||||
assert different_pixel_count > 0, "Different seeds should produce different outputs"
|
||||
finally:
|
||||
if m is not None and hasattr(m, "close"):
|
||||
m.close()
|
||||
different_pixel_count = sum(1 for p1, p2 in zip(images1[0].getdata(), images2[0].getdata()) if p1 != p2)
|
||||
assert different_pixel_count > 0, "Different seeds should produce different outputs"
|
||||
@@ -1,107 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Tests for FluxKontext model pipeline.
|
||||
|
||||
FluxKontext is a text-to-image and image-to-image diffusion model that supports:
|
||||
- Text-to-image generation
|
||||
- Image editing with text guidance
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "black-forest-labs/FLUX.1-Kontext-dev"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux_kontext_text_to_image():
|
||||
"""Test FluxKontext text-to-image generation with real model."""
|
||||
with OmniRunner(
|
||||
MODEL,
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
),
|
||||
enable_cpu_offload=False,
|
||||
) as runner:
|
||||
omni_outputs = list(
|
||||
runner.omni.generate(
|
||||
prompts=["A photo of a cat sitting on a laptop"],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0
|
||||
output = omni_outputs[0]
|
||||
images = None
|
||||
if output.images:
|
||||
images = output.images
|
||||
elif hasattr(output, "request_output") and output.request_output:
|
||||
for stage_out in output.request_output:
|
||||
if hasattr(stage_out, "images") and stage_out.images:
|
||||
images = stage_out.images
|
||||
break
|
||||
|
||||
assert images is not None
|
||||
assert len(images) > 0
|
||||
assert isinstance(images[0], Image.Image)
|
||||
assert images[0].size == (512, 512)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
def test_flux_kontext_image_edit():
|
||||
"""Test FluxKontext image-to-image editing with real model."""
|
||||
input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
|
||||
with OmniRunner(
|
||||
MODEL,
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
),
|
||||
enable_cpu_offload=False,
|
||||
) as runner:
|
||||
omni_outputs = list(
|
||||
runner.omni.generate(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "Transform this image into a Vincent van Gogh style painting",
|
||||
"multi_modal_data": {"img2img": input_image},
|
||||
"modalities": ["img2img"],
|
||||
}
|
||||
],
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=2,
|
||||
seed=42,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert len(omni_outputs) > 0
|
||||
output = omni_outputs[0]
|
||||
images = None
|
||||
if output.images:
|
||||
images = output.images
|
||||
elif hasattr(output, "request_output") and output.request_output:
|
||||
for stage_out in output.request_output:
|
||||
if hasattr(stage_out, "images") and stage_out.images:
|
||||
images = stage_out.images
|
||||
break
|
||||
|
||||
assert images is not None
|
||||
assert len(images) > 0
|
||||
assert isinstance(images[0], Image.Image)
|
||||
assert images[0].size == (512, 512)
|
||||
File renamed without changes.
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Offline E2E test for InternVLA-A1 open-loop inference."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
EXAMPLE_SCRIPT = REPO_ROOT / "examples" / "offline_inference" / "internvla_a1" / "end2end.py"
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
pytest.skip(f"{name} is not set")
|
||||
return value
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_internvla_a1_offline_open_loop(run_level: str) -> None:
|
||||
if run_level != "advanced_model":
|
||||
pytest.skip("InternVLA-A1 offline evaluation requires real local checkpoints and dataset.")
|
||||
|
||||
model_dir = _required_env("INTERNVLA_A1_MODEL_DIR")
|
||||
dataset_dir = _required_env("INTERNVLA_A1_DATASET_DIR")
|
||||
processor_dir = _required_env("INTERNVLA_A1_PROCESSOR_DIR")
|
||||
cosmos_dir = _required_env("INTERNVLA_A1_COSMOS_DIR")
|
||||
|
||||
env = os.environ.copy()
|
||||
env["INTERNVLA_A1_PROCESSOR_DIR"] = processor_dir
|
||||
env["INTERNVLA_A1_COSMOS_DIR"] = cosmos_dir
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="internvla_a1_e2e_") as tmpdir:
|
||||
output_dir = Path(tmpdir) / "outputs"
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(EXAMPLE_SCRIPT),
|
||||
"--model-dir",
|
||||
model_dir,
|
||||
"--dataset-dir",
|
||||
dataset_dir,
|
||||
"--output-dir",
|
||||
str(output_dir),
|
||||
"--num-samples",
|
||||
"1",
|
||||
"--num-episodes",
|
||||
"1",
|
||||
"--dtype",
|
||||
"float32",
|
||||
"--attn-implementation",
|
||||
"eager",
|
||||
"--skip-plots",
|
||||
]
|
||||
subprocess.run(cmd, check=True, env=env)
|
||||
|
||||
summary_path = output_dir / "summary.json"
|
||||
assert summary_path.exists(), f"Missing summary file: {summary_path}"
|
||||
|
||||
with open(summary_path, encoding="utf-8") as f:
|
||||
summary = json.load(f)
|
||||
|
||||
assert summary["mode"] == "registry"
|
||||
assert summary["attn_implementation"] == "eager"
|
||||
assert summary["dtype"] == "float32"
|
||||
assert len(summary["results"]) == 1
|
||||
assert "registry" in summary["eval_summaries"]
|
||||
|
||||
eval_summary = summary["eval_summaries"]["registry"]
|
||||
assert eval_summary["num_episodes"] == 1
|
||||
assert eval_summary["average_mse"] is not None
|
||||
assert eval_summary["average_mae"] is not None
|
||||
|
||||
log_path = output_dir / "registry" / "log.json"
|
||||
assert log_path.exists(), f"Missing evaluation log: {log_path}"
|
||||
@@ -1,243 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
|
||||
I2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py"
|
||||
|
||||
T2V_PROMPT = (
|
||||
"At sunrise, a glowing paper lantern boat drifts through a narrow canal between mossy stone walls, "
|
||||
"soft fog above the water, the camera slowly gliding forward as golden reflections shimmer across "
|
||||
"the ripples, cinematic, realistic, highly detailed."
|
||||
)
|
||||
T2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion, distorted, oversaturated, artifacts"
|
||||
I2V_PROMPT = "A cinematic dolly shot of a boat drifting on calm water at sunset"
|
||||
I2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion"
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
|
||||
def _get_ltx2_model() -> str:
|
||||
return os.environ.get("VLLM_TEST_LTX2_MODEL", "Lightricks/LTX-2")
|
||||
|
||||
|
||||
def _md5(path: Path) -> str:
|
||||
digest = hashlib.md5(usedforsecurity=False)
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _make_deterministic_test_image(path: Path) -> None:
|
||||
"""Create a deterministic 256x256 test image for I2V tests."""
|
||||
rng = np.random.RandomState(42)
|
||||
img = Image.fromarray(rng.randint(0, 255, (256, 256, 3), dtype=np.uint8))
|
||||
img.save(path)
|
||||
|
||||
|
||||
def _run_and_check(cmd: list[str], env: dict, output_path: Path, expected_md5: str) -> None:
|
||||
result = subprocess.run(cmd, cwd=REPO_ROOT, env=env, capture_output=True, text=True, check=False)
|
||||
assert result.returncode == 0, (
|
||||
f"Command failed (exit {result.returncode}).\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
generated_md5 = _md5(output_path)
|
||||
assert generated_md5 == expected_md5, (
|
||||
f"Unexpected output md5: {generated_md5} != {expected_md5}.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
|
||||
|
||||
# ── T2V tests ──
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_ltx2_t2v_cfg_parallel(tmp_path: Path):
|
||||
"""T2V with CFG=4.0, cfg-parallel-size=2."""
|
||||
output = tmp_path / "t2v_cfg4.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(T2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--prompt",
|
||||
T2V_PROMPT,
|
||||
"--negative-prompt",
|
||||
T2V_NEGATIVE_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"145",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"4.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--cfg-parallel-size",
|
||||
"2",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="08e606b9c522fee4b6f30cee8b77db40")
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_ltx2_t2v_no_cfg(tmp_path: Path):
|
||||
"""T2V with CFG=1.0 (no classifier-free guidance)."""
|
||||
output = tmp_path / "t2v_nocfg.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(T2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--prompt",
|
||||
T2V_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"145",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"1.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="a83994b94b6e67c54a524e0383c45ce8")
|
||||
|
||||
|
||||
# ── I2V tests ──
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_ltx2_i2v_cfg_parallel(tmp_path: Path):
|
||||
"""I2V with CFG=4.0, cfg-parallel-size=2."""
|
||||
test_image = tmp_path / "test_input.png"
|
||||
_make_deterministic_test_image(test_image)
|
||||
output = tmp_path / "i2v_cfg4.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(I2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--model-class-name",
|
||||
"LTX2ImageToVideoPipeline",
|
||||
"--image",
|
||||
str(test_image),
|
||||
"--prompt",
|
||||
I2V_PROMPT,
|
||||
"--negative-prompt",
|
||||
I2V_NEGATIVE_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"73",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"4.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--cfg-parallel-size",
|
||||
"2",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="aed7e56084b36373244d8f839b16d115")
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_ltx2_i2v_no_cfg(tmp_path: Path):
|
||||
"""I2V with CFG=1.0 (no classifier-free guidance)."""
|
||||
test_image = tmp_path / "test_input.png"
|
||||
_make_deterministic_test_image(test_image)
|
||||
output = tmp_path / "i2v_nocfg.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(I2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--model-class-name",
|
||||
"LTX2ImageToVideoPipeline",
|
||||
"--image",
|
||||
str(test_image),
|
||||
"--prompt",
|
||||
I2V_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"73",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"1.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="81b21ede12753e9e14a357a6c548b666")
|
||||
+61
-51
@@ -13,6 +13,28 @@ from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "SII-GAIR/daVinci-MagiHuman-Base-1080p"
|
||||
|
||||
# (model, stage_config_path, extra_omni_kwargs) — see ``omni_runner`` in tests.helpers.fixtures.runtime
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL,
|
||||
None,
|
||||
{
|
||||
"init_timeout": 1200,
|
||||
"tensor_parallel_size": 2,
|
||||
},
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
_SKIP_ISSUE_3236 = pytest.mark.skip(
|
||||
reason="https://github.com/vllm-project/vllm-omni/issues/3236",
|
||||
)
|
||||
|
||||
|
||||
def _validate_mp4(video_bytes: bytes, min_frames: int = 10) -> None:
|
||||
"""Validate that the MP4 contains meaningful video and audio tracks."""
|
||||
@@ -38,17 +60,10 @@ def _validate_mp4(video_bytes: bytes, min_frames: int = 10) -> None:
|
||||
container.close()
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@_SKIP_ISSUE_3236
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
def test_magi_human_e2e(run_level):
|
||||
def test_magi_human_e2e(omni_runner: OmniRunner):
|
||||
"""End-to-end test for MagiHuman generating video and audio."""
|
||||
if run_level != "advanced_model":
|
||||
pytest.skip("MagiHuman e2e test requires advanced_model run level with real weights.")
|
||||
|
||||
model_path = "SII-GAIR/daVinci-MagiHuman-Base-1080p"
|
||||
|
||||
prompt = (
|
||||
"A young woman with long, wavy golden blonde hair and bright blue eyes, "
|
||||
"wearing a fitted ivory silk blouse with a delicate lace collar, sits "
|
||||
@@ -88,54 +103,49 @@ def test_magi_human_e2e(run_level):
|
||||
},
|
||||
)
|
||||
|
||||
with OmniRunner(
|
||||
model_path,
|
||||
init_timeout=1200,
|
||||
tensor_parallel_size=2,
|
||||
) as runner:
|
||||
omni = runner.omni
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
prompts=[prompt],
|
||||
sampling_params_list=[sampling_params],
|
||||
)
|
||||
omni = omni_runner.omni
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
prompts=[prompt],
|
||||
sampling_params_list=[sampling_params],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(outputs) > 0, "No outputs returned"
|
||||
first = outputs[0]
|
||||
assert len(outputs) > 0, "No outputs returned"
|
||||
first = outputs[0]
|
||||
|
||||
assert hasattr(first, "images") and first.images, "No video frames in output"
|
||||
video_frames = first.images[0]
|
||||
assert isinstance(video_frames, np.ndarray), f"Expected numpy array, got {type(video_frames)}"
|
||||
assert video_frames.ndim == 4, f"Expected 4D array (T,H,W,3), got shape {video_frames.shape}"
|
||||
assert hasattr(first, "images") and first.images, "No video frames in output"
|
||||
video_frames = first.images[0]
|
||||
assert isinstance(video_frames, np.ndarray), f"Expected numpy array, got {type(video_frames)}"
|
||||
assert video_frames.ndim == 4, f"Expected 4D array (T,H,W,3), got shape {video_frames.shape}"
|
||||
|
||||
mm = first.multimodal_output
|
||||
assert mm, "multimodal_output is empty or missing"
|
||||
mm = first.multimodal_output
|
||||
assert mm, "multimodal_output is empty or missing"
|
||||
|
||||
audio_waveform = mm.get("audio")
|
||||
assert audio_waveform is not None, "No audio waveform in multimodal_output"
|
||||
audio_waveform = mm.get("audio")
|
||||
assert audio_waveform is not None, "No audio waveform in multimodal_output"
|
||||
|
||||
audio_sample_rate = mm.get("audio_sample_rate")
|
||||
assert audio_sample_rate is not None, (
|
||||
"audio_sample_rate not found in multimodal_output; model post-process must propagate it"
|
||||
)
|
||||
assert isinstance(audio_sample_rate, (int, float)), (
|
||||
f"audio_sample_rate should be numeric, got {type(audio_sample_rate)}"
|
||||
)
|
||||
assert int(audio_sample_rate) > 0, f"audio_sample_rate must be positive, got {audio_sample_rate}"
|
||||
audio_sample_rate = mm.get("audio_sample_rate")
|
||||
assert audio_sample_rate is not None, (
|
||||
"audio_sample_rate not found in multimodal_output; model post-process must propagate it"
|
||||
)
|
||||
assert isinstance(audio_sample_rate, (int, float)), (
|
||||
f"audio_sample_rate should be numeric, got {type(audio_sample_rate)}"
|
||||
)
|
||||
assert int(audio_sample_rate) > 0, f"audio_sample_rate must be positive, got {audio_sample_rate}"
|
||||
|
||||
fps = mm.get("fps")
|
||||
assert fps is not None, "fps not found in multimodal_output; model post-process must propagate it"
|
||||
assert isinstance(fps, (int, float)), f"fps should be numeric, got {type(fps)}"
|
||||
assert int(fps) > 0, f"fps must be positive, got {fps}"
|
||||
fps = mm.get("fps")
|
||||
assert fps is not None, "fps not found in multimodal_output; model post-process must propagate it"
|
||||
assert isinstance(fps, (int, float)), f"fps should be numeric, got {type(fps)}"
|
||||
assert int(fps) > 0, f"fps must be positive, got {fps}"
|
||||
|
||||
video_bytes = mux_video_audio_bytes(
|
||||
video_frames,
|
||||
audio_waveform,
|
||||
fps=float(fps),
|
||||
audio_sample_rate=int(audio_sample_rate),
|
||||
)
|
||||
assert isinstance(video_bytes, bytes), f"Expected MP4 bytes, got {type(video_bytes)}"
|
||||
assert len(video_bytes) > 1000, f"MP4 too small ({len(video_bytes)} bytes)"
|
||||
video_bytes = mux_video_audio_bytes(
|
||||
video_frames,
|
||||
audio_waveform,
|
||||
fps=float(fps),
|
||||
audio_sample_rate=int(audio_sample_rate),
|
||||
)
|
||||
assert isinstance(video_bytes, bytes), f"Expected MP4 bytes, got {type(video_bytes)}"
|
||||
assert len(video_bytes) > 1000, f"MP4 too small ({len(video_bytes)} bytes)"
|
||||
|
||||
_validate_mp4(video_bytes)
|
||||
_validate_mp4(video_bytes)
|
||||
@@ -1,212 +0,0 @@
|
||||
"""
|
||||
End-to-end test for MammothModa2 text-to-image generation.
|
||||
|
||||
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
|
||||
match a golden reference.
|
||||
|
||||
Model weights: env var ``MAMMOTHMODA2_MODEL_PATH``
|
||||
(default: <repo_root>/MammothModa2-Preview)
|
||||
Stage config: env var ``MAMMOTHMODA2_T2I_STAGE_CONFIG``
|
||||
(default: vllm_omni/model_executor/stage_configs/mammoth_moda2.yaml)
|
||||
|
||||
Golden pixel file: ``tests/e2e/offline_inference/fixtures/mammoth_moda2_t2i_golden.json``
|
||||
Regenerate with: ``UPDATE_GOLDEN=1 pytest tests/e2e/offline_inference/test_mammoth_moda2.py``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
_IMAGE_TOKEN_ID = 151655 # "<|image_pad|>"
|
||||
_VIDEO_TOKEN_ID = 151656 # "<|video_pad|>"
|
||||
_VISION_START_TOKEN_ID = 151652 # "<|vision_start|>"
|
||||
_VISION_END_TOKEN_ID = 151653 # "<|vision_end|>"
|
||||
_AR_PATCH_SIZE = 16
|
||||
|
||||
_STAGE_CONFIGS_DIR = Path(__file__).resolve().parents[3] / "vllm_omni" / "model_executor" / "stage_configs"
|
||||
MODEL_PATH = os.environ.get(
|
||||
"MAMMOTHMODA2_MODEL_PATH",
|
||||
str(Path(__file__).resolve().parents[3] / "MammothModa2-Preview"),
|
||||
)
|
||||
T2I_STAGE_CONFIG = os.environ.get(
|
||||
"MAMMOTHMODA2_T2I_STAGE_CONFIG",
|
||||
str(_STAGE_CONFIGS_DIR / "mammoth_moda2.yaml"),
|
||||
)
|
||||
|
||||
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
|
||||
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
|
||||
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
|
||||
# Covers corners, centre, and mid-edges across all 3 channels.
|
||||
_PIXEL_SAMPLE_COORDS = [
|
||||
(0, 0.0, 0.0),
|
||||
(0, 0.5, 0.5),
|
||||
(0, 1.0, 1.0),
|
||||
(0, 0.25, 0.75),
|
||||
(1, 0.0, 1.0),
|
||||
(1, 0.5, 0.0),
|
||||
(1, 0.75, 0.25),
|
||||
(1, 1.0, 0.5),
|
||||
(2, 0.0, 0.5),
|
||||
(2, 0.5, 1.0),
|
||||
(2, 0.75, 0.75),
|
||||
(2, 1.0, 0.0),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _load_t2i_gen_config(model_dir: str) -> dict:
|
||||
cfg_path = Path(model_dir) / "t2i_generation_config.json"
|
||||
if not cfg_path.exists():
|
||||
pytest.skip(f"t2i_generation_config.json not found at {cfg_path}")
|
||||
with cfg_path.open() as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _format_t2i_prompt(user_prompt: str, ar_width: int, ar_height: int) -> str:
|
||||
return (
|
||||
"<|im_start|>system\nYou are a helpful image generator.<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
f"<|image start|>{ar_width}*{ar_height}<|image token|>"
|
||||
)
|
||||
|
||||
|
||||
def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
|
||||
"""Sample pixel values at fixed fractional coordinates from a (C, H, W) tensor."""
|
||||
t = img_tensor.float().clamp(0.0, 1.0)
|
||||
if t.ndim == 4:
|
||||
t = t[0] # unbatch
|
||||
C, H, W = t.shape
|
||||
values = []
|
||||
for c, rh, rw in _PIXEL_SAMPLE_COORDS:
|
||||
ri = min(int(rh * (H - 1)), H - 1)
|
||||
ci = min(int(rw * (W - 1)), W - 1)
|
||||
values.append(round(float(t[c, ri, ci]), 6))
|
||||
return values
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end test
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_mammothmoda2_t2i_e2e():
|
||||
"""
|
||||
End-to-end text-to-image generation with MammothModa2 (AR -> DiT).
|
||||
|
||||
Verifies:
|
||||
- Omni pipeline initialises with the two-stage YAML config.
|
||||
- DiT stage outputs an image tensor with the correct shape.
|
||||
- A fixed set of pixel values matches a golden reference
|
||||
(regenerate with ``UPDATE_GOLDEN=1``).
|
||||
"""
|
||||
if not Path(MODEL_PATH).exists():
|
||||
pytest.skip(f"Model weights not found at {MODEL_PATH}")
|
||||
if not Path(T2I_STAGE_CONFIG).exists():
|
||||
pytest.skip(f"Stage config not found at {T2I_STAGE_CONFIG}")
|
||||
|
||||
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
|
||||
eol_token_id = int(gen_cfg["eol_token_id"])
|
||||
visual_start = int(gen_cfg["visual_token_start_id"])
|
||||
visual_end = int(gen_cfg["visual_token_end_id"])
|
||||
|
||||
height, width = 256, 256 # small for CI speed
|
||||
ar_height, ar_width = height // _AR_PATCH_SIZE, width // _AR_PATCH_SIZE
|
||||
expected_grid_tokens = ar_height * (ar_width + 1)
|
||||
|
||||
prompt_text = "A cat sitting on a laptop keyboard"
|
||||
formatted_prompt = _format_t2i_prompt(prompt_text, ar_width, ar_height)
|
||||
|
||||
with OmniRunner(MODEL_PATH, stage_configs_path=T2I_STAGE_CONFIG, trust_remote_code=True) as runner:
|
||||
omni = runner.omni
|
||||
# Greedy / deterministic sampling so pixel values are reproducible.
|
||||
ar_sampling = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_k=1,
|
||||
max_tokens=max(1, expected_grid_tokens + 1),
|
||||
detokenize=False,
|
||||
)
|
||||
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
|
||||
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": formatted_prompt,
|
||||
"additional_information": {
|
||||
"omni_task": ["t2i"],
|
||||
"ar_width": [ar_width],
|
||||
"ar_height": [ar_height],
|
||||
"eol_token_id": [eol_token_id],
|
||||
"visual_token_start_id": [visual_start],
|
||||
"visual_token_end_id": [visual_end],
|
||||
"image_height": [height],
|
||||
"image_width": [width],
|
||||
"num_inference_steps": [2],
|
||||
"text_guidance_scale": [1.0],
|
||||
"cfg_range": [0.0, 1.0],
|
||||
"visual_ids": [
|
||||
_IMAGE_TOKEN_ID,
|
||||
_VIDEO_TOKEN_ID,
|
||||
_VISION_START_TOKEN_ID,
|
||||
_VISION_END_TOKEN_ID,
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
[ar_sampling, dit_sampling],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(outputs) > 0, "Pipeline produced no outputs"
|
||||
|
||||
# Find the image tensor produced by the DiT stage and compare pixels.
|
||||
found_image = False
|
||||
for out in outputs:
|
||||
ro_list = getattr(out, "request_output", out)
|
||||
if not isinstance(ro_list, list):
|
||||
ro_list = [ro_list]
|
||||
for ro in ro_list:
|
||||
completion_outputs = getattr(ro, "outputs", None)
|
||||
if not isinstance(completion_outputs, list):
|
||||
continue
|
||||
for completion in completion_outputs:
|
||||
mm = getattr(completion, "multimodal_output", None)
|
||||
if not (isinstance(mm, dict) and "image" in mm):
|
||||
continue
|
||||
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
|
||||
for img_tensor in img_list:
|
||||
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
|
||||
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
|
||||
|
||||
sampled = _sample_pixels(img_tensor)
|
||||
|
||||
if os.environ.get("UPDATE_GOLDEN"):
|
||||
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
|
||||
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
|
||||
elif _GOLDEN_T2I_PATH.exists():
|
||||
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
|
||||
for i, (got, exp) in enumerate(zip(sampled, golden)):
|
||||
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
|
||||
|
||||
found_image = True
|
||||
|
||||
assert found_image, "No image tensor found in pipeline output"
|
||||
@@ -0,0 +1,204 @@
|
||||
"""
|
||||
End-to-end test for MammothModa2 text-to-image generation.
|
||||
|
||||
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
|
||||
match a golden reference.
|
||||
|
||||
Model Hub repo id: ``bytedance-research/MammothModa2-Preview``.
|
||||
Stage config: ``get_deploy_config_path("mammoth_moda2.yaml")`` → ``vllm_omni/deploy/mammoth_moda2.yaml``
|
||||
|
||||
Golden pixel file: ``tests/e2e/offline_inference/fixtures/mammoth_moda2_t2i_golden.json``
|
||||
Regenerate with: ``UPDATE_GOLDEN=1 pytest tests/e2e/offline_inference/test_mammoth_moda2_expansion.py``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
_IMAGE_TOKEN_ID = 151655 # "<|image_pad|>"
|
||||
_VIDEO_TOKEN_ID = 151656 # "<|video_pad|>"
|
||||
_VISION_START_TOKEN_ID = 151652 # "<|vision_start|>"
|
||||
_VISION_END_TOKEN_ID = 151653 # "<|vision_end|>"
|
||||
_AR_PATCH_SIZE = 16
|
||||
|
||||
MODEL_PATH = "bytedance-research/MammothModa2-Preview"
|
||||
T2I_STAGE_CONFIG = get_deploy_config_path("mammoth_moda2.yaml")
|
||||
|
||||
_OMNI_RUNNER_PARAM = (MODEL_PATH, T2I_STAGE_CONFIG)
|
||||
|
||||
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
|
||||
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
|
||||
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
|
||||
# Covers corners, centre, and mid-edges across all 3 channels.
|
||||
_PIXEL_SAMPLE_COORDS = [
|
||||
(0, 0.0, 0.0),
|
||||
(0, 0.5, 0.5),
|
||||
(0, 1.0, 1.0),
|
||||
(0, 0.25, 0.75),
|
||||
(1, 0.0, 1.0),
|
||||
(1, 0.5, 0.0),
|
||||
(1, 0.75, 0.25),
|
||||
(1, 1.0, 0.5),
|
||||
(2, 0.0, 0.5),
|
||||
(2, 0.5, 1.0),
|
||||
(2, 0.75, 0.75),
|
||||
(2, 1.0, 0.0),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def _load_t2i_gen_config(repo_id: str) -> dict:
|
||||
weights_dir = Path(snapshot_download(repo_id))
|
||||
cfg_path = weights_dir / "t2i_generation_config.json"
|
||||
if not cfg_path.exists():
|
||||
pytest.skip(f"t2i_generation_config.json not found at {cfg_path}")
|
||||
with cfg_path.open() as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _format_t2i_prompt(user_prompt: str, ar_width: int, ar_height: int) -> str:
|
||||
return (
|
||||
"<|im_start|>system\nYou are a helpful image generator.<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
f"<|image start|>{ar_width}*{ar_height}<|image token|>"
|
||||
)
|
||||
|
||||
|
||||
def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
|
||||
"""Sample pixel values at fixed fractional coordinates from a (C, H, W) tensor."""
|
||||
t = img_tensor.float().clamp(0.0, 1.0)
|
||||
if t.ndim == 4:
|
||||
t = t[0] # unbatch
|
||||
C, H, W = t.shape
|
||||
values = []
|
||||
for c, rh, rw in _PIXEL_SAMPLE_COORDS:
|
||||
ri = min(int(rh * (H - 1)), H - 1)
|
||||
ci = min(int(rw * (W - 1)), W - 1)
|
||||
values.append(round(float(t[c, ri, ci]), 6))
|
||||
return values
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end test
|
||||
# ---------------------------------------------------------------------------
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3201")
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
|
||||
"""
|
||||
End-to-end text-to-image generation with MammothModa2 (AR -> DiT).
|
||||
|
||||
Verifies:
|
||||
- Omni pipeline initialises with the two-stage YAML config.
|
||||
- DiT stage outputs an image tensor with the correct shape.
|
||||
- A fixed set of pixel values matches a golden reference
|
||||
(regenerate with ``UPDATE_GOLDEN=1``).
|
||||
"""
|
||||
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
|
||||
eol_token_id = int(gen_cfg["eol_token_id"])
|
||||
visual_start = int(gen_cfg["visual_token_start_id"])
|
||||
visual_end = int(gen_cfg["visual_token_end_id"])
|
||||
|
||||
height, width = 256, 256 # small for CI speed
|
||||
ar_height, ar_width = height // _AR_PATCH_SIZE, width // _AR_PATCH_SIZE
|
||||
expected_grid_tokens = ar_height * (ar_width + 1)
|
||||
|
||||
prompt_text = "A cat sitting on a laptop keyboard"
|
||||
formatted_prompt = _format_t2i_prompt(prompt_text, ar_width, ar_height)
|
||||
|
||||
omni = omni_runner.omni
|
||||
ar_sampling = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_k=1,
|
||||
max_tokens=max(1, expected_grid_tokens + 1),
|
||||
detokenize=False,
|
||||
)
|
||||
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
|
||||
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": formatted_prompt,
|
||||
"additional_information": {
|
||||
"omni_task": ["t2i"],
|
||||
"ar_width": [ar_width],
|
||||
"ar_height": [ar_height],
|
||||
"eol_token_id": [eol_token_id],
|
||||
"visual_token_start_id": [visual_start],
|
||||
"visual_token_end_id": [visual_end],
|
||||
"image_height": [height],
|
||||
"image_width": [width],
|
||||
"num_inference_steps": [2],
|
||||
"text_guidance_scale": [1.0],
|
||||
"cfg_range": [0.0, 1.0],
|
||||
"visual_ids": [
|
||||
_IMAGE_TOKEN_ID,
|
||||
_VIDEO_TOKEN_ID,
|
||||
_VISION_START_TOKEN_ID,
|
||||
_VISION_END_TOKEN_ID,
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
[ar_sampling, dit_sampling],
|
||||
)
|
||||
)
|
||||
|
||||
assert len(outputs) > 0, "Pipeline produced no outputs"
|
||||
|
||||
found_image = False
|
||||
for out in outputs:
|
||||
ro_list = getattr(out, "request_output", out)
|
||||
if not isinstance(ro_list, list):
|
||||
ro_list = [ro_list]
|
||||
for ro in ro_list:
|
||||
completion_outputs = getattr(ro, "outputs", None)
|
||||
if not isinstance(completion_outputs, list):
|
||||
continue
|
||||
for completion in completion_outputs:
|
||||
mm = getattr(completion, "multimodal_output", None)
|
||||
if not (isinstance(mm, dict) and "image" in mm):
|
||||
continue
|
||||
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
|
||||
for img_tensor in img_list:
|
||||
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
|
||||
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
|
||||
|
||||
sampled = _sample_pixels(img_tensor)
|
||||
|
||||
if os.environ.get("UPDATE_GOLDEN"):
|
||||
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
|
||||
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
|
||||
elif _GOLDEN_T2I_PATH.exists():
|
||||
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
|
||||
for i, (got, exp) in enumerate(zip(sampled, golden)):
|
||||
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
|
||||
|
||||
found_image = True
|
||||
|
||||
assert found_image, "No image tensor found in pipeline output"
|
||||
+7
-7
@@ -63,7 +63,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt("请详细介绍鹦鹉的生活习性。")
|
||||
request_config = {"prompts": prompt, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@@ -80,7 +80,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
|
||||
request_config = {"prompts": prompt, "images": image, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@@ -99,7 +99,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt(f"{AUDIO_TOKEN}Please recognize the language of this speech and transcribe it. Format: oral.")
|
||||
request_config = {"prompts": prompt, "audios": audio, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@@ -116,7 +116,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt(f"{VIDEO_TOKEN}Describe what is happening in this video.")
|
||||
request_config = {"prompts": prompt, "videos": video, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@@ -136,7 +136,7 @@ def test_mixed_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt(f"{IMAGE_TOKEN}{AUDIO_TOKEN}Describe the image and transcribe the audio.")
|
||||
request_config = {"prompts": prompt, "images": image, "audios": audio, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -152,7 +152,7 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt("请简单介绍一下北京。")
|
||||
request_config = {"prompts": prompt, "modalities": ["audio"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -169,4 +169,4 @@ def test_image_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
|
||||
request_config = {"prompts": prompt, "images": image, "modalities": ["audio"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
+40
-34
@@ -18,9 +18,25 @@ import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni import Omni
|
||||
|
||||
MODEL_NAME = "OpenMOSS-Team/MOSS-TTS-Nano"
|
||||
STAGE_CONFIG = get_deploy_config_path("moss_tts_nano.yaml")
|
||||
|
||||
# (model, stage_configs_path, extra_omni_kwargs) for ``omni_runner`` indirect parametrize
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL_NAME,
|
||||
STAGE_CONFIG,
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS-Nano/main/assets/audio/zh_1.wav"
|
||||
|
||||
@@ -90,74 +106,64 @@ def _build_request(
|
||||
|
||||
def _collect_audio(omni: Omni, request: dict) -> tuple[torch.Tensor, int]:
|
||||
"""Run a single request and return (waveform, sample_rate)."""
|
||||
for stage_outputs in omni.generate(request, DEFAULT_SAMPLING):
|
||||
for req_output in stage_outputs.request_output:
|
||||
mm = req_output.outputs[0].multimodal_output
|
||||
assert mm is not None, "Expected multimodal_output to be non-None"
|
||||
audio = mm.get("audio")
|
||||
sr = mm.get("sr")
|
||||
assert audio is not None, "Expected 'audio' key in multimodal_output"
|
||||
assert isinstance(audio, torch.Tensor), f"audio should be Tensor, got {type(audio)}"
|
||||
return audio.cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
|
||||
# Omni.generate returns list[OmniRequestOutput] by default (py_generator=False).
|
||||
for omni_out in omni.generate(request, DEFAULT_SAMPLING):
|
||||
mm = omni_out.multimodal_output
|
||||
assert mm is not None, "Expected multimodal_output to be non-None"
|
||||
audio = mm.get("audio")
|
||||
sr = mm.get("sr")
|
||||
assert audio is not None, "Expected 'audio' key in multimodal_output"
|
||||
assert isinstance(audio, torch.Tensor), f"audio should be Tensor, got {type(audio)}"
|
||||
return audio.cpu(), int(sr.item()) if sr is not None else SAMPLE_RATE
|
||||
raise AssertionError("No stage outputs received")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_engine():
|
||||
"""Module-scoped Omni engine to avoid re-loading the model for each test."""
|
||||
return Omni(model=MODEL_NAME, stage_init_timeout=180)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_english(omni_engine, ref_audio_path):
|
||||
"""English voice_clone mode (no prompt_text) produces non-empty 48 kHz audio."""
|
||||
def test_moss_tts_nano_english(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""English TTS produces non-empty 48 kHz stereo audio."""
|
||||
req = _build_request("Hello, this is a short voice cloning demo for testing.", ref_audio_path)
|
||||
audio, sr = _collect_audio(omni_engine, req)
|
||||
audio, sr = _collect_audio(omni_runner.omni, req)
|
||||
|
||||
assert sr == SAMPLE_RATE, f"Expected sample_rate={SAMPLE_RATE}, got {sr}"
|
||||
assert audio.numel() > 0, "Audio tensor should not be empty"
|
||||
assert not torch.all(audio == 0), "Audio should not be all-zeros (silence)"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_chinese(omni_engine, ref_audio_path):
|
||||
"""Chinese voice_clone mode produces non-empty audio."""
|
||||
def test_moss_tts_nano_chinese(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""Chinese TTS produces non-empty audio."""
|
||||
req = _build_request("你好,这是语音合成测试。", ref_audio_path)
|
||||
audio, sr = _collect_audio(omni_engine, req)
|
||||
audio, sr = _collect_audio(omni_runner.omni, req)
|
||||
|
||||
assert sr == SAMPLE_RATE
|
||||
assert audio.numel() > 0
|
||||
assert not torch.all(audio == 0)
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_deterministic(omni_engine, ref_audio_path):
|
||||
def test_moss_tts_nano_deterministic(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""Same seed produces identical waveforms."""
|
||||
req = _build_request("Reproducible output test.", ref_audio_path, seed=123)
|
||||
audio1, _ = _collect_audio(omni_engine, req)
|
||||
audio2, _ = _collect_audio(omni_engine, req)
|
||||
audio1, _ = _collect_audio(omni_runner.omni, req)
|
||||
audio2, _ = _collect_audio(omni_runner.omni, req)
|
||||
|
||||
assert audio1.shape == audio2.shape, "Waveform shapes should match with same seed"
|
||||
assert torch.allclose(audio1, audio2, atol=1e-4), "Waveforms should match with same seed"
|
||||
|
||||
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_moss_tts_nano_batch(omni_engine, ref_audio_path):
|
||||
def test_moss_tts_nano_batch(omni_runner: OmniRunner, ref_audio_path) -> None:
|
||||
"""Batch of two requests returns audio for each."""
|
||||
requests = [
|
||||
_build_request("First request.", ref_audio_path),
|
||||
_build_request("Second request.", ref_audio_path),
|
||||
]
|
||||
results = []
|
||||
for stage_outputs in omni_engine.generate(requests, [DEFAULT_SAMPLING] * 2):
|
||||
for req_output in stage_outputs.request_output:
|
||||
mm = req_output.outputs[0].multimodal_output
|
||||
assert mm is not None
|
||||
results.append(mm["audio"].cpu())
|
||||
# Single-stage pipeline: one SamplingParams object is broadcast to all prompts.
|
||||
for omni_out in omni_runner.omni.generate(requests, DEFAULT_SAMPLING):
|
||||
mm = omni_out.multimodal_output
|
||||
assert mm is not None
|
||||
results.append(mm["audio"].cpu())
|
||||
|
||||
assert len(results) == 2, f"Expected 2 outputs, got {len(results)}"
|
||||
for i, audio in enumerate(results):
|
||||
@@ -1,159 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end tests for Omni Sleep Mode across various model architectures.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
MODEL_DIFF = "riverclouds/qwen_image_random"
|
||||
|
||||
|
||||
def get_ack_info(ack, key, default=None):
|
||||
if hasattr(ack, key):
|
||||
return getattr(ack, key)
|
||||
if isinstance(ack, dict):
|
||||
return ack.get(key, default)
|
||||
return default
|
||||
|
||||
|
||||
def get_dynamic_devices(stage_idx, num_stages, tp_size):
|
||||
total_gpus = torch.accelerator.device_count()
|
||||
gpus_per_stage = tp_size
|
||||
start_idx = stage_idx * gpus_per_stage
|
||||
if start_idx + gpus_per_stage > total_gpus:
|
||||
start_idx = start_idx % total_gpus
|
||||
device_ids = [str(start_idx + i) for i in range(gpus_per_stage)]
|
||||
return ",".join(device_ids)
|
||||
|
||||
|
||||
# Test 1: Diffusion Model (2-Stage BAGEL)
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)
|
||||
@pytest.mark.asyncio
|
||||
async def test_diffusion_model_sleep_tp(tp_size):
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
if num_gpus < tp_size:
|
||||
pytest.skip(f"Skipping TP={tp_size}")
|
||||
|
||||
engine_args = {
|
||||
"model": MODEL,
|
||||
"enable_sleep_mode": True,
|
||||
"tensor_parallel_size": tp_size,
|
||||
"enforce_eager": True,
|
||||
"trust_remote_code": True,
|
||||
"dtype": "bfloat16",
|
||||
"gpu_memory_utilization": 0.5,
|
||||
}
|
||||
|
||||
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
|
||||
try:
|
||||
# BAGEL requires 2 params
|
||||
diff_sp = OmniDiffusionSamplingParams(num_inference_steps=2, height=256, width=256)
|
||||
llm_sp = SamplingParams()
|
||||
|
||||
# Warmup
|
||||
async for _ in engine.generate("test", sampling_params=[llm_sp, diff_sp]):
|
||||
pass
|
||||
|
||||
# Sleep all
|
||||
acks = await engine.sleep(level=2)
|
||||
statuses = [get_ack_info(ack, "status") for ack in acks]
|
||||
assert all(s == "SUCCESS" for s in statuses), f"Sleep failed. Statuses: {statuses}"
|
||||
|
||||
# Wakeup & Verify
|
||||
await engine.wake_up()
|
||||
async for _ in engine.generate("verify", sampling_params=[llm_sp, diff_sp]):
|
||||
pass
|
||||
|
||||
print(f"Diffusion TP={tp_size} Lifecycle OK")
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
# Test 2: Multi-stage Manual Config
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_multistage_sleep_h100(tp_size):
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
if num_gpus < tp_size * 2:
|
||||
pytest.skip("Not enough GPUs")
|
||||
|
||||
stages = []
|
||||
for i in range(2):
|
||||
devs = get_dynamic_devices(i, 2, tp_size)
|
||||
stages.append(
|
||||
{
|
||||
"stage_id": i,
|
||||
"stage_type": "llm" if i == 0 else "diffusion",
|
||||
"runtime": {"process": True, "devices": devs},
|
||||
"engine_args": {
|
||||
"model": MODEL,
|
||||
"model_stage": "thinker" if i == 0 else "base",
|
||||
"tensor_parallel_size": tp_size,
|
||||
"gpu_memory_utilization": 0.4,
|
||||
"dtype": "bfloat16",
|
||||
"enable_sleep_mode": True,
|
||||
"trust_remote_code": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
|
||||
|
||||
engine = AsyncOmni(
|
||||
model=MODEL, stages=stages, connectors=connectors, enable_sleep_mode=True, stage_init_timeout=1200
|
||||
)
|
||||
try:
|
||||
sp = OmniDiffusionSamplingParams(num_inference_steps=2)
|
||||
async for _ in engine.generate("warmup", sampling_params=[SamplingParams(), sp]):
|
||||
pass
|
||||
|
||||
acks = await engine.sleep(stage_ids=[0, 1], level=2)
|
||||
assert len(acks) == 2
|
||||
|
||||
await engine.wake_up(stage_ids=[0, 1])
|
||||
async for _ in engine.generate("verify", sampling_params=[SamplingParams(), sp]):
|
||||
pass
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
# Test 3: Pure Diffusion Single-Stage
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_pure_diffusion_scenario(tp_size):
|
||||
engine_args = {
|
||||
"model": MODEL_DIFF,
|
||||
"enable_sleep_mode": True,
|
||||
"tensor_parallel_size": tp_size,
|
||||
"enforce_eager": True,
|
||||
"dtype": "bfloat16",
|
||||
"gpu_memory_utilization": 0.5,
|
||||
}
|
||||
|
||||
engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
|
||||
try:
|
||||
await engine.sleep(level=1)
|
||||
await engine.wake_up()
|
||||
async for _ in engine.generate("test", sampling_params=[SamplingParams()]):
|
||||
pass
|
||||
print("Pure Diffusion OK")
|
||||
finally:
|
||||
engine.shutdown()
|
||||
+22
-21
@@ -11,27 +11,35 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "k2-fsa/OmniVoice"
|
||||
STAGE_CONFIG = get_deploy_config_path("omnivoice.yaml")
|
||||
|
||||
# (model, stage_config_path, extra_omni_kwargs) — see ``omni_runner`` in tests.helpers.fixtures.runtime
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL,
|
||||
STAGE_CONFIG,
|
||||
{
|
||||
"trust_remote_code": True,
|
||||
"log_stats": True,
|
||||
},
|
||||
)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def get_stage_config():
|
||||
return str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "omnivoice.yaml"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_omnivoice_text_to_audio() -> None:
|
||||
def test_omnivoice_text_to_audio(omni_runner: OmniRunner) -> None:
|
||||
"""
|
||||
Test OmniVoice text-to-audio generation via offline Omni runner.
|
||||
Deploy Setting: omnivoice.yaml (enforce_eager=true)
|
||||
@@ -40,17 +48,10 @@ def test_omnivoice_text_to_audio() -> None:
|
||||
"""
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
with OmniRunner(
|
||||
MODEL,
|
||||
stage_configs_path=get_stage_config(),
|
||||
trust_remote_code=True,
|
||||
log_stats=True,
|
||||
) as runner:
|
||||
prompts = {"prompt": "Hello, this is a test for text to audio."}
|
||||
prompts = {"prompt": "Hello, this is a test for text to audio."}
|
||||
sampling_params_list = [OmniDiffusionSamplingParams()]
|
||||
|
||||
sampling_params_list = [OmniDiffusionSamplingParams()]
|
||||
|
||||
outputs = list(runner.omni.generate(prompts, sampling_params_list=sampling_params_list))
|
||||
outputs = list(omni_runner.omni.generate(prompts, sampling_params_list=sampling_params_list))
|
||||
|
||||
assert len(outputs) > 0, "No outputs generated"
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
"""
|
||||
Tests for Ovis Image model pipeline.
|
||||
|
||||
Strategy:
|
||||
1. `mock_dependencies` fixture mocks heavy external components (VAE, Scheduler, TextEncoder)
|
||||
to allow fast testing of the pipeline logic without downloading weights.
|
||||
- Mocks are configured to return tensors on the correct device.
|
||||
- Transformer is mocked dynamically to return random noise of correct shape.
|
||||
|
||||
2. `test_real_transformer_init_and_forward` tests the actual `OvisImageTransformer2DModel`
|
||||
initialization and forward pass with a small configuration to ensure code coverage
|
||||
and correctness of the model definition itself, independent of the pipeline mocks.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
|
||||
# Mock the OvisImageTransformer2DModel to avoid complex init if needed,
|
||||
# or let it run if it's lightweight. It's likely not lightweight.
|
||||
# Better to mock the transformer forwarding to return random noise.
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image import OvisImagePipeline
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_dependencies(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Mock external dependencies to avoid loading real models.
|
||||
"""
|
||||
device = get_local_device()
|
||||
|
||||
# Mock Tokenizer
|
||||
mock_tokenizer = mocker.MagicMock()
|
||||
mock_tokenizer.return_value = mocker.MagicMock(
|
||||
input_ids=torch.zeros((1, 50), dtype=torch.long, device=device),
|
||||
attention_mask=torch.ones((1, 50), dtype=torch.long, device=device),
|
||||
)
|
||||
mock_tokenizer.apply_chat_template.return_value = "dummy prompt"
|
||||
mock_tokenizer.model_max_length = 1024
|
||||
|
||||
# Mock Text Encoder
|
||||
mock_text_encoder = mocker.MagicMock()
|
||||
mock_text_encoder.dtype = torch.float32
|
||||
# Output of text encoder must be on the same device as inputs (which are moved to execution_device)
|
||||
mock_text_encoder.return_value.last_hidden_state = torch.randn(1, 50, 32, device=device)
|
||||
|
||||
# Mock VAE
|
||||
mock_vae = mocker.MagicMock()
|
||||
mock_vae.config.block_out_channels = [128, 256, 512, 512] # Scale factor 8
|
||||
mock_vae.config.scale_factor_temporal = 1
|
||||
mock_vae.config.scale_factor_spatial = 8
|
||||
mock_vae.config.scaling_factor = 0.18215
|
||||
mock_vae.config.shift_factor = 0.0
|
||||
# Decode return value
|
||||
mock_vae.decode.return_value = [torch.randn(1, 3, 128, 128, device=device)]
|
||||
# Ensure .to() returns self so configuration persists
|
||||
mock_vae.to.return_value = mock_vae
|
||||
|
||||
# Mock Scheduler
|
||||
mock_scheduler = mocker.MagicMock()
|
||||
mock_scheduler.config = mocker.MagicMock()
|
||||
# Timesteps on device to match latents during denoising loop interaction if needed
|
||||
mock_scheduler.timesteps = torch.tensor([1.0, 0.5, 0.0], device=device)
|
||||
mock_scheduler.set_timesteps.return_value = None
|
||||
|
||||
# Make step return dynamic based on input sample shape
|
||||
def mock_scheduler_step(model_output, timestep, sample, **kwargs):
|
||||
# sample is the latents, should be preserved
|
||||
return (torch.randn_like(sample),)
|
||||
|
||||
mock_scheduler.step.side_effect = mock_scheduler_step
|
||||
|
||||
module_path = "vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image"
|
||||
|
||||
monkeypatch.setattr(f"{module_path}.Qwen2TokenizerFast.from_pretrained", lambda *a, **k: mock_tokenizer)
|
||||
monkeypatch.setattr(f"{module_path}.Qwen3Model.from_pretrained", lambda *a, **k: mock_text_encoder)
|
||||
monkeypatch.setattr(f"{module_path}.AutoencoderKL.from_pretrained", lambda *a, **k: mock_vae)
|
||||
monkeypatch.setattr(
|
||||
f"{module_path}.FlowMatchEulerDiscreteScheduler.from_pretrained", lambda *a, **k: mock_scheduler
|
||||
)
|
||||
|
||||
return {
|
||||
"tokenizer": mock_tokenizer,
|
||||
"text_encoder": mock_text_encoder,
|
||||
"vae": mock_vae,
|
||||
"scheduler": mock_scheduler,
|
||||
"device": device,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ovis_pipeline(mocker: MockerFixture, mock_dependencies, monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
Creates an OvisImagePipeline instance with mocked components.
|
||||
"""
|
||||
# Create config
|
||||
tf_config = TransformerConfig(
|
||||
params={
|
||||
"in_channels": 4,
|
||||
"out_channels": 4,
|
||||
"sample_size": 32,
|
||||
"patch_size": 2,
|
||||
"num_attention_heads": 4,
|
||||
"attention_head_dim": 8,
|
||||
"num_layers": 1,
|
||||
"caption_channels": 32,
|
||||
}
|
||||
)
|
||||
|
||||
od_config = OmniDiffusionConfig(
|
||||
model="dummy-ovis",
|
||||
tf_model_config=tf_config,
|
||||
dtype=torch.float32,
|
||||
num_gpus=1,
|
||||
)
|
||||
|
||||
# Mock Transformer Layer separately to avoid full init
|
||||
# We patch OvisImageTransformer2DModel class in the module
|
||||
mock_transformer_cls = mocker.MagicMock()
|
||||
mock_transformer_instance = mocker.MagicMock()
|
||||
mock_transformer_instance.dtype = torch.float32
|
||||
mock_transformer_instance.in_channels = 16 # Must be 16 so num_channel_latents=4, packed=16
|
||||
# Forward return: noise prediction
|
||||
|
||||
def mock_forward(hidden_states, *args, **kwargs):
|
||||
# hidden_states shape: (B, SeqLen, Channels)
|
||||
return (torch.randn_like(hidden_states),)
|
||||
|
||||
mock_transformer_instance.forward.side_effect = mock_forward
|
||||
# Also make the instance itself callable to mimic __call__
|
||||
mock_transformer_instance.side_effect = mock_forward
|
||||
|
||||
mock_transformer_cls.return_value = mock_transformer_instance
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image.OvisImageTransformer2DModel", mock_transformer_cls
|
||||
)
|
||||
|
||||
# Initialize pipeline
|
||||
# We use a dummy model path check override
|
||||
mocker.patch("os.path.exists", return_value=True)
|
||||
pipeline = OvisImagePipeline(od_config=od_config)
|
||||
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_interface_compliance(ovis_pipeline):
|
||||
"""Verify methods required by vllm-omni framework."""
|
||||
assert hasattr(ovis_pipeline, "load_weights")
|
||||
assert hasattr(ovis_pipeline, "scheduler")
|
||||
assert hasattr(ovis_pipeline, "transformer")
|
||||
assert hasattr(ovis_pipeline, "text_encoder")
|
||||
# assert hasattr(ovis_pipeline, "vae") # Ovis uses VAE
|
||||
|
||||
|
||||
def test_basic_generation(ovis_pipeline):
|
||||
"""Test the forward pass logic."""
|
||||
# Setup request
|
||||
req = OmniDiffusionRequest(
|
||||
prompts=["A photo of a cat"],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
),
|
||||
)
|
||||
|
||||
output = ovis_pipeline(req)
|
||||
|
||||
assert output is not None
|
||||
assert output.output is not None
|
||||
# Output should be a tensor from mocked VAE decode [torch.randn(1, 3, 128, 128)]
|
||||
assert isinstance(output.output, torch.Tensor)
|
||||
assert output.output.shape == (1, 3, 128, 128)
|
||||
|
||||
# Check that transformer was called
|
||||
assert ovis_pipeline.transformer.call_count > 0
|
||||
|
||||
|
||||
def test_guidance_scale(ovis_pipeline):
|
||||
"""Test that classifier-free guidance path is taken when scale > 1.0."""
|
||||
req = OmniDiffusionRequest(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A photo of a cat",
|
||||
"negative_prompt": "bad quality",
|
||||
}
|
||||
],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=1,
|
||||
guidance_scale=2.0, # Trigger CFG
|
||||
),
|
||||
)
|
||||
|
||||
ovis_pipeline(req)
|
||||
assert ovis_pipeline.transformer.call_count >= 2
|
||||
|
||||
|
||||
def test_resolution_check(ovis_pipeline):
|
||||
"""Test resolution divisible validation logic if present."""
|
||||
# Pass odd resolution
|
||||
req = OmniDiffusionRequest(
|
||||
prompts=["test"],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=250, # Not divisible by 16 (8*2)
|
||||
width=250,
|
||||
),
|
||||
)
|
||||
|
||||
# Should warn but proceed (as per code I read earlier) or resize?
|
||||
# The code had `logger.warning(...)`
|
||||
|
||||
output = ovis_pipeline(req)
|
||||
assert output is not None
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
|
||||
def test_real_transformer_init_and_forward(mocker: MockerFixture):
|
||||
"""Test the real OvisImageTransformer2DModel initialization and forward pass for coverage."""
|
||||
|
||||
from vllm_omni.diffusion.models.ovis_image.ovis_image_transformer import OvisImageTransformer2DModel
|
||||
|
||||
device = get_local_device()
|
||||
tf_config = TransformerConfig(
|
||||
params={
|
||||
"patch_size": 2,
|
||||
"in_channels": 16,
|
||||
"out_channels": 16,
|
||||
"num_layers": 1,
|
||||
"num_single_layers": 1,
|
||||
"attention_head_dim": 8,
|
||||
"num_attention_heads": 2,
|
||||
"joint_attention_dim": 32,
|
||||
"axes_dims_rope": (4, 4, 4),
|
||||
}
|
||||
)
|
||||
|
||||
od_config = OmniDiffusionConfig(model="dummy-ovis", tf_model_config=tf_config, dtype=torch.bfloat16, num_gpus=1)
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
|
||||
# Mock distributed state for QKVParallelLinear initialization
|
||||
# We patch get_tp_group because get_tensor_model_parallel_rank calls it and asserts _TP is not None
|
||||
mock_group = mocker.MagicMock()
|
||||
mock_group.rank_in_group = 0
|
||||
mock_group.world_size = 1
|
||||
|
||||
mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_group)
|
||||
# Initialize real model
|
||||
model = OvisImageTransformer2DModel(
|
||||
od_config=od_config,
|
||||
patch_size=1,
|
||||
in_channels=16,
|
||||
out_channels=16,
|
||||
num_single_layers=1,
|
||||
attention_head_dim=8,
|
||||
num_attention_heads=2,
|
||||
joint_attention_dim=32,
|
||||
axes_dims_rope=(2, 2, 4),
|
||||
).to(device)
|
||||
|
||||
# Create dummy inputs
|
||||
B, Seq, C = 1, 16, 16
|
||||
hidden_states = torch.randn(B, Seq, C, device=device)
|
||||
encoder_hidden_states = torch.randn(B, 10, 32, device=device) # joint_attention_dim=32
|
||||
timestep = torch.tensor([1], device=device)
|
||||
img_ids = torch.zeros(Seq, 3, device=device)
|
||||
txt_ids = torch.zeros(10, 3, device=device)
|
||||
|
||||
# Run forward
|
||||
output = model(
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
timestep=timestep,
|
||||
img_ids=img_ids,
|
||||
txt_ids=txt_ids,
|
||||
return_dict=False,
|
||||
)
|
||||
|
||||
assert output is not None
|
||||
assert isinstance(output, tuple)
|
||||
assert output[0].shape == hidden_states.shape
|
||||
@@ -1,319 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
End-to-end tests for the unified quantization framework (PR #1764).
|
||||
|
||||
Validates FP8 quantization works correctly for all supported model types:
|
||||
- Single-stage diffusion models (FLUX.1-dev, Qwen-Image, Z-Image-Turbo)
|
||||
- Multi-stage models (BAGEL: LLM + Diffusion)
|
||||
|
||||
Tests verify:
|
||||
1. FP8 quantization produces valid images
|
||||
2. Memory usage is lower than BF16 baseline
|
||||
3. Multi-stage models only quantize the diffusion stage (not the LLM stage)
|
||||
|
||||
Usage:
|
||||
# Run all FP8 quantization tests
|
||||
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v
|
||||
|
||||
# Run single-stage tests only (faster, needs ~25GB VRAM)
|
||||
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v -k "single_stage"
|
||||
|
||||
# Run BAGEL multi-stage test (needs ~55GB VRAM, H100 recommended)
|
||||
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v -k "bagel"
|
||||
|
||||
# Run FLUX test only
|
||||
pytest tests/e2e/offline_inference/test_quantization_fp8.py -v -k "flux"
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
# ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _generate_single_stage_image(
|
||||
model: str,
|
||||
quantization: str | None = None,
|
||||
height: int = 256,
|
||||
width: int = 256,
|
||||
num_inference_steps: int = 2,
|
||||
seed: int = 42,
|
||||
**extra_omni_kwargs: Any,
|
||||
) -> tuple[list, float]:
|
||||
"""Generate an image with a single-stage diffusion model.
|
||||
|
||||
Returns (images, peak_memory_gib).
|
||||
"""
|
||||
omni_kwargs: dict[str, Any] = dict(extra_omni_kwargs)
|
||||
if quantization:
|
||||
omni_kwargs["quantization"] = quantization
|
||||
|
||||
with OmniRunner(model, **omni_kwargs) as runner:
|
||||
torch.accelerator.reset_peak_memory_stats()
|
||||
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
|
||||
outputs = runner.omni.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=generator,
|
||||
),
|
||||
)
|
||||
|
||||
peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
|
||||
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
assert hasattr(first_output, "request_output") and first_output.request_output
|
||||
req_out = first_output.request_output[0]
|
||||
assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
|
||||
images = req_out.images
|
||||
assert len(images) >= 1
|
||||
assert images[0].width == width
|
||||
assert images[0].height == height
|
||||
|
||||
return images, peak_mem
|
||||
|
||||
|
||||
def _generate_bagel_image(
|
||||
quantization_config: str | None = None,
|
||||
num_inference_steps: int = 15,
|
||||
) -> tuple[Any, float]:
|
||||
"""Generate an image with BAGEL (multi-stage: LLM + Diffusion).
|
||||
|
||||
Returns (generated_image, peak_memory_gib).
|
||||
"""
|
||||
config_path = get_deploy_config_path("ci/bagel.yaml")
|
||||
omni_kwargs: dict[str, Any] = {
|
||||
"model": "ByteDance-Seed/BAGEL-7B-MoT",
|
||||
"stage_configs_path": config_path,
|
||||
"stage_init_timeout": 300,
|
||||
}
|
||||
if quantization_config:
|
||||
omni_kwargs["quantization_config"] = quantization_config
|
||||
|
||||
model_name = omni_kwargs.pop("model")
|
||||
with OmniRunner(model_name, **omni_kwargs) as runner:
|
||||
omni = runner.omni
|
||||
torch.accelerator.reset_peak_memory_stats()
|
||||
|
||||
params_list = omni.default_sampling_params_list
|
||||
if len(params_list) > 1:
|
||||
params_list[1].num_inference_steps = num_inference_steps # type: ignore
|
||||
params_list[1].extra_args = { # type: ignore
|
||||
"cfg_text_scale": 4.0,
|
||||
"cfg_img_scale": 1.5,
|
||||
}
|
||||
|
||||
prompt = "<|im_start|>A futuristic city skyline at twilight, cyberpunk style<|im_end|>"
|
||||
omni_outputs = list(
|
||||
omni.generate(
|
||||
prompts=[{"prompt": prompt, "modalities": ["image"]}],
|
||||
sampling_params_list=params_list,
|
||||
)
|
||||
)
|
||||
|
||||
peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
|
||||
|
||||
# Extract image
|
||||
generated_image = None
|
||||
for req_output in omni_outputs:
|
||||
if images := getattr(req_output, "images", None):
|
||||
generated_image = images[0]
|
||||
break
|
||||
if hasattr(req_output, "request_output") and req_output.request_output:
|
||||
for stage_out in req_output.request_output:
|
||||
if hasattr(stage_out, "images") and stage_out.images:
|
||||
generated_image = stage_out.images[0]
|
||||
break
|
||||
if generated_image:
|
||||
break
|
||||
|
||||
assert generated_image is not None, "No images generated from BAGEL"
|
||||
assert generated_image.size == (1024, 1024), f"Expected 1024x1024, got {generated_image.size}"
|
||||
|
||||
# Check LLM stage output — should have finish_reason=stop (not length)
|
||||
for req_output in omni_outputs:
|
||||
if hasattr(req_output, "request_output"):
|
||||
for stage_out in req_output.request_output:
|
||||
if hasattr(stage_out, "outputs"):
|
||||
for comp_out in stage_out.outputs:
|
||||
if hasattr(comp_out, "finish_reason"):
|
||||
assert comp_out.finish_reason == "stop", (
|
||||
f"LLM stage finish_reason={comp_out.finish_reason}, "
|
||||
f"text={comp_out.text!r}. "
|
||||
"FP8 may have leaked to the LLM stage."
|
||||
)
|
||||
|
||||
return generated_image, peak_mem
|
||||
|
||||
|
||||
# ─── Single-stage diffusion model tests ──────────────────────────────────────
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_single_stage_zimage_fp8():
|
||||
"""Z-Image-Turbo with FP8 generates valid images."""
|
||||
images, _ = _generate_single_stage_image(
|
||||
model="Tongyi-MAI/Z-Image-Turbo",
|
||||
quantization="fp8",
|
||||
)
|
||||
assert len(images) >= 1
|
||||
images[0].save("test_zimage_fp8.png")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_single_stage_zimage_fp8_uses_less_memory():
|
||||
"""FP8 should use less peak memory than BF16 for Z-Image-Turbo."""
|
||||
_, mem_bf16 = _generate_single_stage_image(
|
||||
model="Tongyi-MAI/Z-Image-Turbo",
|
||||
quantization=None,
|
||||
)
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
_, mem_fp8 = _generate_single_stage_image(
|
||||
model="Tongyi-MAI/Z-Image-Turbo",
|
||||
quantization="fp8",
|
||||
)
|
||||
|
||||
print(f"Z-Image BF16 peak memory: {mem_bf16:.2f} GiB")
|
||||
print(f"Z-Image FP8 peak memory: {mem_fp8:.2f} GiB")
|
||||
assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_single_stage_qwen_image_fp8():
|
||||
"""Qwen-Image (random weights) with FP8 generates valid images."""
|
||||
model = "riverclouds/qwen_image_random"
|
||||
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
pytest.skip("qwen_image_random not available on this platform")
|
||||
|
||||
images, _ = _generate_single_stage_image(
|
||||
model=model,
|
||||
quantization="fp8",
|
||||
)
|
||||
assert len(images) >= 1
|
||||
images[0].save("test_qwen_image_fp8.png")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_single_stage_flux_fp8():
|
||||
"""FLUX.1-dev with FP8 generates valid images."""
|
||||
images, _ = _generate_single_stage_image(
|
||||
model="black-forest-labs/FLUX.1-dev",
|
||||
quantization="fp8",
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=4,
|
||||
)
|
||||
assert len(images) >= 1
|
||||
images[0].save("test_flux_fp8.png")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_single_stage_flux_fp8_uses_less_memory():
|
||||
"""FP8 should use less peak memory than BF16 for FLUX.1-dev."""
|
||||
_, mem_bf16 = _generate_single_stage_image(
|
||||
model="black-forest-labs/FLUX.1-dev",
|
||||
quantization=None,
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=4,
|
||||
)
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
_, mem_fp8 = _generate_single_stage_image(
|
||||
model="black-forest-labs/FLUX.1-dev",
|
||||
quantization="fp8",
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=4,
|
||||
)
|
||||
|
||||
print(f"FLUX BF16 peak memory: {mem_bf16:.2f} GiB")
|
||||
print(f"FLUX FP8 peak memory: {mem_fp8:.2f} GiB")
|
||||
assert mem_fp8 < mem_bf16, f"FP8 ({mem_fp8:.2f} GiB) should use less memory than BF16 ({mem_bf16:.2f} GiB)"
|
||||
|
||||
|
||||
# ─── Multi-stage model tests (BAGEL) ─────────────────────────────────────────
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_bagel_fp8_generates_image():
|
||||
"""BAGEL with FP8 quantization_config generates a valid image.
|
||||
|
||||
FP8 should only apply to the diffusion stage (Stage-1), not the
|
||||
LLM stage (Stage-0). We verify this by checking:
|
||||
1. Image is generated successfully
|
||||
2. LLM stage finish_reason is 'stop' (not 'length' from garbled output)
|
||||
"""
|
||||
image, _ = _generate_bagel_image(quantization_config="fp8")
|
||||
image.save("test_bagel_fp8.png")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_bagel_bf16_generates_image():
|
||||
"""BAGEL without quantization generates a valid image (baseline)."""
|
||||
image, _ = _generate_bagel_image(quantization_config=None)
|
||||
image.save("test_bagel_bf16.png")
|
||||
|
||||
|
||||
# ─── Quantization config routing tests ────────────────────────────────────────
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_quantization_key_maps_to_quantization_config():
|
||||
"""The old 'quantization' kwarg should map to 'quantization_config'
|
||||
in OmniDiffusionConfig.from_kwargs for backwards compatibility."""
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
config = OmniDiffusionConfig.from_kwargs(model="test", quantization="fp8")
|
||||
assert config.quantization_config is not None
|
||||
assert config.quantization_config.get_name() == "fp8"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_quantization_config_key_takes_priority():
|
||||
"""When both 'quantization' and 'quantization_config' are set,
|
||||
'quantization_config' takes priority."""
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
config = OmniDiffusionConfig.from_kwargs(
|
||||
model="test",
|
||||
quantization="fp8",
|
||||
quantization_config={"method": "fp8", "activation_scheme": "static"},
|
||||
)
|
||||
assert config.quantization_config is not None
|
||||
assert config.quantization_config.activation_scheme == "static"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"})
|
||||
def test_single_stage_quantization_config_key():
|
||||
"""Single-stage model using quantization_config (dict) key generates images."""
|
||||
images, _ = _generate_single_stage_image(
|
||||
model="Tongyi-MAI/Z-Image-Turbo",
|
||||
quantization=None,
|
||||
quantization_config="fp8",
|
||||
)
|
||||
assert len(images) >= 1
|
||||
+10
-15
@@ -22,6 +22,11 @@ from tests.helpers.media import (
|
||||
)
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.omni,
|
||||
]
|
||||
|
||||
QUANTIZED_MODEL = "Intel/Qwen2.5-Omni-7B-int4-AutoRound"
|
||||
BASELINE_MODEL = "Qwen/Qwen2.5-Omni-7B"
|
||||
|
||||
@@ -56,8 +61,6 @@ quant_params = [(QUANTIZED_MODEL, stage_config)]
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_text_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -66,7 +69,7 @@ def test_text_to_text(omni_runner, omni_runner_handler):
|
||||
"prompts": "What is the capital of China?",
|
||||
"modalities": ["text"],
|
||||
}
|
||||
response = omni_runner_handler.send_request(request_config)
|
||||
response = omni_runner_handler.send_omni_request(request_config)
|
||||
assert response.success, f"Request failed: {response.error_message}"
|
||||
assert response.text_content and len(response.text_content.strip()) > 0
|
||||
|
||||
@@ -76,8 +79,6 @@ def test_text_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_audio_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -91,7 +92,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler):
|
||||
"audios": (audio, 16000),
|
||||
"modalities": ["text"],
|
||||
}
|
||||
response = omni_runner_handler.send_request(request_config)
|
||||
response = omni_runner_handler.send_omni_request(request_config)
|
||||
assert response.success, f"Request failed: {response.error_message}"
|
||||
assert response.text_content and len(response.text_content.strip()) > 0
|
||||
|
||||
@@ -101,8 +102,6 @@ def test_audio_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_image_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -114,7 +113,7 @@ def test_image_to_text(omni_runner, omni_runner_handler):
|
||||
"images": image,
|
||||
"modalities": ["text"],
|
||||
}
|
||||
response = omni_runner_handler.send_request(request_config)
|
||||
response = omni_runner_handler.send_omni_request(request_config)
|
||||
assert response.success, f"Request failed: {response.error_message}"
|
||||
assert response.text_content and len(response.text_content.strip()) > 0
|
||||
|
||||
@@ -124,8 +123,6 @@ def test_image_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_video_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -137,7 +134,7 @@ def test_video_to_text(omni_runner, omni_runner_handler):
|
||||
"videos": video,
|
||||
"modalities": ["text"],
|
||||
}
|
||||
response = omni_runner_handler.send_request(request_config)
|
||||
response = omni_runner_handler.send_omni_request(request_config)
|
||||
assert response.success, f"Request failed: {response.error_message}"
|
||||
assert response.text_content and len(response.text_content.strip()) > 0
|
||||
|
||||
@@ -147,8 +144,6 @@ def test_video_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_mix_to_audio(omni_runner, omni_runner_handler):
|
||||
@@ -166,5 +161,5 @@ def test_mix_to_audio(omni_runner, omni_runner_handler):
|
||||
"audios": (audio, 16000),
|
||||
"modalities": ["audio"],
|
||||
}
|
||||
response = omni_runner_handler.send_request(request_config)
|
||||
response = omni_runner_handler.send_omni_request(request_config)
|
||||
assert response.success, f"Request failed: {response.error_message}"
|
||||
@@ -78,7 +78,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
}
|
||||
|
||||
# Test single completion
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -98,4 +98,4 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
request_config = {"prompts": get_question("text_only"), "modalities": ["text"]}
|
||||
|
||||
# Test single completion
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
@@ -61,4 +61,4 @@ def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
request_config = {"prompts": get_question(), "videos": video, "modalities": ["audio"]}
|
||||
|
||||
# Test single completion
|
||||
omni_runner_handler.send_request(request_config)
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
+15
-12
@@ -22,6 +22,15 @@ from tests.helpers.media import (
|
||||
)
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.omni,
|
||||
]
|
||||
|
||||
_SKIP_ISSUE_3195 = pytest.mark.skip(
|
||||
reason="https://github.com/vllm-project/vllm-omni/issues/3195",
|
||||
)
|
||||
|
||||
QUANTIZED_MODEL = "Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound"
|
||||
BASELINE_MODEL = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
@@ -69,8 +78,7 @@ quant_params = [(QUANTIZED_MODEL, stage_config)]
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_ISSUE_3195
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_text_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -89,8 +97,7 @@ def test_text_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_ISSUE_3195
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_audio_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -114,8 +121,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_ISSUE_3195
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_image_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -137,8 +143,7 @@ def test_image_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_ISSUE_3195
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_video_to_text(omni_runner, omni_runner_handler):
|
||||
@@ -160,8 +165,7 @@ def test_video_to_text(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_ISSUE_3195
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_video_to_audio(omni_runner, omni_runner_handler):
|
||||
@@ -182,8 +186,7 @@ def test_video_to_audio(omni_runner, omni_runner_handler):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_ISSUE_3195
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", quant_params, indirect=True)
|
||||
def test_mix_to_audio(omni_runner, omni_runner_handler):
|
||||
@@ -68,7 +68,7 @@ def get_prompt():
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@@ -64,7 +64,7 @@ def get_prompt():
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@@ -1,626 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
End-to-end test for diffusion batching via AsyncOmni.
|
||||
|
||||
This test fires multiple concurrent ``AsyncOmni.generate()`` calls for a
|
||||
diffusion model and validates that every caller receives its correct
|
||||
individual result. When the underlying diffusion stage is configured with
|
||||
``batch_size > 1`` (via stage config or ``StageDiffusionClient``), the
|
||||
requests will be batched internally.
|
||||
|
||||
Even without explicit batching config this test is useful for verifying
|
||||
that concurrent async requests are handled correctly.
|
||||
|
||||
Usage (standalone):
|
||||
|
||||
python tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py \
|
||||
--model <model_name_or_path> \
|
||||
--num-prompts 8
|
||||
|
||||
Or via pytest:
|
||||
|
||||
pytest tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -s
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
models = ["tiny-random/Qwen-Image"]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Prompt fixtures
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
WARMUP_PROMPTS: list[dict[str, str]] = [
|
||||
{"prompt": "a sunflower in a glass vase", "negative_prompt": "blurry"},
|
||||
{"prompt": "a rocket launching into space", "negative_prompt": "low detail"},
|
||||
{"prompt": "a small cottage in the snowy mountains", "negative_prompt": "foggy"},
|
||||
{"prompt": "a colorful parrot sitting on a tree branch", "negative_prompt": "low contrast"},
|
||||
]
|
||||
|
||||
TEST_PROMPTS: list[dict[str, str]] = [
|
||||
{"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"},
|
||||
{"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": "cinematic, realistic"},
|
||||
{"prompt": "a futuristic city skyline at sunset", "negative_prompt": "blurry, foggy"},
|
||||
{"prompt": "a bowl of fresh strawberries", "negative_prompt": "low detail"},
|
||||
{"prompt": "a medieval knight standing in the rain", "negative_prompt": "modern clothing"},
|
||||
{"prompt": "a cat wearing sunglasses lounging in a garden", "negative_prompt": "dark lighting"},
|
||||
{"prompt": "a spaceship flying above a volcano", "negative_prompt": "low contrast"},
|
||||
{"prompt": "a watercolor painting of a mountain lake", "negative_prompt": "photo, realistic"},
|
||||
]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _default_sampling_params(**overrides) -> OmniDiffusionSamplingParams:
|
||||
defaults = dict(
|
||||
num_inference_steps=2,
|
||||
width=256,
|
||||
height=256,
|
||||
guidance_scale=0.0,
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return OmniDiffusionSamplingParams(**defaults)
|
||||
|
||||
|
||||
def _default_sync_sampling_params(**overrides) -> OmniDiffusionSamplingParams:
|
||||
"""Create sampling params for the synchronous Omni.generate() API."""
|
||||
defaults = dict(
|
||||
num_inference_steps=2,
|
||||
width=256,
|
||||
height=256,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return OmniDiffusionSamplingParams(**defaults)
|
||||
|
||||
|
||||
async def _collect_generate(omni: AsyncOmni, prompt, request_id, sampling_params_list) -> OmniRequestOutput:
|
||||
"""Consume the AsyncOmni.generate() async generator and return the last output."""
|
||||
last_output: OmniRequestOutput | None = None
|
||||
async for output in omni.generate(
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=sampling_params_list,
|
||||
):
|
||||
last_output = output
|
||||
if last_output is None:
|
||||
raise RuntimeError(f"No output received for request {request_id}")
|
||||
return last_output
|
||||
|
||||
|
||||
def _extract_images(output: OmniRequestOutput) -> list:
|
||||
"""Extract images from an OmniRequestOutput, handling both direct
|
||||
and nested request_output structures."""
|
||||
if output.images:
|
||||
return output.images
|
||||
# When the output comes from the orchestrator pipeline, images may be
|
||||
# nested inside request_output.
|
||||
inner = getattr(output, "request_output", None)
|
||||
if inner is not None and hasattr(inner, "images") and inner.images:
|
||||
return inner.images
|
||||
return []
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Warm-up (async)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def warmup(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
|
||||
"""Warm-up: send prompts in parallel to pre-load the model."""
|
||||
print(f"🔥 Warming up with {len(prompts)} prompts ...")
|
||||
sp = _default_sampling_params(num_inference_steps=2)
|
||||
start = time.perf_counter()
|
||||
|
||||
tasks = [
|
||||
_collect_generate(
|
||||
omni,
|
||||
prompt=p,
|
||||
request_id=f"warmup-{i}-{uuid.uuid4().hex[:8]}",
|
||||
sampling_params_list=[sp],
|
||||
)
|
||||
for i, p in enumerate(prompts)
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
elapsed = time.perf_counter() - start
|
||||
print(f" Warm-up done in {elapsed:.2f}s\n")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Single (sequential) benchmark
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_single(omni: AsyncOmni, prompts: list[dict[str, str]]) -> float:
|
||||
"""Run prompts one-by-one sequentially."""
|
||||
print(f"🧩 Running SINGLE (sequential) mode – {len(prompts)} prompts ...")
|
||||
sp = _default_sampling_params()
|
||||
total_start = time.perf_counter()
|
||||
|
||||
for i, prompt in enumerate(prompts):
|
||||
start = time.perf_counter()
|
||||
result = await _collect_generate(
|
||||
omni,
|
||||
prompt=prompt,
|
||||
request_id=f"single-{i}-{uuid.uuid4().hex[:8]}",
|
||||
sampling_params_list=[sp],
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
images = _extract_images(result)
|
||||
print(f" prompt {i}: {elapsed:.2f}s ({len(images)} images)")
|
||||
|
||||
total = time.perf_counter() - total_start
|
||||
print(f" ✅ Total single-mode: {total:.2f}s\n")
|
||||
return total
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Batch (parallel) benchmark — concurrent individual requests
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_batch(
|
||||
omni: AsyncOmni,
|
||||
prompts: list[dict[str, str]],
|
||||
label: str = "batch",
|
||||
) -> float:
|
||||
"""Send all prompts concurrently via asyncio.gather (one request per prompt)."""
|
||||
print(f"⚙️ Running {label.upper()} mode – {len(prompts)} prompts concurrently ...")
|
||||
sp = _default_sampling_params()
|
||||
start = time.perf_counter()
|
||||
|
||||
tasks = [
|
||||
_collect_generate(
|
||||
omni,
|
||||
prompt=p,
|
||||
request_id=f"{label}-{i}-{uuid.uuid4().hex[:8]}",
|
||||
sampling_params_list=[sp],
|
||||
)
|
||||
for i, p in enumerate(prompts)
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
elapsed = time.perf_counter() - start
|
||||
for i, result in enumerate(results):
|
||||
images = _extract_images(result)
|
||||
print(f" prompt {i}: {len(images)} images, request_id={result.request_id}")
|
||||
|
||||
print(f" ✅ Total {label} mode: {elapsed:.2f}s\n")
|
||||
return elapsed
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Explicit batch — single generate() call with list of prompts
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_batch_explicit(
|
||||
omni: AsyncOmni,
|
||||
prompts: list[dict[str, str]],
|
||||
label: str = "batch_explicit",
|
||||
) -> float:
|
||||
"""Send all prompts as a single batch via generate(prompt=[...]).
|
||||
|
||||
Passes a *list* of prompts so that all are processed in **one**
|
||||
``DiffusionEngine.step()`` call. The orchestrator detects
|
||||
``isinstance(prompt, list)`` and routes to the batch path.
|
||||
|
||||
A single ``OmniRequestOutput`` is yielded containing ALL generated
|
||||
images combined.
|
||||
"""
|
||||
print(f"⚡ Running {label.upper()} mode – {len(prompts)} prompts in ONE engine call ...")
|
||||
sp = _default_sampling_params()
|
||||
request_id = f"{label}-{uuid.uuid4().hex[:8]}"
|
||||
start = time.perf_counter()
|
||||
|
||||
result: OmniRequestOutput | None = None
|
||||
async for output in omni.generate(
|
||||
prompt=prompts,
|
||||
request_id=request_id,
|
||||
sampling_params_list=[sp],
|
||||
):
|
||||
result = output
|
||||
|
||||
elapsed = time.perf_counter() - start
|
||||
if result is not None:
|
||||
images = _extract_images(result)
|
||||
print(f" Got {len(images)} images total from batch, request_id={result.request_id}")
|
||||
else:
|
||||
print(" ⚠️ No output received from batch generate()")
|
||||
|
||||
print(f" ✅ Total {label} mode: {elapsed:.2f}s\n")
|
||||
return elapsed
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Async validation helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def validate_batch_explicit(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
|
||||
"""Validate generate(prompt=[...]) returns a single result with all images."""
|
||||
print(f"🔍 Validating batch generate() correctness with {len(prompts)} prompts ...")
|
||||
sp = _default_sampling_params()
|
||||
request_id = f"validate-batch-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
result: OmniRequestOutput | None = None
|
||||
async for output in omni.generate(
|
||||
prompt=prompts,
|
||||
request_id=request_id,
|
||||
sampling_params_list=[sp],
|
||||
):
|
||||
result = output
|
||||
|
||||
assert result is not None, "No output received from batch generate()"
|
||||
images = _extract_images(result)
|
||||
# Batch mode returns ONE output with ALL images combined
|
||||
assert len(images) == len(prompts), f"Expected {len(prompts)} images (one per prompt), got {len(images)}"
|
||||
assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}"
|
||||
print(f" ✅ Batch returned {len(images)} images with correct request_id.\n")
|
||||
|
||||
|
||||
async def validate_concurrent(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
|
||||
"""Validate that every concurrent request receives a distinct result
|
||||
with its own request_id."""
|
||||
print(f"🔍 Validating concurrent correctness with {len(prompts)} prompts ...")
|
||||
sp = _default_sampling_params()
|
||||
|
||||
request_ids = [f"validate-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))]
|
||||
|
||||
tasks = [
|
||||
_collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp])
|
||||
for p, rid in zip(prompts, request_ids)
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}"
|
||||
|
||||
returned_ids = [r.request_id for r in results]
|
||||
for rid in request_ids:
|
||||
assert rid in returned_ids, f"Missing request_id {rid} in results"
|
||||
|
||||
print(" ✅ All request_ids matched, results count correct.\n")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Single vs Parallel comparison (CLI only)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def compare_single_vs_parallel(
|
||||
model: str,
|
||||
prompts: list[dict[str, str]],
|
||||
batch_size: int = 1,
|
||||
) -> None:
|
||||
"""Run the same prompts sequentially then in parallel and print a comparison."""
|
||||
|
||||
omni = AsyncOmni(model=model, diffusion_batch_size=batch_size)
|
||||
try:
|
||||
await warmup(omni, WARMUP_PROMPTS)
|
||||
single_time = await run_single(omni, prompts)
|
||||
parallel_time = await run_batch(omni, prompts, label="parallel")
|
||||
explicit_time = await run_batch_explicit(omni, prompts, label="batch_explicit")
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
speedup_parallel = single_time / parallel_time if parallel_time > 0 else float("inf")
|
||||
speedup_explicit = single_time / explicit_time if explicit_time > 0 else float("inf")
|
||||
print("=" * 60)
|
||||
print(f"📊 Summary ({len(prompts)} prompts)")
|
||||
print(f" Sequential : {single_time:.2f}s")
|
||||
print(f" Parallel (gather) : {parallel_time:.2f}s ({speedup_parallel:.2f}x)")
|
||||
print(f" Explicit batch : {explicit_time:.2f}s ({speedup_explicit:.2f}x)")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CLI main entrypoint
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
async def main(model: str, num_prompts: int, mode: str, batch_size: int = 1) -> None:
|
||||
prompts = (TEST_PROMPTS * ((num_prompts // len(TEST_PROMPTS)) + 1))[:num_prompts]
|
||||
|
||||
if mode == "compare":
|
||||
await compare_single_vs_parallel(model, prompts, batch_size=batch_size)
|
||||
return
|
||||
|
||||
omni = AsyncOmni(model=model, diffusion_batch_size=batch_size)
|
||||
try:
|
||||
await warmup(omni, WARMUP_PROMPTS)
|
||||
|
||||
if mode == "validate":
|
||||
await validate_concurrent(omni, prompts)
|
||||
elif mode == "validate_batch":
|
||||
await validate_batch_explicit(omni, prompts)
|
||||
elif mode == "batch":
|
||||
await run_batch(omni, prompts, label="measurement")
|
||||
elif mode == "batch_explicit":
|
||||
await run_batch_explicit(omni, prompts)
|
||||
elif mode == "single":
|
||||
await run_single(omni, prompts)
|
||||
else:
|
||||
raise ValueError(f"Unknown mode: {mode}")
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
# ==================================================================
|
||||
# pytest test cases
|
||||
# ==================================================================
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_batching_sync_sequential(model_name: str):
|
||||
"""Test that synchronous Omni can generate images for multiple prompts
|
||||
submitted sequentially (one at a time) and each returns a valid image."""
|
||||
try:
|
||||
with OmniRunner(model_name) as runner:
|
||||
m = runner.omni
|
||||
sp = _default_sync_sampling_params()
|
||||
prompts = TEST_PROMPTS[:4]
|
||||
|
||||
for i, prompt in enumerate(prompts):
|
||||
outputs = m.generate(prompt, sp)
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image", (
|
||||
f"Expected 'image', got '{first_output.final_output_type}'"
|
||||
)
|
||||
|
||||
# Images are surfaced both at top-level and inside request_output
|
||||
images = _extract_images(first_output)
|
||||
assert len(images) >= 1, f"Expected at least 1 image for prompt {i}, got {len(images)}"
|
||||
assert images[0].width == 256
|
||||
assert images[0].height == 256
|
||||
print(f" prompt {i}: OK ({len(images)} images)")
|
||||
except Exception as e:
|
||||
print(f"Test failed with error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_batching_sync_multi_prompt(model_name: str):
|
||||
"""Test that synchronous Omni correctly handles a list of multiple
|
||||
prompts submitted at once and returns one result per prompt.
|
||||
|
||||
Note: Omni.generate() iterates the list and submits each prompt
|
||||
individually with its own request_id. This tests concurrent request
|
||||
handling at the diffusion stage, not the explicit list-batch path
|
||||
(which is only available via AsyncOmni).
|
||||
"""
|
||||
try:
|
||||
with OmniRunner(model_name) as runner:
|
||||
m = runner.omni
|
||||
sp = _default_sync_sampling_params()
|
||||
prompts = TEST_PROMPTS[:4]
|
||||
|
||||
outputs = m.generate(prompts, sp)
|
||||
assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}"
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
assert output.final_output_type == "image", (
|
||||
f"Output {i} final_output_type expected 'image', got '{output.final_output_type}'"
|
||||
)
|
||||
images = _extract_images(output)
|
||||
assert images and len(images) >= 1, f"Expected at least 1 image for prompt {i}"
|
||||
assert images[0].width == 256
|
||||
assert images[0].height == 256
|
||||
print(f" prompt {i}: OK ({len(images)} images, request_id={output.request_id})")
|
||||
|
||||
# Verify all request_ids are distinct
|
||||
request_ids = [o.request_id for o in outputs]
|
||||
assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids found: {request_ids}"
|
||||
except Exception as e:
|
||||
print(f"Test failed with error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_batching_async_concurrent(model_name: str):
|
||||
"""Test that AsyncOmni correctly handles multiple concurrent requests
|
||||
fired via asyncio.gather. Each request_id must appear in the results."""
|
||||
|
||||
async def _inner():
|
||||
omni = AsyncOmni(model=model_name, diffusion_batch_size=1)
|
||||
try:
|
||||
prompts = TEST_PROMPTS[:4]
|
||||
sp = _default_sampling_params()
|
||||
request_ids = [f"async-concurrent-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))]
|
||||
|
||||
tasks = [
|
||||
_collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp])
|
||||
for p, rid in zip(prompts, request_ids)
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}"
|
||||
|
||||
returned_ids = [r.request_id for r in results]
|
||||
for rid in request_ids:
|
||||
assert rid in returned_ids, f"Missing request_id {rid} in results"
|
||||
|
||||
for i, result in enumerate(results):
|
||||
images = _extract_images(result)
|
||||
assert len(images) >= 1, f"No images for prompt {i}"
|
||||
assert images[0].width == 256
|
||||
assert images[0].height == 256
|
||||
print(f" prompt {i}: OK ({len(images)} images, request_id={result.request_id})")
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
asyncio.run(_inner())
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_batching_async_explicit_batch(model_name: str):
|
||||
"""Test that AsyncOmni batch mode (generate(prompt=[...])) dispatches
|
||||
all prompts in a single engine call and returns a single combined result.
|
||||
|
||||
The list-prompt path routes through the orchestrator's
|
||||
``add_batch_request_async`` → ``AsyncOmni.generate_batch``
|
||||
and yields ONE ``OmniRequestOutput`` with ALL images combined.
|
||||
"""
|
||||
|
||||
async def _inner():
|
||||
omni = AsyncOmni(model=model_name, diffusion_batch_size=4)
|
||||
try:
|
||||
prompts = TEST_PROMPTS[:4]
|
||||
sp = _default_sampling_params()
|
||||
request_id = f"explicit-batch-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
# Batch mode: pass list of prompts → single request_id
|
||||
result: OmniRequestOutput | None = None
|
||||
async for output in omni.generate(
|
||||
prompt=prompts,
|
||||
request_id=request_id,
|
||||
sampling_params_list=[sp],
|
||||
):
|
||||
result = output
|
||||
|
||||
assert result is not None, "No output received from batch generate()"
|
||||
|
||||
images = _extract_images(result)
|
||||
# One image per prompt, all in a single output
|
||||
assert len(images) == len(prompts), f"Expected {len(prompts)} images in combined output, got {len(images)}"
|
||||
assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}"
|
||||
for i, img in enumerate(images):
|
||||
assert img.width == 256, f"Image {i} width mismatch"
|
||||
assert img.height == 256, f"Image {i} height mismatch"
|
||||
print(f" ✅ Batch returned {len(images)} images, request_id={result.request_id}")
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
asyncio.run(_inner())
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_batching_num_outputs(model_name: str):
|
||||
"""Test that the diffusion model respects num_outputs_per_prompt and
|
||||
generates the correct number of images per request."""
|
||||
try:
|
||||
with OmniRunner(model_name) as runner:
|
||||
m = runner.omni
|
||||
num_outputs = 2
|
||||
sp = _default_sync_sampling_params(num_outputs_per_prompt=num_outputs)
|
||||
|
||||
outputs = m.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
sp,
|
||||
)
|
||||
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
images = _extract_images(first_output)
|
||||
assert images is not None and len(images) == num_outputs, (
|
||||
f"Expected {num_outputs} images, got {len(images) if images else 0}"
|
||||
)
|
||||
for img in images:
|
||||
assert img.width == 256
|
||||
assert img.height == 256
|
||||
except Exception as e:
|
||||
print(f"Test failed with error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_batching_distinct_results(model_name: str):
|
||||
"""Test that different prompts produce distinct images when batched,
|
||||
ensuring the batching logic does not mix up results across requests."""
|
||||
try:
|
||||
with OmniRunner(model_name) as runner:
|
||||
m = runner.omni
|
||||
sp = _default_sync_sampling_params()
|
||||
prompts = [
|
||||
{"prompt": "a bright red apple on a white table", "negative_prompt": "blurry"},
|
||||
{"prompt": "a blue ocean with white waves crashing", "negative_prompt": "blurry"},
|
||||
]
|
||||
|
||||
outputs = m.generate(prompts, sp)
|
||||
assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}"
|
||||
|
||||
# Verify each output has a unique request_id
|
||||
request_ids = [o.request_id for o in outputs]
|
||||
assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids: {request_ids}"
|
||||
|
||||
# Verify each output has images
|
||||
for i, output in enumerate(outputs):
|
||||
images = _extract_images(output)
|
||||
assert images and len(images) >= 1, f"No images for prompt {i}"
|
||||
assert images[0].width == 256
|
||||
assert images[0].height == 256
|
||||
except Exception as e:
|
||||
print(f"Test failed with error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CLI
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="E2E diffusion concurrent benchmark / validation")
|
||||
parser.add_argument("--model", type=str, required=True, help="Model name or path")
|
||||
parser.add_argument("--num-prompts", type=int, default=8, help="Number of prompts to run")
|
||||
parser.add_argument("--batch-size", type=int, default=1, help="Diffusion batch size (1 = no batching)")
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=["batch", "batch_explicit", "single", "compare", "validate", "validate_batch"],
|
||||
default="compare",
|
||||
help=(
|
||||
"Run mode: 'batch' (parallel gather), 'batch_explicit' (list-prompt batch API), "
|
||||
"'single' (sequential), 'compare' (all three), "
|
||||
"'validate' (concurrent correctness), 'validate_batch' (list-prompt correctness)"
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
asyncio.run(main(args.model, args.num_prompts, args.mode, batch_size=args.batch_size))
|
||||
@@ -1,414 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
System test for Sequence Parallel (SP) backends: Ulysses and Ring attention.
|
||||
|
||||
Tests verify that SP inference produces correct outputs compared to baseline.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
# Test configuration
|
||||
MODELS = ["riverclouds/qwen_image_random"]
|
||||
PROMPT = "a photo of a cat sitting on a laptop keyboard"
|
||||
DEFAULT_HEIGHT = 256
|
||||
DEFAULT_WIDTH = 256
|
||||
DEFAULT_SEED = 42
|
||||
DEFAULT_STEPS = 4
|
||||
DIFF_MEAN_THRESHOLD = 2e-2
|
||||
DIFF_MAX_THRESHOLD = 2e-1
|
||||
|
||||
|
||||
class InferenceResult(NamedTuple):
|
||||
"""Result of an inference run."""
|
||||
|
||||
images: list[Image.Image]
|
||||
elapsed_ms: float
|
||||
|
||||
|
||||
def _cleanup_distributed():
|
||||
"""Clean up distributed environment and GPU resources."""
|
||||
if dist.is_initialized():
|
||||
dist.destroy_process_group()
|
||||
|
||||
for key in ["MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"]:
|
||||
os.environ.pop(key, None)
|
||||
|
||||
gc.collect()
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
|
||||
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
|
||||
ta = torch.from_numpy(np.asarray(a.convert("RGB"), dtype=np.float32) / 255.0)
|
||||
tb = torch.from_numpy(np.asarray(b.convert("RGB"), dtype=np.float32) / 255.0)
|
||||
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
|
||||
abs_diff = torch.abs(ta - tb)
|
||||
return abs_diff.mean().item(), abs_diff.max().item()
|
||||
|
||||
|
||||
def _run_inference(
|
||||
model_name: str,
|
||||
dtype: torch.dtype,
|
||||
attn_backend: str,
|
||||
ulysses_degree: int = 1,
|
||||
ring_degree: int = 1,
|
||||
height: int = DEFAULT_HEIGHT,
|
||||
width: int = DEFAULT_WIDTH,
|
||||
seed: int = DEFAULT_SEED,
|
||||
warmup: bool = True,
|
||||
) -> InferenceResult:
|
||||
"""Run inference with specified configuration.
|
||||
|
||||
Args:
|
||||
warmup: If True, run one warmup iteration before the timed run.
|
||||
"""
|
||||
parallel_config = DiffusionParallelConfig(ulysses_degree=ulysses_degree, ring_degree=ring_degree)
|
||||
try:
|
||||
with OmniRunner(
|
||||
model_name,
|
||||
parallel_config=parallel_config,
|
||||
dtype=dtype,
|
||||
attention_backend=attn_backend,
|
||||
) as runner:
|
||||
omni = runner.omni
|
||||
# Warmup run (not timed)
|
||||
if warmup:
|
||||
_ = omni.generate(
|
||||
PROMPT,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=DEFAULT_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed + 1000),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
|
||||
# Timed run
|
||||
start = time.time()
|
||||
outputs = omni.generate(
|
||||
PROMPT,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=DEFAULT_STEPS,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
elapsed_ms = (time.time() - start) * 1000
|
||||
|
||||
return InferenceResult(
|
||||
images=outputs[0].request_output.images,
|
||||
elapsed_ms=elapsed_ms,
|
||||
)
|
||||
finally:
|
||||
_cleanup_distributed()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Correctness & Performance Tests
|
||||
# =============================================================================
|
||||
|
||||
# SP configurations: (ulysses_degree, ring_degree, height, width, warmup, is_perf_test)
|
||||
# - warmup: whether to run warmup for this SP config
|
||||
# - is_perf_test: whether this is a performance test (show speedup metrics)
|
||||
SP_CONFIGS_L2 = [
|
||||
# Hybrid - correctness only
|
||||
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
|
||||
]
|
||||
|
||||
SP_CONFIGS_L3 = [
|
||||
# Ulysses-2 - performance test
|
||||
(2, 1, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True),
|
||||
(1, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True), # Ring-2 - performance test
|
||||
# Hybrid - correctness only
|
||||
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
|
||||
(4, 1, 272, 272, False, False), # Ulysses-4 - shape and correctness
|
||||
]
|
||||
|
||||
|
||||
def _get_sp_mode(ulysses_degree: int, ring_degree: int) -> str:
|
||||
"""Get SP mode name for logging."""
|
||||
if ulysses_degree > 1 and ring_degree == 1:
|
||||
return f"ulysses-{ulysses_degree}"
|
||||
elif ring_degree > 1 and ulysses_degree == 1:
|
||||
return f"ring-{ring_degree}"
|
||||
else:
|
||||
return f"hybrid-{ulysses_degree}x{ring_degree}"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 2, "rocm": 2})
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_sp_correctness(model_name: str):
|
||||
"""Test that SP inference produces correct outputs and measure performance.
|
||||
|
||||
Runs baseline once per unique (height, width), then tests all SP configs.
|
||||
|
||||
Note: Run with `pytest -v -s` to see detailed output.
|
||||
"""
|
||||
device_count = current_omni_platform.get_device_count()
|
||||
|
||||
# Cache baseline results by (height, width)
|
||||
# Key: (height, width), Value: (result, warmup_used)
|
||||
baseline_cache: dict[tuple[int, int], InferenceResult] = {}
|
||||
|
||||
# Collect results for summary
|
||||
results: list[dict] = []
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(f"Sequence Parallel Test - Model: {model_name}")
|
||||
print(f"Available GPUs: {device_count}")
|
||||
print("=" * 70)
|
||||
|
||||
for ulysses_degree, ring_degree, height, width, sp_warmup, is_perf_test in SP_CONFIGS_L2:
|
||||
sp_size = ulysses_degree * ring_degree
|
||||
sp_mode = _get_sp_mode(ulysses_degree, ring_degree)
|
||||
|
||||
if device_count < sp_size:
|
||||
print(f"\n[{sp_mode}] SKIPPED (requires {sp_size} GPUs)")
|
||||
continue
|
||||
|
||||
# Determine baseline warmup: only for default size (performance tests)
|
||||
cache_key = (height, width)
|
||||
baseline_warmup = height == DEFAULT_HEIGHT and width == DEFAULT_WIDTH
|
||||
|
||||
# Get or compute baseline for this (height, width)
|
||||
if cache_key not in baseline_cache:
|
||||
print(f"\n--- Running baseline {height}x{width} (warmup={baseline_warmup}) ---")
|
||||
baseline = _run_inference(
|
||||
model_name,
|
||||
torch.bfloat16,
|
||||
"sdpa",
|
||||
height=height,
|
||||
width=width,
|
||||
warmup=baseline_warmup,
|
||||
)
|
||||
assert len(baseline.images) == 1
|
||||
baseline_cache[cache_key] = baseline
|
||||
print(f"[baseline] {height}x{width}: {baseline.elapsed_ms:.0f}ms")
|
||||
else:
|
||||
baseline = baseline_cache[cache_key]
|
||||
|
||||
# Run SP
|
||||
print(f"\n--- Running {sp_mode} (warmup={sp_warmup}) ---")
|
||||
sp_result = _run_inference(
|
||||
model_name,
|
||||
torch.bfloat16,
|
||||
"sdpa",
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
height=height,
|
||||
width=width,
|
||||
warmup=sp_warmup,
|
||||
)
|
||||
assert len(sp_result.images) == 1
|
||||
|
||||
# Compare outputs (correctness)
|
||||
mean_diff, max_diff = _diff_metrics(baseline.images[0], sp_result.images[0])
|
||||
|
||||
# Build result entry
|
||||
result = {
|
||||
"mode": sp_mode,
|
||||
"sp_size": sp_size,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"baseline_ms": baseline.elapsed_ms,
|
||||
"sp_ms": sp_result.elapsed_ms,
|
||||
"mean_diff": mean_diff,
|
||||
"max_diff": max_diff,
|
||||
"is_perf_test": is_perf_test,
|
||||
}
|
||||
results.append(result)
|
||||
|
||||
# Output based on test type
|
||||
if is_perf_test:
|
||||
speedup = baseline.elapsed_ms / sp_result.elapsed_ms if sp_result.elapsed_ms > 0 else 0
|
||||
result["speedup"] = speedup
|
||||
print(
|
||||
f"[{sp_mode}] {sp_size} GPUs | "
|
||||
f"baseline: {baseline.elapsed_ms:.0f}ms, sp: {sp_result.elapsed_ms:.0f}ms, "
|
||||
f"speedup: {speedup:.2f}x"
|
||||
)
|
||||
else:
|
||||
print(f"[{sp_mode}] {sp_size} GPUs | sp: {sp_result.elapsed_ms:.0f}ms (correctness only)")
|
||||
|
||||
print(f"[{sp_mode}] diff: mean={mean_diff:.6e}, max={max_diff:.6e}")
|
||||
|
||||
# Assert correctness
|
||||
assert mean_diff <= DIFF_MEAN_THRESHOLD and max_diff <= DIFF_MAX_THRESHOLD, (
|
||||
f"[{sp_mode}] SP output differs from baseline: mean={mean_diff:.6e}, max={max_diff:.6e}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 70)
|
||||
print("SUMMARY")
|
||||
print("=" * 70)
|
||||
print(f"{'Mode':<15} {'GPUs':<6} {'Size':<10} {'Baseline':<12} {'SP':<12} {'Speedup':<10} {'Status'}")
|
||||
print("-" * 70)
|
||||
for r in results:
|
||||
speedup_str = f"{r['speedup']:.2f}x" if r.get("speedup") else "N/A"
|
||||
baseline_str = f"{r['baseline_ms']:.0f}ms" if r["is_perf_test"] else "N/A"
|
||||
status = "PASS" if r["mean_diff"] <= DIFF_MEAN_THRESHOLD else "FAIL"
|
||||
print(
|
||||
f"{r['mode']:<15} {r['sp_size']:<6} {r['height']}x{r['width']:<5} "
|
||||
f"{baseline_str:<12} {r['sp_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
|
||||
)
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
# TODO: After PR#1272 is merged, add markers
|
||||
# @pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_sp_correctness_advanced(model_name: str):
|
||||
"""Test that SP inference produces correct outputs and measure performance.
|
||||
|
||||
Runs baseline once per unique (height, width), then tests all SP configs.
|
||||
|
||||
Note: Run with `pytest -v -s` to see detailed output.
|
||||
"""
|
||||
device_count = current_omni_platform.get_device_count()
|
||||
|
||||
# Cache baseline results by (height, width)
|
||||
# Key: (height, width), Value: (result, warmup_used)
|
||||
baseline_cache: dict[tuple[int, int], InferenceResult] = {}
|
||||
|
||||
# Collect results for summary
|
||||
results: list[dict] = []
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(f"Sequence Parallel Test - Model: {model_name}")
|
||||
print(f"Available GPUs: {device_count}")
|
||||
print("=" * 70)
|
||||
|
||||
for ulysses_degree, ring_degree, height, width, sp_warmup, is_perf_test in SP_CONFIGS_L3:
|
||||
sp_size = ulysses_degree * ring_degree
|
||||
sp_mode = _get_sp_mode(ulysses_degree, ring_degree)
|
||||
|
||||
if device_count < sp_size:
|
||||
print(f"\n[{sp_mode}] SKIPPED (requires {sp_size} GPUs)")
|
||||
continue
|
||||
|
||||
# Determine baseline warmup: only for default size (performance tests)
|
||||
cache_key = (height, width)
|
||||
baseline_warmup = height == DEFAULT_HEIGHT and width == DEFAULT_WIDTH
|
||||
|
||||
# Get or compute baseline for this (height, width)
|
||||
if cache_key not in baseline_cache:
|
||||
print(f"\n--- Running baseline {height}x{width} (warmup={baseline_warmup}) ---")
|
||||
baseline = _run_inference(
|
||||
model_name,
|
||||
torch.bfloat16,
|
||||
"sdpa",
|
||||
height=height,
|
||||
width=width,
|
||||
warmup=baseline_warmup,
|
||||
)
|
||||
assert len(baseline.images) == 1
|
||||
baseline_cache[cache_key] = baseline
|
||||
print(f"[baseline] {height}x{width}: {baseline.elapsed_ms:.0f}ms")
|
||||
else:
|
||||
baseline = baseline_cache[cache_key]
|
||||
|
||||
# Run SP
|
||||
print(f"\n--- Running {sp_mode} (warmup={sp_warmup}) ---")
|
||||
sp_result = _run_inference(
|
||||
model_name,
|
||||
torch.bfloat16,
|
||||
"sdpa",
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
height=height,
|
||||
width=width,
|
||||
warmup=sp_warmup,
|
||||
)
|
||||
assert len(sp_result.images) == 1
|
||||
|
||||
# Compare outputs (correctness)
|
||||
mean_diff, max_diff = _diff_metrics(baseline.images[0], sp_result.images[0])
|
||||
|
||||
# Build result entry
|
||||
result = {
|
||||
"mode": sp_mode,
|
||||
"sp_size": sp_size,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"baseline_ms": baseline.elapsed_ms,
|
||||
"sp_ms": sp_result.elapsed_ms,
|
||||
"mean_diff": mean_diff,
|
||||
"max_diff": max_diff,
|
||||
"is_perf_test": is_perf_test,
|
||||
}
|
||||
results.append(result)
|
||||
|
||||
# Output based on test type
|
||||
if is_perf_test:
|
||||
speedup = baseline.elapsed_ms / sp_result.elapsed_ms if sp_result.elapsed_ms > 0 else 0
|
||||
result["speedup"] = speedup
|
||||
print(
|
||||
f"[{sp_mode}] {sp_size} GPUs | "
|
||||
f"baseline: {baseline.elapsed_ms:.0f}ms, sp: {sp_result.elapsed_ms:.0f}ms, "
|
||||
f"speedup: {speedup:.2f}x"
|
||||
)
|
||||
else:
|
||||
print(f"[{sp_mode}] {sp_size} GPUs | sp: {sp_result.elapsed_ms:.0f}ms (correctness only)")
|
||||
|
||||
print(f"[{sp_mode}] diff: mean={mean_diff:.6e}, max={max_diff:.6e}")
|
||||
|
||||
# Assert correctness
|
||||
assert mean_diff <= DIFF_MEAN_THRESHOLD and max_diff <= DIFF_MAX_THRESHOLD, (
|
||||
f"[{sp_mode}] SP output differs from baseline: mean={mean_diff:.6e}, max={max_diff:.6e}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 70)
|
||||
print("SUMMARY")
|
||||
print("=" * 70)
|
||||
print(f"{'Mode':<15} {'GPUs':<6} {'Size':<10} {'Baseline':<12} {'SP':<12} {'Speedup':<10} {'Status'}")
|
||||
print("-" * 70)
|
||||
for r in results:
|
||||
speedup_str = f"{r['speedup']:.2f}x" if r.get("speedup") else "N/A"
|
||||
baseline_str = f"{r['baseline_ms']:.0f}ms" if r["is_perf_test"] else "N/A"
|
||||
status = "PASS" if r["mean_diff"] <= DIFF_MEAN_THRESHOLD else "FAIL"
|
||||
print(
|
||||
f"{r['mode']:<15} {r['sp_size']:<6} {r['height']}x{r['width']:<5} "
|
||||
f"{baseline_str:<12} {r['sp_ms']:.0f}ms{'':<7} {speedup_str:<10} {status}"
|
||||
)
|
||||
print("=" * 70)
|
||||
@@ -1,75 +0,0 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Match unprefixed HF id even when MODEL_PREFIX is set (omni_runner resolves full path).
|
||||
_QWEN_IMAGE_RANDOM_ID = "riverclouds/qwen_image_random"
|
||||
|
||||
|
||||
def _is_qwen_image_random(model_path: str) -> bool:
|
||||
return model_path.rstrip("/").endswith(_QWEN_IMAGE_RANDOM_ID)
|
||||
|
||||
|
||||
models = ["Tongyi-MAI/Z-Image-Turbo", "riverclouds/qwen_image_random"]
|
||||
|
||||
# Modelscope can't find riverclouds/qwen_image_random
|
||||
# TODO: When NPU support is ready, remove this branch.
|
||||
if current_omni_platform.is_npu():
|
||||
models = ["Tongyi-MAI/Z-Image-Turbo", "Qwen/Qwen-Image"]
|
||||
|
||||
# omni_runner expects (model, stage_configs_path); single-stage diffusion has no YAML.
|
||||
test_params = [(m, None) for m in models]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2})
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_diffusion_model(omni_runner, run_level):
|
||||
resolved = omni_runner.model_name
|
||||
if run_level == "core_model" and not _is_qwen_image_random(resolved):
|
||||
pytest.skip()
|
||||
|
||||
if run_level == "advanced_model" and _is_qwen_image_random(resolved):
|
||||
pytest.skip()
|
||||
|
||||
# high resolution may cause OOM on L4
|
||||
height = 256
|
||||
width = 256
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=2,
|
||||
)
|
||||
|
||||
# OmniRunner.generate() is typed for list[TextPrompt]; diffusion uses Omni.generate(str, ...).
|
||||
outputs = omni_runner.omni.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
sampling,
|
||||
)
|
||||
|
||||
# Extract images from request_output['images']
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
if not hasattr(first_output, "request_output") or not first_output.request_output:
|
||||
raise ValueError("No request_output found in OmniRequestOutput")
|
||||
|
||||
req_out = first_output.request_output
|
||||
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
|
||||
raise ValueError("Invalid request_output structure or missing 'images' key")
|
||||
|
||||
images = req_out.images
|
||||
|
||||
assert len(images) == 2
|
||||
# check image size
|
||||
assert images[0].width == width
|
||||
assert images[0].height == height
|
||||
images[0].save("image_output.png")
|
||||
@@ -1,56 +0,0 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
models = ["Wan-AI/Wan2.2-T2V-A14B-Diffusers"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_video_diffusion_model(model_name: str):
|
||||
with OmniRunner(
|
||||
model_name,
|
||||
boundary_ratio=0.875,
|
||||
flow_shift=5.0,
|
||||
) as runner:
|
||||
# Use minimal settings for testing
|
||||
# num_frames must satisfy: num_frames % vae_scale_factor_temporal == 1
|
||||
# For Wan2.2, vae_scale_factor_temporal=4, so valid values are 5, 9, 13, 17, ...
|
||||
height = 480
|
||||
width = 640
|
||||
num_frames = 5
|
||||
outputs = runner.omni.generate(
|
||||
prompts="A cat sitting on a table",
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
),
|
||||
)
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
if not hasattr(first_output, "request_output") or not first_output.request_output:
|
||||
raise ValueError("No request_output found in OmniRequestOutput")
|
||||
|
||||
req_out = first_output.request_output
|
||||
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
|
||||
raise ValueError("Invalid request_output structure or missing 'images' key")
|
||||
|
||||
frames = req_out.images[0]
|
||||
|
||||
assert frames is not None
|
||||
assert hasattr(frames, "shape")
|
||||
# frames shape: (batch, num_frames, height, width, channels)
|
||||
assert frames.shape[1] == num_frames
|
||||
assert frames.shape[2] == height
|
||||
assert frames.shape[3] == width
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
System test for TeaCache backend.
|
||||
|
||||
This test verifies that TeaCache acceleration works correctly with diffusion models.
|
||||
It uses minimal settings to keep test time short for CI.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Use random weights model for testing
|
||||
models = ["riverclouds/qwen_image_random"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.cache
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_teacache(model_name: str):
|
||||
"""Test TeaCache backend with diffusion model."""
|
||||
# Configure TeaCache with default settings for fast testing
|
||||
cache_config = {
|
||||
"rel_l1_thresh": 0.2, # Default threshold
|
||||
}
|
||||
with OmniRunner(
|
||||
model_name,
|
||||
cache_backend="tea_cache",
|
||||
cache_config=cache_config,
|
||||
) as runner:
|
||||
# Use minimal settings for fast testing
|
||||
height = 256
|
||||
width = 256
|
||||
num_inference_steps = 4 # Minimal steps for fast test
|
||||
|
||||
outputs = runner.omni.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
num_outputs_per_prompt=1, # Single output for speed
|
||||
),
|
||||
)
|
||||
# Extract images from request_output['images']
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
if not hasattr(first_output, "request_output") or not first_output.request_output:
|
||||
raise ValueError("No request_output found in OmniRequestOutput")
|
||||
|
||||
req_out = first_output.request_output
|
||||
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
|
||||
raise ValueError("Invalid request_output structure or missing 'images' key")
|
||||
|
||||
images = req_out.images
|
||||
|
||||
# Verify generation succeeded
|
||||
assert images is not None
|
||||
assert len(images) == 1
|
||||
# Check image size
|
||||
assert images[0].width == width
|
||||
assert images[0].height == height
|
||||
@@ -1,152 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import time
|
||||
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
all_model_configs = [
|
||||
{
|
||||
"model_name": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||||
"out_width": 1280,
|
||||
"out_height": 720,
|
||||
"out_frames": 16,
|
||||
},
|
||||
{
|
||||
"model_name": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||
"out_width": 1280,
|
||||
"out_height": 720,
|
||||
"out_frames": 16,
|
||||
},
|
||||
{
|
||||
"model_name": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"out_width": 1280,
|
||||
"out_height": 720,
|
||||
"out_frames": 16,
|
||||
},
|
||||
{
|
||||
"model_name": "Qwen/Qwen-Image",
|
||||
"out_width": 3072,
|
||||
"out_height": 3072,
|
||||
"out_frames": 1,
|
||||
},
|
||||
# {
|
||||
# "model_name": "stepfun-ai/NextStep-1.1",
|
||||
# "out_width": 3072,
|
||||
# "out_height": 3072,
|
||||
# "out_frames": 1,
|
||||
# },
|
||||
]
|
||||
|
||||
|
||||
def is_nextstep_model(model_name: str) -> bool:
|
||||
"""Check if the model is a NextStep model by reading its config."""
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
try:
|
||||
cfg = get_hf_file_to_dict("config.json", model_name)
|
||||
if cfg and cfg.get("model_type") == "nextstep":
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def model_run(model_configs, tp, out_height, out_width, out_frames, using_tile, vae_patch_parallel_size=1):
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
tensor_parallel_size=tp,
|
||||
vae_patch_parallel_size=vae_patch_parallel_size,
|
||||
)
|
||||
|
||||
omni_kwargs = {
|
||||
"vae_use_tiling": using_tile,
|
||||
"parallel_config": parallel_config,
|
||||
}
|
||||
use_nextstep = is_nextstep_model(model_configs["model_name"])
|
||||
if use_nextstep:
|
||||
# NextStep-1.1 requires explicit pipeline class
|
||||
omni_kwargs["model_class_name"] = "NextStep11Pipeline"
|
||||
|
||||
with OmniRunner(model_configs["model_name"], **omni_kwargs) as runner:
|
||||
m = runner.omni
|
||||
image = Image.new("RGB", (out_width, out_height), (0, 0, 0))
|
||||
start = time.perf_counter()
|
||||
outputs = m.generate(
|
||||
{
|
||||
"prompt": "A cat sitting on a table",
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=model_configs["out_height"],
|
||||
width=model_configs["out_width"],
|
||||
num_frames=model_configs["out_frames"],
|
||||
num_inference_steps=2,
|
||||
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
|
||||
),
|
||||
)
|
||||
end = time.perf_counter()
|
||||
first_output = outputs[0]
|
||||
req_out = first_output.request_output
|
||||
frames = req_out.images[0]
|
||||
if isinstance(frames, torch.Tensor):
|
||||
frames = frames.detach().cpu().numpy()
|
||||
elif isinstance(frames, Image.Image):
|
||||
frames = np.array(frames)
|
||||
# frames shape: (batch, num_frames, height, width, channels)
|
||||
cost = (end - start) * 1000
|
||||
return frames, cost
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_configs", all_model_configs)
|
||||
def test_vae_parallel_model(model_configs: str):
|
||||
parallel_result, parallel_time = model_run(
|
||||
model_configs=model_configs,
|
||||
tp=2,
|
||||
out_width=1280,
|
||||
out_height=704,
|
||||
out_frames=5,
|
||||
using_tile=True,
|
||||
vae_patch_parallel_size=2,
|
||||
)
|
||||
non_parallel_result, non_parallel_time = model_run(
|
||||
model_configs=model_configs,
|
||||
tp=2,
|
||||
out_width=1280,
|
||||
out_height=704,
|
||||
out_frames=5,
|
||||
using_tile=True,
|
||||
vae_patch_parallel_size=1,
|
||||
)
|
||||
result_diff = np.abs(non_parallel_result - parallel_result)
|
||||
|
||||
mean_threshold = 3e-2
|
||||
max_threshold = 3e-2 # they should be totally same
|
||||
print(
|
||||
f"{model_configs['model_name']} TP = 2 (tile + parallel vs tile): "
|
||||
f"mean_abs_diff={result_diff.mean():.6e}, max_abs_diff={result_diff.max():.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
|
||||
f"parallel generate take time: {parallel_time:.2f} ms, non-parallel take time: {non_parallel_time:.2f} ms"
|
||||
)
|
||||
# assert non_parallel_time > parallel_time
|
||||
assert result_diff.mean() < mean_threshold
|
||||
assert result_diff.max() < max_threshold
|
||||
@@ -41,7 +41,8 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor:
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_zero_shot_001(voxcpm2_engine):
|
||||
"""Test zero-shot TTS produces valid audio output."""
|
||||
@@ -54,7 +55,8 @@ def test_voxcpm2_zero_shot_001(voxcpm2_engine):
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_voice_clone_002(voxcpm2_engine):
|
||||
"""Test voice cloning with a reference audio file.
|
||||
@@ -95,7 +97,8 @@ def test_voxcpm2_voice_clone_002(voxcpm2_engine):
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_prefill_decode_mixed_batch_003(voxcpm2_engine):
|
||||
"""Regression: prefill+decode mixed batch must not crash (PR #2903)."""
|
||||
|
||||
+12
-69
@@ -3,44 +3,28 @@
|
||||
|
||||
"""E2E test for VoxCPM offline inference."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import tests.helpers.runtime as omni_runtime
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.model_executor.models.voxcpm.voxcpm_runtime_utils import (
|
||||
prepare_voxcpm_hf_config_dir,
|
||||
resolve_voxcpm_model_dir,
|
||||
)
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
VOXCPM_MODEL = os.environ.get("VOXCPM_MODEL", "OpenBMB/VoxCPM1.5")
|
||||
STAGE_CONFIG = str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
|
||||
)
|
||||
VOXCPM_MODEL = "OpenBMB/VoxCPM1.5"
|
||||
STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml")
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
# (model, stage_config_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
|
||||
_OMNI_RUNNER_PARAM = (VOXCPM_MODEL, STAGE_CONFIG)
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Limit the NPU cleanup workaround to this VoxCPM test module only."""
|
||||
original_cleanup = omni_runtime.cleanup_dist_env_and_memory
|
||||
|
||||
def _safe_cleanup() -> None:
|
||||
try:
|
||||
original_cleanup()
|
||||
except RuntimeError as exc:
|
||||
if "Allocator for npu is not a DeviceAllocator" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
monkeypatch.setattr(omni_runtime, "cleanup_dist_env_and_memory", _safe_cleanup)
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _build_prompt(text: str) -> dict[str, Any]:
|
||||
@@ -94,50 +78,9 @@ def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
|
||||
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def voxcpm_model_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> str:
|
||||
model_dir = resolve_voxcpm_model_dir(VOXCPM_MODEL)
|
||||
|
||||
hf_config_env = os.environ.get("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH")
|
||||
if hf_config_env:
|
||||
hf_config_dir = Path(hf_config_env).expanduser()
|
||||
else:
|
||||
hf_config_dir = tmp_path / "voxcpm_hf_config"
|
||||
|
||||
if not (hf_config_dir / "config.json").exists():
|
||||
prepare_voxcpm_hf_config_dir(model_dir, hf_config_dir)
|
||||
|
||||
monkeypatch.setenv("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH", str(hf_config_dir))
|
||||
return str(model_dir)
|
||||
|
||||
|
||||
def test_prepare_voxcpm_hf_config_dir(tmp_path: Path):
|
||||
model_dir = tmp_path / "model"
|
||||
model_dir.mkdir()
|
||||
(model_dir / "config.json").write_text(json.dumps({"hidden_size": 1024}), encoding="utf-8")
|
||||
(model_dir / "generation_config.json").write_text(json.dumps({"do_sample": False}), encoding="utf-8")
|
||||
|
||||
hf_config_dir = prepare_voxcpm_hf_config_dir(model_dir, tmp_path / "voxcpm_hf_config")
|
||||
|
||||
prepared_config = json.loads((hf_config_dir / "config.json").read_text(encoding="utf-8"))
|
||||
assert prepared_config["model_type"] == "voxcpm"
|
||||
assert prepared_config["architectures"] == ["VoxCPMForConditionalGeneration"]
|
||||
assert (hf_config_dir / "generation_config.json").exists()
|
||||
|
||||
|
||||
def test_resolve_voxcpm_model_dir_local_path(tmp_path: Path):
|
||||
model_dir = tmp_path / "OpenBMB" / "VoxCPM1.5"
|
||||
model_dir.mkdir(parents=True)
|
||||
|
||||
assert resolve_voxcpm_model_dir(str(model_dir)) == model_dir
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm_zero_shot_001(voxcpm_model_path: str):
|
||||
with OmniRunner(voxcpm_model_path, stage_configs_path=STAGE_CONFIG) as runner:
|
||||
outputs = list(runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
|
||||
def test_voxcpm_zero_shot_001(omni_runner: OmniRunner) -> None:
|
||||
outputs = list(omni_runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
|
||||
@@ -31,7 +31,7 @@ from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
MODEL = "mistralai/Voxtral-4B-TTS-2603"
|
||||
@@ -42,6 +42,14 @@ MIN_AUDIO_SAMPLES = 1000
|
||||
VOICE = "casual_female"
|
||||
TEST_TEXT = "Hello, how are you?"
|
||||
|
||||
# (model, stage_config_path, extra_omni_kwargs) for indirect parametrize on
|
||||
# ``omni_runner_function`` (function-scoped: must exit before ``AsyncOmni`` test).
|
||||
_OMNI_RUNNER_PARAM = (
|
||||
MODEL,
|
||||
STAGE_CONFIG,
|
||||
{"enforce_eager": True},
|
||||
)
|
||||
|
||||
|
||||
def _compose_request(model_name: str, text: str, voice: str) -> dict:
|
||||
"""Build the TTS input dict using mistral tokenizer."""
|
||||
@@ -58,83 +66,60 @@ def _compose_request(model_name: str, text: str, voice: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _resolve_stage_config(run_level: str) -> str:
|
||||
"""Resolve stage config: strip load_format for advanced_model (real weights)."""
|
||||
if run_level == "advanced_model":
|
||||
return modify_stage_config(
|
||||
STAGE_CONFIG,
|
||||
deletes={
|
||||
"stages": {
|
||||
0: ["load_format"],
|
||||
1: ["load_format"],
|
||||
}
|
||||
},
|
||||
)
|
||||
return STAGE_CONFIG
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.tts
|
||||
@pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True)
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None:
|
||||
"""Offline sync path; function-scoped runner so AsyncOmni test does not overlap a live Omni."""
|
||||
omni = omni_runner_function.omni
|
||||
inputs = _compose_request(MODEL, TEST_TEXT, VOICE)
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=2500)
|
||||
sampling_params_list = [sampling_params, sampling_params]
|
||||
|
||||
outputs = list(omni.generate([inputs], sampling_params_list))
|
||||
|
||||
assert len(outputs) > 0, "No outputs generated"
|
||||
|
||||
# Find audio output from the final stage
|
||||
audio_data = None
|
||||
for o in outputs:
|
||||
mm = getattr(o, "multimodal_output", None)
|
||||
if mm and "audio" in mm:
|
||||
audio_data = mm["audio"]
|
||||
break
|
||||
|
||||
assert audio_data is not None, "No audio output found in any stage output"
|
||||
|
||||
# Concatenate audio chunks if returned as a list of tensors
|
||||
if isinstance(audio_data, list):
|
||||
audio_tensor = torch.cat(audio_data)
|
||||
elif isinstance(audio_data, torch.Tensor):
|
||||
audio_tensor = audio_data
|
||||
else:
|
||||
audio_tensor = torch.tensor(audio_data)
|
||||
|
||||
audio_array = audio_tensor.float().cpu().numpy()
|
||||
|
||||
assert len(audio_array) > MIN_AUDIO_SAMPLES, (
|
||||
f"Audio too short: {len(audio_array)} samples, expected > {MIN_AUDIO_SAMPLES}"
|
||||
)
|
||||
|
||||
# Verify audio isn't all zeros / silence
|
||||
assert np.max(np.abs(audio_array)) > 0.01, "Audio appears to be silence"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_voxtral_tts_offline_basic(run_level):
|
||||
"""Test basic Voxtral TTS offline inference with a voice preset."""
|
||||
stage_config = _resolve_stage_config(run_level)
|
||||
|
||||
with OmniRunner(
|
||||
MODEL,
|
||||
stage_configs_path=stage_config,
|
||||
enforce_eager=True,
|
||||
) as runner:
|
||||
omni = runner.omni
|
||||
inputs = _compose_request(MODEL, TEST_TEXT, VOICE)
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=2500)
|
||||
sampling_params_list = [sampling_params, sampling_params]
|
||||
|
||||
outputs = list(omni.generate([inputs], sampling_params_list))
|
||||
|
||||
assert len(outputs) > 0, "No outputs generated"
|
||||
|
||||
# Find audio output from the final stage
|
||||
audio_data = None
|
||||
for o in outputs:
|
||||
mm = getattr(o, "multimodal_output", None)
|
||||
if mm and "audio" in mm:
|
||||
audio_data = mm["audio"]
|
||||
break
|
||||
|
||||
assert audio_data is not None, "No audio output found in any stage output"
|
||||
|
||||
# Concatenate audio chunks if returned as a list of tensors
|
||||
if isinstance(audio_data, list):
|
||||
audio_tensor = torch.cat(audio_data)
|
||||
elif isinstance(audio_data, torch.Tensor):
|
||||
audio_tensor = audio_data
|
||||
else:
|
||||
audio_tensor = torch.tensor(audio_data)
|
||||
|
||||
audio_array = audio_tensor.float().cpu().numpy()
|
||||
|
||||
assert len(audio_array) > MIN_AUDIO_SAMPLES, (
|
||||
f"Audio too short: {len(audio_array)} samples, expected > {MIN_AUDIO_SAMPLES}"
|
||||
)
|
||||
|
||||
# Verify audio isn't all zeros / silence
|
||||
assert np.max(np.abs(audio_array)) > 0.01, "Audio appears to be silence"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_voxtral_tts_offline_streaming(run_level):
|
||||
def test_voxtral_tts_offline_streaming():
|
||||
"""Test AsyncOmni streaming inference for Voxtral TTS."""
|
||||
|
||||
async def _run():
|
||||
stage_config = _resolve_stage_config(run_level)
|
||||
|
||||
async_omni = AsyncOmni(
|
||||
model=MODEL,
|
||||
stage_configs_path=stage_config,
|
||||
stage_configs_path=STAGE_CONFIG,
|
||||
stage_init_timeout=300,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunnerHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||
PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
|
||||
NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 1})
|
||||
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
|
||||
def test_text_to_video_001(omni_runner_handler: OmniRunnerHandler):
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_frames=8,
|
||||
fps=8,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=4.0,
|
||||
seed=42,
|
||||
)
|
||||
request_config = {
|
||||
"model": MODEL,
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"sampling_params": sampling,
|
||||
}
|
||||
omni_runner_handler.send_diffusion_request(request_config)
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunnerHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
MODEL = "Tongyi-MAI/Z-Image-Turbo"
|
||||
PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard."
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2})
|
||||
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
|
||||
def test_zimage(omni_runner_handler: OmniRunnerHandler):
|
||||
# high resolution may cause OOM on L4
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
seed=42,
|
||||
num_outputs_per_prompt=2,
|
||||
)
|
||||
request_config = {
|
||||
"model": MODEL,
|
||||
"prompt": PROMPT,
|
||||
"sampling_params": sampling,
|
||||
}
|
||||
omni_runner_handler.send_diffusion_request(request_config)
|
||||
@@ -1,275 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Z-Image end-to-end tests for diffusion parallelism.
|
||||
|
||||
This file currently covers:
|
||||
- DiT tensor parallelism (TP=2) vs TP=1.
|
||||
- VAE patch parallelism (vae_patch_parallel_size=2) vs baseline on TP=2.
|
||||
|
||||
Note: CUDA-only (>=2 GPUs). We use `enforce_eager=False` (default) to enable
|
||||
`torch.compile`.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.env import DeviceMemoryMonitor
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
PROMPT = "a photo of a cat sitting on a laptop keyboard"
|
||||
|
||||
|
||||
def _get_zimage_model() -> str:
|
||||
# Allow overriding the model for local/offline environments.
|
||||
# Can be either a HuggingFace repo id or a local path.
|
||||
return os.environ.get("VLLM_TEST_ZIMAGE_MODEL", "Tongyi-MAI/Z-Image-Turbo")
|
||||
|
||||
|
||||
def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor:
|
||||
"""Convert PIL image to float32 RGB tensor in [0, 1] with shape [H, W, 3]."""
|
||||
arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0
|
||||
return torch.from_numpy(arr)
|
||||
|
||||
|
||||
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
|
||||
"""Return (mean_abs_diff, p99_abs_diff) over RGB pixels in [0, 1]."""
|
||||
ta = _pil_to_float_rgb_tensor(a)
|
||||
tb = _pil_to_float_rgb_tensor(b)
|
||||
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
|
||||
abs_diff = torch.abs(ta - tb)
|
||||
p99_abs_diff = torch.quantile(abs_diff.flatten(), 0.99).item()
|
||||
return abs_diff.mean().item(), p99_abs_diff
|
||||
|
||||
|
||||
def _extract_single_image(outputs) -> Image.Image:
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
if not hasattr(first_output, "request_output") or not first_output.request_output:
|
||||
raise ValueError("No request_output found in OmniRequestOutput")
|
||||
|
||||
req_out = first_output.request_output
|
||||
if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
|
||||
raise ValueError("Invalid request_output structure or missing 'images' key")
|
||||
|
||||
images = req_out.images
|
||||
if images is None or len(images) != 1:
|
||||
raise ValueError(f"Expected 1 image, got {0 if images is None else len(images)}")
|
||||
return images[0]
|
||||
|
||||
|
||||
def _run_zimage_generate(
|
||||
*,
|
||||
tp_size: int,
|
||||
height: int,
|
||||
width: int,
|
||||
num_inference_steps: int,
|
||||
seed: int,
|
||||
enforce_eager: bool,
|
||||
vae_use_tiling: bool = False,
|
||||
vae_patch_parallel_size: int = 1,
|
||||
num_requests: int = 4,
|
||||
) -> tuple[Image.Image, float, float]:
|
||||
if num_requests < 2:
|
||||
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
|
||||
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = current_omni_platform.current_device()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
try:
|
||||
# Each run needs a distinct DiffusionParallelConfig; use OmniRunner per call (not the
|
||||
# parametrized omni_runner fixture, which is fixed per module).
|
||||
with OmniRunner(
|
||||
_get_zimage_model(),
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
tensor_parallel_size=tp_size,
|
||||
vae_patch_parallel_size=vae_patch_parallel_size,
|
||||
),
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=vae_use_tiling,
|
||||
) as runner:
|
||||
# NOTE: Omni closes itself when a generate() call is exhausted.
|
||||
# To avoid measuring teardown time (process shutdown, memory cleanup),
|
||||
# we measure the latency to produce *subsequent* outputs within a single
|
||||
# generator run.
|
||||
#
|
||||
# This also serves as a warmup: the first output may include extra
|
||||
# compilation/caching overhead, while later outputs are closer to
|
||||
# steady-state inference.
|
||||
gen = runner.omni.generate(
|
||||
[PROMPT] * num_requests,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
seed=seed,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
py_generator=True,
|
||||
)
|
||||
|
||||
warmup_output = next(gen)
|
||||
|
||||
t_prev = time.perf_counter()
|
||||
per_request_times_s: list[float] = []
|
||||
last_output = warmup_output
|
||||
for _ in range(num_requests - 1):
|
||||
last_output = next(gen)
|
||||
t_now = time.perf_counter()
|
||||
per_request_times_s.append(t_now - t_prev)
|
||||
t_prev = t_now
|
||||
|
||||
# Ensure the generator is fully consumed so it can clean up.
|
||||
for _ in gen:
|
||||
pass
|
||||
|
||||
median_time_s = float(np.median(per_request_times_s))
|
||||
|
||||
peak_memory_mb = monitor.peak_used_mb
|
||||
|
||||
return _extract_single_image([last_output]), median_time_s, peak_memory_mb
|
||||
finally:
|
||||
monitor.stop()
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu():
|
||||
pytest.skip("Z-Image TP e2e test is only supported on CUDA and ROCm for now.")
|
||||
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
|
||||
pytest.skip("Z-Image TP=2 requires >= 2 devices.")
|
||||
|
||||
enforce_eager = False
|
||||
|
||||
height = 512
|
||||
width = 512
|
||||
num_inference_steps = 2
|
||||
seed = 42
|
||||
|
||||
tp1_img, tp1_time_s, tp1_peak_mem = _run_zimage_generate(
|
||||
tp_size=1,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
)
|
||||
tp2_img, tp2_time_s, tp2_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
)
|
||||
|
||||
tp1_path = tmp_path / "zimage_tp1.png"
|
||||
tp2_path = tmp_path / "zimage_tp2.png"
|
||||
tp1_img.save(tp1_path)
|
||||
tp2_img.save(tp2_path)
|
||||
|
||||
assert tp1_img.width == width and tp1_img.height == height
|
||||
assert tp2_img.width == width and tp2_img.height == height
|
||||
|
||||
mean_abs_diff, p99_abs_diff = _diff_metrics(tp1_img, tp2_img)
|
||||
mean_threshold = 3e-2
|
||||
p99_threshold = 2.5e-1
|
||||
print(
|
||||
"Z-Image TP image diff stats (TP=1 vs TP=2): "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
|
||||
f"tp1_img={tp1_path}, tp2_img={tp2_path}"
|
||||
)
|
||||
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
|
||||
)
|
||||
|
||||
print(f"Z-Image TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}")
|
||||
# ROCm is not optimized TP2 can be slower than TP1
|
||||
if not current_omni_platform.is_rocm():
|
||||
assert tp2_time_s < tp1_time_s, f"Expected TP=2 to be faster than TP=1 (tp1={tp1_time_s}, tp2={tp2_time_s})"
|
||||
|
||||
print(f"Z-Image TP peak memory (MB): tp1_peak_mem={tp1_peak_mem:.2f}, tp2_peak_mem={tp2_peak_mem:.2f}")
|
||||
assert tp2_peak_mem < tp1_peak_mem, (
|
||||
f"Expected TP=2 to use less peak memory than TP=1 (tp1={tp1_peak_mem}, tp2={tp2_peak_mem})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu():
|
||||
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA and ROCm for now.")
|
||||
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
|
||||
pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 devices.")
|
||||
|
||||
enforce_eager = False
|
||||
|
||||
# Use a larger image to ensure there are multiple VAE tiles.
|
||||
height = 1152
|
||||
width = 1152
|
||||
num_inference_steps = 2
|
||||
seed = 42
|
||||
|
||||
baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=True,
|
||||
vae_patch_parallel_size=1,
|
||||
num_requests=2,
|
||||
)
|
||||
pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=True,
|
||||
vae_patch_parallel_size=2,
|
||||
num_requests=2,
|
||||
)
|
||||
|
||||
baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
|
||||
pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
|
||||
baseline_img.save(baseline_path)
|
||||
pp2_img.save(pp2_path)
|
||||
|
||||
mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img)
|
||||
mean_threshold = 5e-3
|
||||
p99_threshold = 1e-1
|
||||
print(
|
||||
"Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
|
||||
f"pp1_img={baseline_path}, pp2_img={pp2_path}"
|
||||
)
|
||||
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
|
||||
)
|
||||
+1
-1
@@ -35,7 +35,7 @@ def _audiox_server_cases(model: str):
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _audiox_server_cases(AUDIOX_TEST_MODEL), indirect=True)
|
||||
def test_audiox_t2a_online(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
File renamed without changes.
+2
-6
@@ -19,6 +19,8 @@ from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
|
||||
|
||||
# Official CosyVoice zero-shot prompt audio and its transcript. Vendored under
|
||||
@@ -64,8 +66,6 @@ tts_async_chunk_server_params = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_zh_001(omni_server, openai_client) -> None:
|
||||
@@ -88,8 +88,6 @@ def test_voice_clone_zh_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_async_chunk_server_params, indirect=True)
|
||||
def test_voice_clone_zh_002(omni_server, openai_client) -> None:
|
||||
@@ -112,8 +110,6 @@ def test_voice_clone_zh_002(omni_server, openai_client) -> None:
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_en_001(omni_server, openai_client) -> None:
|
||||
@@ -7,13 +7,13 @@ Example online tests for Dynin-Omni model.
|
||||
import base64
|
||||
import os
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
@@ -21,7 +21,7 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "snu-aidas/Dynin-Omni"
|
||||
STAGE_CONFIG = str(Path(__file__).parent.parent / "stage_configs" / "dynin_omni_ci.yaml")
|
||||
STAGE_CONFIG = get_deploy_config_path("dynin_omni_ci.yaml")
|
||||
|
||||
T2I_PROMPT = "A high quality detailed living room interior photo."
|
||||
T2S_PROMPT = "Please read this sentence naturally: Hello from online serving."
|
||||
|
||||
@@ -12,7 +12,12 @@ Coverage:
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
@@ -96,9 +101,10 @@ def _get_diffusion_feature_cases(model: str):
|
||||
indirect=True,
|
||||
)
|
||||
def test_flux2_klein(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": [{"role": "user", "content": POSITIVE_PROMPT}],
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Tests for Flux2 Klein; currently Dev is implemented separately,
|
||||
but ideally these models will fold together in the future.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
|
||||
POSITIVE_PROMPT = "A cat sitting on a windowsill"
|
||||
NEGATIVE_PROMPT = "blurry, low quality"
|
||||
|
||||
|
||||
# For now, we only test a good configuration for this model to keep the CI
|
||||
# light, but in the near future may explore using random models to check
|
||||
# cross-feature compatibility more generally.
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
return [
|
||||
# FP8 / Hybrid sequence parallelism
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
),
|
||||
id="default",
|
||||
marks=FOUR_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(
|
||||
model="black-forest-labs/FLUX.2-klein-4B",
|
||||
),
|
||||
indirect=True,
|
||||
)
|
||||
def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
File renamed without changes.
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests of common diffusion feature combinations in online serving mode
|
||||
for HunyuanVideo-1.5-T2V (480p).
|
||||
|
||||
Coverage (H100, since model cannot fit L4):
|
||||
- CacheDiT + Layerwise CPU offloading (1 GPU)
|
||||
- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs)
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
)
|
||||
|
||||
PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion."
|
||||
NEGATIVE_PROMPT = "low quality, blurry, distorted"
|
||||
|
||||
MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v"
|
||||
|
||||
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return L4 diffusion feature cases for HunyuanVideo-1.5.
|
||||
|
||||
Designed for 2x H100 environment per issue #1832.
|
||||
"""
|
||||
return [
|
||||
# (1 GPU) CacheDiT + Layerwise CPU offloading
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(MODEL),
|
||||
indirect=True,
|
||||
)
|
||||
def test_text_to_video_001(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
):
|
||||
"""L4 diffusion feature coverage for HunyuanVideo-1.5-T2V on H100."""
|
||||
form_data = {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"height": 480,
|
||||
"width": 640,
|
||||
"num_frames": 5,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 6.0,
|
||||
"seed": 42,
|
||||
}
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"form_data": form_data,
|
||||
}
|
||||
|
||||
openai_client.send_video_diffusion_request(request_config)
|
||||
@@ -1,278 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online serving test for Qwen-Image-Edit-2509 multi-image input.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import requests
|
||||
from PIL import Image
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
# Increase timeout for downloading assets from S3 (default 5s is too short for CI)
|
||||
os.environ.setdefault("VLLM_IMAGE_FETCH_TIMEOUT", "60")
|
||||
|
||||
models = ["Qwen/Qwen-Image-Edit-2509"]
|
||||
test_params = models
|
||||
t2i_models = ["Tongyi-MAI/Z-Image-Turbo"]
|
||||
|
||||
|
||||
class OmniServer:
|
||||
"""Omniserver for vLLM-Omni tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
serve_args: list[str],
|
||||
*,
|
||||
env_dict: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.serve_args = serve_args
|
||||
self.env_dict = env_dict
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self.host = "127.0.0.1"
|
||||
self.port = get_open_port()
|
||||
|
||||
def _start_server(self) -> None:
|
||||
"""Start the vLLM-Omni server subprocess."""
|
||||
env = os.environ.copy()
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if self.env_dict is not None:
|
||||
env.update(self.env_dict)
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm_omni.entrypoints.cli.main",
|
||||
"serve",
|
||||
self.model,
|
||||
"--omni",
|
||||
"--host",
|
||||
self.host,
|
||||
"--port",
|
||||
str(self.port),
|
||||
] + self.serve_args
|
||||
|
||||
print(f"Launching OmniServer with: {' '.join(cmd)}")
|
||||
self.proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=env,
|
||||
cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # Set working directory to vllm-omni root
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
# Wait for server to be ready
|
||||
max_wait = 600 # 10 minutes
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < max_wait:
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(1)
|
||||
result = sock.connect_ex((self.host, self.port))
|
||||
if result == 0:
|
||||
print(f"Server ready on {self.host}:{self.port}")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
|
||||
raise RuntimeError(f"Server failed to start within {max_wait} seconds")
|
||||
|
||||
def __enter__(self):
|
||||
self._start_server()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.proc:
|
||||
try:
|
||||
os.killpg(self.proc.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
try:
|
||||
self.proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(self.proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
self.proc.wait()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def omni_server(request):
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights."""
|
||||
model = request.param
|
||||
with OmniServer(model, ["--num-gpus", "1"]) as server:
|
||||
yield server
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(omni_server):
|
||||
"""OpenAI client for the running vLLM-Omni server."""
|
||||
return openai.OpenAI(
|
||||
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
|
||||
api_key="EMPTY",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def base64_encoded_images() -> list[str]:
|
||||
"""Base64 encoded PNG images for testing."""
|
||||
images = [
|
||||
ImageAsset("cherry_blossom").pil_image.convert("RGB"),
|
||||
ImageAsset("stop_sign").pil_image.convert("RGB"),
|
||||
]
|
||||
encoded: list[str] = []
|
||||
for img in images:
|
||||
with BytesIO() as buffer:
|
||||
img.save(buffer, format="PNG")
|
||||
encoded.append(base64.b64encode(buffer.getvalue()).decode("utf-8"))
|
||||
return encoded
|
||||
|
||||
|
||||
def dummy_messages_from_image_data(
|
||||
image_data_urls: list[str],
|
||||
content_text: str = "Combine these two images into one scene.",
|
||||
):
|
||||
"""Create messages with image data URLs for OpenAI API."""
|
||||
content = [{"type": "text", "text": content_text}]
|
||||
for image_url in image_data_urls:
|
||||
content.append({"type": "image_url", "image_url": {"url": image_url}})
|
||||
return [{"role": "user", "content": content}]
|
||||
|
||||
|
||||
def _extract_image_data_url(message_content) -> str:
|
||||
assert isinstance(message_content, list) and len(message_content) >= 1
|
||||
content_part = message_content[0]
|
||||
if isinstance(content_part, dict):
|
||||
image_url = content_part.get("image_url", {}).get("url", "")
|
||||
else:
|
||||
image_url_obj = getattr(content_part, "image_url", None)
|
||||
if isinstance(image_url_obj, dict):
|
||||
image_url = image_url_obj.get("url", "")
|
||||
else:
|
||||
image_url = getattr(image_url_obj, "url", "")
|
||||
assert isinstance(image_url, str) and image_url
|
||||
return image_url
|
||||
|
||||
|
||||
def _decode_data_url_to_image_bytes(data_url: str) -> bytes:
|
||||
assert data_url.startswith("data:image")
|
||||
_, b64_data = data_url.split(",", 1)
|
||||
return base64.b64decode(b64_data)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_i2i_multi_image_input_qwen_image_edit_2509(
|
||||
omni_server,
|
||||
base64_encoded_images: list[str],
|
||||
) -> None:
|
||||
"""Test multi-image input editing via OpenAI API with concurrent requests."""
|
||||
image_data_urls = [f"data:image/png;base64,{img}" for img in base64_encoded_images]
|
||||
messages = dummy_messages_from_image_data(image_data_urls)
|
||||
|
||||
barrier = threading.Barrier(2)
|
||||
results: list[tuple[int, int]] = []
|
||||
|
||||
def _call_chat(width: int, height: int) -> None:
|
||||
client = openai.OpenAI(
|
||||
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
|
||||
api_key="EMPTY",
|
||||
)
|
||||
barrier.wait()
|
||||
chat_completion = client.chat.completions.create(
|
||||
model=omni_server.model,
|
||||
messages=messages,
|
||||
extra_body={
|
||||
"height": height,
|
||||
"width": width,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": 42,
|
||||
},
|
||||
)
|
||||
|
||||
assert len(chat_completion.choices) == 1
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "stop"
|
||||
assert choice.message.role == "assistant"
|
||||
|
||||
image_data_url = _extract_image_data_url(choice.message.content)
|
||||
image_bytes = _decode_data_url_to_image_bytes(image_data_url)
|
||||
img = Image.open(BytesIO(image_bytes))
|
||||
img.load()
|
||||
results.append(img.size)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=_call_chat, args=(1248, 832)),
|
||||
threading.Thread(target=_call_chat, args=(1024, 768)),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# TODO @ZJY
|
||||
# assert (1248, 832) in results
|
||||
# assert (1024, 768) in results
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", t2i_models, indirect=True)
|
||||
def test_t2i_concurrent_requests_different_sizes(omni_server) -> None:
|
||||
"""Test /v1/images/generations concurrent requests with different sizes."""
|
||||
base_url = f"http://{omni_server.host}:{omni_server.port}"
|
||||
url = f"{base_url}/v1/images/generations"
|
||||
|
||||
barrier = threading.Barrier(2)
|
||||
results: list[tuple[int, int]] = []
|
||||
|
||||
def _call_generate(size: str) -> None:
|
||||
payload: dict[str, Any] = {
|
||||
"prompt": "cute cat playing with a ball",
|
||||
"n": 1,
|
||||
"size": size,
|
||||
"response_format": "b64_json",
|
||||
"num_inference_steps": 2,
|
||||
}
|
||||
barrier.wait()
|
||||
response = requests.post(url, json=payload, timeout=120)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
image_b64 = data["data"][0]["b64_json"]
|
||||
image_bytes = base64.b64decode(image_b64)
|
||||
img = Image.open(BytesIO(image_bytes))
|
||||
img.load()
|
||||
results.append(img.size)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=_call_generate, args=("512x512",)),
|
||||
threading.Thread(target=_call_generate, args=("768x512",)),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert (512, 512) in results
|
||||
assert (768, 512) in results
|
||||
@@ -1,208 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online serving test for /v1/images/generations with per-request LoRA.
|
||||
|
||||
This validates:
|
||||
- The API server accepts a per-request `lora` object in the Images API payload.
|
||||
- LoRA can be switched per request (adapter A -> adapter B -> no LoRA).
|
||||
- Output correctness is asserted using a small image slice with tolerance.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
from PIL import Image
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL = "Tongyi-MAI/Z-Image-Turbo"
|
||||
DIFFUSION_INIT_TIMEOUT_S = 900
|
||||
|
||||
|
||||
PROMPT = "a photo of a cat sitting on a laptop keyboard"
|
||||
SIZE = "256x256"
|
||||
SEED = 42
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server():
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--num-gpus",
|
||||
"1",
|
||||
"--stage-init-timeout",
|
||||
str(DIFFUSION_INIT_TIMEOUT_S),
|
||||
"--init-timeout",
|
||||
str(DIFFUSION_INIT_TIMEOUT_S),
|
||||
],
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
def _write_zimage_lora(adapter_dir: Path, *, q_scale: float = 0.0, k_scale: float = 0.0, v_scale: float = 0.0):
|
||||
adapter_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Z-Image transformer uses dim=3840 by default.
|
||||
dim = 3840
|
||||
module_name = "transformer.layers.0.attention.to_qkv"
|
||||
rank = 1
|
||||
|
||||
lora_a = torch.zeros((rank, dim), dtype=torch.float32)
|
||||
lora_a[0, 0] = 1.0
|
||||
|
||||
# QKVParallelLinear packs (Q, K, V) => out dim is 3 * dim (tp=1).
|
||||
lora_b = torch.zeros((3 * dim, rank), dtype=torch.float32)
|
||||
if q_scale:
|
||||
lora_b[:dim, 0] = q_scale
|
||||
if k_scale:
|
||||
lora_b[dim : 2 * dim, 0] = k_scale
|
||||
if v_scale:
|
||||
lora_b[2 * dim :, 0] = v_scale
|
||||
|
||||
save_file(
|
||||
{
|
||||
f"base_model.model.{module_name}.lora_A.weight": lora_a,
|
||||
f"base_model.model.{module_name}.lora_B.weight": lora_b,
|
||||
},
|
||||
str(adapter_dir / "adapter_model.safetensors"),
|
||||
)
|
||||
(adapter_dir / "adapter_config.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"r": rank,
|
||||
"lora_alpha": rank,
|
||||
"target_modules": [module_name],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _post_images(server: OmniServer, payload: dict) -> Image.Image:
|
||||
url = f"http://{server.host}:{server.port}/v1/images/generations"
|
||||
resp = requests.post(url, json=payload, headers={"Authorization": "Bearer EMPTY"}, timeout=900)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
b64 = data["data"][0]["b64_json"]
|
||||
img_bytes = base64.b64decode(b64)
|
||||
img = Image.open(BytesIO(img_bytes))
|
||||
img.load()
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def _image_blue_tail_slice(img: Image.Image) -> np.ndarray:
|
||||
arr = np.asarray(img, dtype=np.uint8)
|
||||
assert arr.ndim == 3 and arr.shape[-1] == 3
|
||||
tail = arr[-3:, -3:, -1].astype(np.float32)
|
||||
assert tail.shape == (3, 3)
|
||||
return tail
|
||||
|
||||
|
||||
def _slice_diff_stats(actual: np.ndarray, expected: np.ndarray) -> tuple[float, float]:
|
||||
diff = np.abs(actual - expected)
|
||||
return float(diff.max()), float(diff.mean())
|
||||
|
||||
|
||||
def _assert_slice_close(
|
||||
actual: np.ndarray,
|
||||
expected: np.ndarray,
|
||||
*,
|
||||
label: str,
|
||||
base_max: float,
|
||||
base_mean: float,
|
||||
) -> None:
|
||||
assert actual.shape == (3, 3)
|
||||
assert expected.shape == (3, 3)
|
||||
max_diff, mean_diff = _slice_diff_stats(actual, expected)
|
||||
# NOTE: Different attention backends / torch.compile can introduce small
|
||||
# floating-point drift that shows up as a few LSBs in uint8 pixels. Keep
|
||||
# the reset check tolerant but bounded to avoid flaky CI.
|
||||
max_thresh = max(10.0, base_max + 4.0)
|
||||
mean_thresh = max(6.0, base_mean + 4.0)
|
||||
assert max_diff <= max_thresh and mean_diff <= mean_thresh, (
|
||||
f"{label} slice mismatch (max={max_diff:.1f} > {max_thresh:.1f} or "
|
||||
f"mean={mean_diff:.1f} > {mean_thresh:.1f}): {actual.tolist()}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_slice_diff(actual: np.ndarray, baseline: np.ndarray, *, label: str) -> None:
|
||||
assert actual.shape == (3, 3)
|
||||
assert baseline.shape == (3, 3)
|
||||
diff = np.abs(actual - baseline).mean()
|
||||
assert diff > 0.1, f"{label} slice diff too small: {diff} ({actual.tolist()} vs {baseline.tolist()})"
|
||||
|
||||
|
||||
def _basic_payload() -> dict:
|
||||
return {
|
||||
"prompt": PROMPT,
|
||||
"n": 1,
|
||||
"size": SIZE,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": SEED,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
|
||||
def test_images_generations_per_request_lora_switching(omni_server: OmniServer, tmp_path: Path) -> None:
|
||||
# Base generation.
|
||||
base_img = _post_images(omni_server, _basic_payload())
|
||||
base_slice = _image_blue_tail_slice(base_img)
|
||||
base_ref_img = _post_images(omni_server, _basic_payload())
|
||||
base_ref_slice = _image_blue_tail_slice(base_ref_img)
|
||||
base_ref_max, base_ref_mean = _slice_diff_stats(base_ref_slice, base_slice)
|
||||
|
||||
# Adapter A: apply delta to V slice only.
|
||||
lora_a_dir = tmp_path / "zimage_lora_a"
|
||||
_write_zimage_lora(lora_a_dir, v_scale=8.0)
|
||||
payload_a = _basic_payload()
|
||||
payload_a["lora"] = {"name": "a", "path": str(lora_a_dir), "scale": 64.0}
|
||||
img_a = _post_images(omni_server, payload_a)
|
||||
a_slice = _image_blue_tail_slice(img_a)
|
||||
_assert_slice_diff(a_slice, base_slice, label="lora_a_vs_base")
|
||||
a_vs_base = float(np.abs(a_slice - base_slice).mean())
|
||||
|
||||
# Adapter B: apply delta to K slice only (should differ from adapter A).
|
||||
lora_b_dir = tmp_path / "zimage_lora_b"
|
||||
_write_zimage_lora(lora_b_dir, k_scale=4.0)
|
||||
payload_b = _basic_payload()
|
||||
payload_b["lora"] = {"name": "b", "path": str(lora_b_dir), "scale": 64.0}
|
||||
img_b = _post_images(omni_server, payload_b)
|
||||
b_slice = _image_blue_tail_slice(img_b)
|
||||
_assert_slice_diff(b_slice, base_slice, label="lora_b_vs_base")
|
||||
_assert_slice_diff(b_slice, a_slice, label="lora_b_vs_lora_a")
|
||||
b_vs_base = float(np.abs(b_slice - base_slice).mean())
|
||||
b_vs_a = float(np.abs(b_slice - a_slice).mean())
|
||||
|
||||
# Ensure switching back to no-LoRA restores the base output.
|
||||
base_img_2 = _post_images(omni_server, _basic_payload())
|
||||
base_slice_2 = _image_blue_tail_slice(base_img_2)
|
||||
_, base_reset_mean = _slice_diff_stats(base_slice_2, base_slice)
|
||||
_assert_slice_close(
|
||||
base_slice_2,
|
||||
base_slice,
|
||||
label="base_after_reset",
|
||||
base_max=base_ref_max,
|
||||
base_mean=base_ref_mean,
|
||||
)
|
||||
|
||||
# Ensure LoRA effects are clearly above the baseline drift.
|
||||
min_delta = max(base_reset_mean + 1.0, 1.5)
|
||||
assert a_vs_base > min_delta, f"lora_a_vs_base drift too small: {a_vs_base} <= {min_delta}"
|
||||
assert b_vs_base > min_delta, f"lora_b_vs_base drift too small: {b_vs_base} <= {min_delta}"
|
||||
assert b_vs_a > min_delta, f"lora_b_vs_lora_a drift too small: {b_vs_a} <= {min_delta}"
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Recommended tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following model:
|
||||
- LongCat-Image: text-to-image with single prompt input
|
||||
Coverage:
|
||||
- CPU offloading (model-level sequential offload via --enable-cpu-offload)
|
||||
- Cache-DiT
|
||||
- SP (Ulysses)
|
||||
|
||||
This validates:
|
||||
- Successful image generation at the expected 768x1344 resolution with recommended feature combinations
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
TEXT_TO_IMAGE_PROMPT = (
|
||||
"A cinematic illustration of a cat typing on a silver laptop, soft window light, highly detailed."
|
||||
)
|
||||
NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return diffusion feature cases for LongCat-Image."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("meituan-longcat/LongCat-Image"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_longcat_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test the recommended feature combinations for LongCat-Image."""
|
||||
messages = dummy_messages_from_mix_data(content_text=TEXT_TO_IMAGE_PROMPT)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 768,
|
||||
"width": 1344,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"guidance_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Recommended tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following model:
|
||||
- LongCat-Image-Edit: image-to-image edit with a single image + single edit prompt input
|
||||
Coverage:
|
||||
- CPU offloading (model-level sequential offload via --enable-cpu-offload)
|
||||
- Cache-DiT
|
||||
- SP (Ulysses)
|
||||
|
||||
This validates:
|
||||
- Successful image generation at the expected 1024x1024 resolution with recommended feature combinations
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
EDIT_PROMPT = "Transform this modern image into a cinematic animation style with vibrant colors and soft lighting."
|
||||
NEGATIVE_PROMPT = "blurry, low quality, distorted, oversaturated"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return diffusion feature cases for LongCat-Image-Edit."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("meituan-longcat/LongCat-Image-Edit"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_longcat_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test the recommended feature combinations for LongCat-Image-Edit."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 1024,
|
||||
"width": 1024,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"guidance_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -53,7 +53,7 @@ try:
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
stage_config_path=stage_config,
|
||||
server_args=["--chat-template", CHAT_TEMPLATE_PATH, "--load-format", "dummy"],
|
||||
server_args=["--chat-template", CHAT_TEMPLATE_PATH],
|
||||
)
|
||||
for model in models
|
||||
for stage_config in stage_configs
|
||||
@@ -68,7 +68,7 @@ except Exception as exc:
|
||||
def get_prompt(prompt_type="text_only"):
|
||||
prompts = {
|
||||
"text_only": "What is the capital of China? Answer in 20 words.",
|
||||
"audio": "What is recited in the audio?",
|
||||
"audio": "What one English word is repeated in the audio?",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["text_only"])
|
||||
|
||||
@@ -104,6 +104,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"sampling_params_list": [{"max_tokens": 64}, {"max_tokens": 64}],
|
||||
"key_words": {
|
||||
"audio": ["test"],
|
||||
},
|
||||
|
||||
+12
-13
@@ -21,6 +21,12 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.full_model]
|
||||
|
||||
_SKIP_NEED_4_H100_NOT_CI = pytest.mark.skip(
|
||||
reason="Requires 4x H100 GPUs; skipped in CI for now.",
|
||||
)
|
||||
|
||||
models = ["Jonathan1909/Ming-flash-omni-2.0"]
|
||||
|
||||
# Use thinker-only topology to test text outputs
|
||||
@@ -62,8 +68,7 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_NEED_4_H100_NOT_CI
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
@@ -89,8 +94,7 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_NEED_4_H100_NOT_CI
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_text_stream_001(omni_server, openai_client) -> None:
|
||||
@@ -116,8 +120,7 @@ def test_text_to_text_stream_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_NEED_4_H100_NOT_CI
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_image_to_text_001(omni_server, openai_client) -> None:
|
||||
@@ -144,8 +147,7 @@ def test_image_to_text_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_NEED_4_H100_NOT_CI
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_audio_to_text_001(omni_server, openai_client) -> None:
|
||||
@@ -172,8 +174,7 @@ def test_audio_to_text_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_NEED_4_H100_NOT_CI
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_video_to_text_001(omni_server, openai_client) -> None:
|
||||
@@ -200,9 +201,7 @@ def test_video_to_text_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@_SKIP_NEED_4_H100_NOT_CI
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_mix_to_text_001(omni_server, openai_client) -> None:
|
||||
+7
-8
@@ -25,6 +25,13 @@ from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
# ``omni`` for all tests; 001/002 also get ``full_model`` via ``TestMossTtsNanoFull`` (003 is core+advanced only, no full_model).
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
_SKIP_ISSUE_3168 = pytest.mark.skip(
|
||||
reason="https://github.com/vllm-project/vllm-omni/issues/3168",
|
||||
)
|
||||
|
||||
MODEL = "OpenMOSS-Team/MOSS-TTS-Nano"
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/OpenMOSS/MOSS-TTS-Nano/main/assets/audio/zh_1.wav"
|
||||
REF_AUDIO_TRANSCRIPT = "欢迎关注模思智能、上海创智学院与复旦大学自然语言处理实验室。"
|
||||
@@ -79,8 +86,6 @@ tts_server_params = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
@@ -103,8 +108,6 @@ def test_text_to_audio_001(omni_server, openai_client, ref_audio_data_url) -> No
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_002(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
@@ -136,8 +139,6 @@ def test_text_to_audio_002(omni_server, openai_client, ref_audio_data_url) -> No
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_003(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
@@ -160,8 +161,6 @@ def test_text_to_audio_003(omni_server, openai_client, ref_audio_data_url) -> No
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_004_ref_text_ignored(omni_server, openai_client, ref_audio_data_url) -> None:
|
||||
@@ -1,201 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for OmniVoice TTS model via /v1/audio/speech endpoint.
|
||||
|
||||
Tests verify that the OmniVoice model generates valid audio when
|
||||
accessed through the standard OpenAI-compatible speech API.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import generate_synthetic_audio
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
|
||||
try:
|
||||
from transformers import HiggsAudioV2TokenizerModel # noqa: F401
|
||||
|
||||
_HAS_VOICE_CLONE = True
|
||||
except ImportError:
|
||||
_HAS_VOICE_CLONE = False
|
||||
|
||||
MODEL = "k2-fsa/OmniVoice"
|
||||
|
||||
STAGE_CONFIG = str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "omnivoice.yaml"
|
||||
)
|
||||
EXTRA_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--disable-log-stats",
|
||||
]
|
||||
TEST_PARAMS = [
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=STAGE_CONFIG,
|
||||
server_args=EXTRA_ARGS,
|
||||
)
|
||||
]
|
||||
|
||||
MIN_AUDIO_BYTES = 5000
|
||||
|
||||
|
||||
def _get_ref_audio_b64() -> str:
|
||||
"""Generate synthetic speech for reference audio.
|
||||
|
||||
Returns:
|
||||
Base64 data URL string (data:audio/wav;base64,...)
|
||||
"""
|
||||
audio_data = generate_synthetic_audio(duration=2, num_channels=1, sample_rate=24000)
|
||||
return f"data:audio/wav;base64,{audio_data['base64']}"
|
||||
|
||||
|
||||
def make_speech_request(
|
||||
host: str,
|
||||
port: int,
|
||||
text: str,
|
||||
timeout: float = 180.0,
|
||||
) -> httpx.Response:
|
||||
"""Make a request to the /v1/audio/speech endpoint for OmniVoice."""
|
||||
url = f"http://{host}:{port}/v1/audio/speech"
|
||||
payload = {"input": text}
|
||||
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
return client.post(url, json=payload)
|
||||
|
||||
|
||||
def verify_wav_audio(content: bytes) -> bool:
|
||||
"""Verify that content is valid WAV audio data."""
|
||||
if len(content) < 44:
|
||||
return False
|
||||
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||
class TestOmniVoiceTTS:
|
||||
"""E2E tests for OmniVoice TTS model."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speech_auto_voice(self, omni_server) -> None:
|
||||
"""Test auto voice TTS generation (text only, no reference audio)."""
|
||||
response = make_speech_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="Hello, this is a test of the OmniVoice text to speech system.",
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert response.headers.get("content-type") == "audio/wav"
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES, (
|
||||
f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
|
||||
)
|
||||
|
||||
|
||||
def make_voice_clone_request(
|
||||
host: str,
|
||||
port: int,
|
||||
text: str,
|
||||
ref_audio_b64: str,
|
||||
ref_text: str | None = None,
|
||||
timeout: float = 180.0,
|
||||
) -> httpx.Response:
|
||||
"""Make a voice cloning request to the /v1/audio/speech endpoint.
|
||||
|
||||
Args:
|
||||
host: Server host
|
||||
port: Server port
|
||||
text: Text to synthesize
|
||||
ref_audio_b64: Base64-encoded reference audio data URL
|
||||
ref_text: Optional transcript of reference audio
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
httpx.Response object
|
||||
"""
|
||||
url = f"http://{host}:{port}/v1/audio/speech"
|
||||
payload = {
|
||||
"input": text,
|
||||
"ref_audio": ref_audio_b64,
|
||||
}
|
||||
if ref_text:
|
||||
payload["ref_text"] = ref_text
|
||||
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
return client.post(url, json=payload)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_VOICE_CLONE, reason="Voice cloning requires transformers>=5.3.0")
|
||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||
class TestOmniVoiceVoiceCloning:
|
||||
"""E2E tests for OmniVoice voice cloning functionality."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_ref_audio_only(self, omni_server) -> None:
|
||||
"""Test voice cloning with ref_audio only (x_vector mode)."""
|
||||
ref_audio_b64 = _get_ref_audio_b64()
|
||||
|
||||
response = make_voice_clone_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="Hello, this is a voice cloning test.",
|
||||
ref_audio_b64=ref_audio_b64,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert response.headers.get("content-type") == "audio/wav"
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES, (
|
||||
f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_ref_audio_and_text(self, omni_server) -> None:
|
||||
"""Test voice cloning with ref_audio and ref_text (in-context mode)."""
|
||||
ref_audio_b64 = _get_ref_audio_b64()
|
||||
ref_text = "This is the reference transcript."
|
||||
|
||||
response = make_voice_clone_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="Hello, this is a voice cloning test with in-context learning.",
|
||||
ref_audio_b64=ref_audio_b64,
|
||||
ref_text=ref_text,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert response.headers.get("content-type") == "audio/wav"
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES, (
|
||||
f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_invalid_ref_audio_format(self, omni_server) -> None:
|
||||
"""Test that invalid ref_audio format returns a clear error."""
|
||||
response = make_voice_clone_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="This should fail with invalid ref_audio.",
|
||||
ref_audio_b64="not_a_valid_uri",
|
||||
)
|
||||
|
||||
assert response.status_code in (400, 422), (
|
||||
f"Expected 400/422 for invalid ref_audio format, got {response.status_code}"
|
||||
)
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for OmniVoice TTS model via /v1/audio/speech endpoint.
|
||||
|
||||
Tests verify that the OmniVoice model generates valid audio when
|
||||
accessed through the standard OpenAI-compatible speech API.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
try:
|
||||
from transformers import HiggsAudioV2TokenizerModel # noqa: F401
|
||||
|
||||
_HAS_VOICE_CLONE = True
|
||||
except ImportError:
|
||||
_HAS_VOICE_CLONE = False
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
MODEL = "k2-fsa/OmniVoice"
|
||||
|
||||
STAGE_CONFIG = get_deploy_config_path("omnivoice.yaml")
|
||||
EXTRA_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--disable-log-stats",
|
||||
]
|
||||
TEST_PARAMS = [
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=STAGE_CONFIG,
|
||||
server_args=EXTRA_ARGS,
|
||||
)
|
||||
]
|
||||
|
||||
# Lower this in ``request_config`` via ``min_audio_bytes`` if a run produces legitimately short WAVs.
|
||||
_DEFAULT_MIN_AUDIO_BYTES = 5000
|
||||
|
||||
|
||||
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
||||
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
||||
|
||||
|
||||
def get_prompt(prompt_type="text"):
|
||||
prompts = {
|
||||
"text": "The weather is nice today, perfect for a walk in the park.",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["text"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||
class TestOmniVoiceTTS:
|
||||
"""E2E tests for OmniVoice TTS model."""
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speech_auto_voice(self, omni_server, openai_client) -> None:
|
||||
"""Test auto voice TTS generation (text only, no reference audio)."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("text"),
|
||||
"response_format": "wav",
|
||||
"timeout": 180.0,
|
||||
"min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_VOICE_CLONE, reason="Voice cloning requires transformers>=5.3.0")
|
||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||
class TestOmniVoiceVoiceCloning:
|
||||
"""E2E tests for OmniVoice voice cloning functionality."""
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_ref_audio_only(self, omni_server, openai_client) -> None:
|
||||
"""Test voice cloning with ref_audio only (x_vector mode)."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("text"),
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"response_format": "wav",
|
||||
"timeout": 180.0,
|
||||
"min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_ref_audio_and_text(self, omni_server, openai_client) -> None:
|
||||
"""Test voice cloning with ref_audio and ref_text (in-context mode)."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("text"),
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"ref_text": REF_TEXT,
|
||||
"response_format": "wav",
|
||||
"timeout": 180.0,
|
||||
"min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_invalid_ref_audio_format(self, omni_server, openai_client) -> None:
|
||||
"""Test that invalid ref_audio format returns a clear error."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("text"),
|
||||
"ref_audio": "not_a_valid_uri",
|
||||
"response_format": "wav",
|
||||
"timeout": 180.0,
|
||||
"status_code": (400, 422),
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
@@ -87,7 +87,7 @@ def _assert_audio_outputs(responses: list[OmniResponse], *, expect_text: bool) -
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
@@ -104,7 +104,7 @@ def test_text_only_batch_uses_multi_replica_talker(omni_server, openai_client) -
|
||||
_assert_text_outputs(responses)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
@@ -121,7 +121,7 @@ def test_text_to_audio_stream_batch_uses_multi_replica_vocoder(omni_server, open
|
||||
_assert_audio_outputs(responses, expect_text=False)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
|
||||
@@ -20,6 +20,7 @@ from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 180.0
|
||||
|
||||
# Vendored under tests/assets/qwen3_tts/clone_2.wav so the server does not need
|
||||
# to fetch the reference audio over HTTPS at request time (CI runners in some
|
||||
@@ -56,7 +57,7 @@ tts_server_params = [
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
@@ -73,6 +74,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": False,
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"response_format": "wav",
|
||||
"task_type": "Base",
|
||||
"voice": "clone",
|
||||
@@ -83,7 +85,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_002(omni_server, openai_client) -> None:
|
||||
@@ -99,6 +101,7 @@ def test_text_to_audio_002(omni_server, openai_client) -> None:
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt(),
|
||||
"stream": True,
|
||||
"timeout": DEFAULT_AUDIO_SPEECH_TIMEOUT_S,
|
||||
"response_format": "wav",
|
||||
"task_type": "Base",
|
||||
"voice": "clone",
|
||||
|
||||
@@ -16,7 +16,7 @@ from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
@@ -1,433 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E tests for the /v1/audio/speech/batch endpoint.
|
||||
|
||||
Validates bulk synthesis via the batch API with actual model inference.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import base64
|
||||
import shutil
|
||||
import struct
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_text
|
||||
from tests.helpers.runtime import OmniServer
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
||||
STAGE_INIT_TIMEOUT_S = 120
|
||||
|
||||
|
||||
def get_stage_config(name: str = "qwen3_tts.yaml") -> str:
|
||||
"""Resolve a deploy config path under vllm_omni/deploy/."""
|
||||
return get_deploy_config_path(name)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server():
|
||||
"""Start vLLM-Omni server with CustomVoice model."""
|
||||
stage_config_path = get_stage_config()
|
||||
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--stage-configs-path",
|
||||
stage_config_path,
|
||||
"--stage-init-timeout",
|
||||
str(STAGE_INIT_TIMEOUT_S),
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
def make_batch_request(
|
||||
host: str,
|
||||
port: int,
|
||||
items: list[dict],
|
||||
voice: str | None = "vivian",
|
||||
language: str | None = "English",
|
||||
response_format: str = "wav",
|
||||
timeout: float = 300.0,
|
||||
**kwargs,
|
||||
) -> httpx.Response:
|
||||
"""Make a request to the /v1/audio/speech/batch endpoint."""
|
||||
url = f"http://{host}:{port}/v1/audio/speech/batch"
|
||||
payload: dict = {
|
||||
"items": items,
|
||||
"response_format": response_format,
|
||||
}
|
||||
if voice is not None:
|
||||
payload["voice"] = voice
|
||||
if language is not None:
|
||||
payload["language"] = language
|
||||
payload.update(kwargs)
|
||||
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
return client.post(url, json=payload)
|
||||
|
||||
|
||||
def verify_wav_bytes(audio_bytes: bytes) -> bool:
|
||||
"""Verify decoded audio bytes form a valid WAV."""
|
||||
if len(audio_bytes) < 44:
|
||||
return False
|
||||
return audio_bytes[:4] == b"RIFF" and audio_bytes[8:12] == b"WAVE"
|
||||
|
||||
|
||||
def assert_not_silence(pcm_bytes: bytes):
|
||||
"""Assert PCM16 samples are not all identical."""
|
||||
samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
|
||||
unique = set(samples)
|
||||
assert len(unique) > 1, f"All-silence: {len(samples)} samples, unique={unique}"
|
||||
|
||||
|
||||
MIN_AUDIO_BYTES = 10000
|
||||
|
||||
|
||||
class TestSpeechBatchE2E:
|
||||
"""E2E tests for /v1/audio/speech/batch endpoint."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_basic_two_items(self, omni_server) -> None:
|
||||
"""Batch with two items returns two successful base64-encoded results."""
|
||||
items = [
|
||||
{"input": "Hello, how are you today?"},
|
||||
{"input": "The weather is nice outside."},
|
||||
]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
|
||||
assert data["total"] == 2
|
||||
assert data["succeeded"] == 2
|
||||
assert data["failed"] == 0
|
||||
assert data["id"].startswith("speech-batch-")
|
||||
assert len(data["results"]) == 2
|
||||
|
||||
for i, result in enumerate(data["results"]):
|
||||
assert result["index"] == i
|
||||
assert result["status"] == "success"
|
||||
assert result["audio_data"] is not None
|
||||
assert result["error"] is None
|
||||
audio_bytes = base64.b64decode(result["audio_data"])
|
||||
assert verify_wav_bytes(audio_bytes), f"Item {i}: invalid WAV"
|
||||
assert len(audio_bytes) > MIN_AUDIO_BYTES, f"Item {i}: audio too small ({len(audio_bytes)} bytes)"
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_single_item(self, omni_server) -> None:
|
||||
"""Batch with a single item works correctly."""
|
||||
items = [{"input": "Single item batch test."}]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
|
||||
assert data["total"] == 1
|
||||
assert data["succeeded"] == 1
|
||||
assert data["failed"] == 0
|
||||
assert len(data["results"]) == 1
|
||||
assert data["results"][0]["status"] == "success"
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_per_item_voice_override(self, omni_server) -> None:
|
||||
"""Per-item voice overrides the batch-level default."""
|
||||
items = [
|
||||
{"input": "First item with default voice."},
|
||||
{"input": "Second item with ryan voice.", "voice": "ryan"},
|
||||
]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
voice="vivian",
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
|
||||
assert data["total"] == 2
|
||||
assert data["succeeded"] == 2
|
||||
# Both items should produce valid audio
|
||||
for result in data["results"]:
|
||||
assert result["status"] == "success"
|
||||
audio_bytes = base64.b64decode(result["audio_data"])
|
||||
assert verify_wav_bytes(audio_bytes)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_multiple_languages(self, omni_server) -> None:
|
||||
"""Batch items with different languages per item."""
|
||||
items = [
|
||||
{"input": "Hello, nice to meet you.", "language": "English"},
|
||||
{"input": "你好,很高兴认识你。", "language": "Chinese"},
|
||||
]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
|
||||
assert data["succeeded"] == 2
|
||||
for result in data["results"]:
|
||||
assert result["status"] == "success"
|
||||
audio_bytes = base64.b64decode(result["audio_data"])
|
||||
assert len(audio_bytes) > MIN_AUDIO_BYTES
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_whisper_transcription(self, omni_server) -> None:
|
||||
"""Whisper transcription of batch output matches input text."""
|
||||
input_text = "Good morning, welcome to the speech synthesis test."
|
||||
items = [{"input": input_text}]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
assert data["succeeded"] == 1
|
||||
|
||||
audio_bytes = base64.b64decode(data["results"][0]["audio_data"])
|
||||
assert verify_wav_bytes(audio_bytes)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
||||
f.write(audio_bytes)
|
||||
wav_path = f.name
|
||||
|
||||
try:
|
||||
transcript = convert_audio_file_to_text(wav_path)
|
||||
print(f"Whisper transcript: {transcript}")
|
||||
assert len(transcript.strip()) > 0, "Empty transcript — likely silence"
|
||||
similarity = cosine_similarity_text(transcript.lower(), input_text.lower())
|
||||
print(f"Cosine similarity: {similarity:.3f}")
|
||||
assert similarity > 0.7, f"Transcript mismatch: similarity={similarity:.2f}, transcript='{transcript}'"
|
||||
finally:
|
||||
os.unlink(wav_path)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_result_indices_ordered(self, omni_server) -> None:
|
||||
"""Result indices match the input item order."""
|
||||
items = [
|
||||
{"input": "First sentence."},
|
||||
{"input": "Second sentence."},
|
||||
{"input": "Third sentence."},
|
||||
]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
|
||||
indices = [r["index"] for r in data["results"]]
|
||||
assert indices == [0, 1, 2]
|
||||
|
||||
|
||||
class TestSpeechBatchValidation:
|
||||
"""Validation / error-handling tests for the batch endpoint."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_empty_items_rejected(self, omni_server) -> None:
|
||||
"""Empty items list returns a 4xx error."""
|
||||
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech/batch"
|
||||
payload = {"items": [], "voice": "vivian"}
|
||||
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code in (400, 422)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch_exceeds_max_items(self, omni_server) -> None:
|
||||
"""Batch exceeding 32 items returns 400 error."""
|
||||
items = [{"input": f"Item {i}"} for i in range(33)]
|
||||
response = make_batch_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def _make_batch2_stage_config() -> str:
|
||||
"""Create a temporary stage config with max_batch_size=2 for both stages."""
|
||||
src = Path(get_stage_config("qwen3_tts_batch.yaml"))
|
||||
with open(src) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
|
||||
for stage in cfg["stage_args"]:
|
||||
stage["runtime"]["max_batch_size"] = 2
|
||||
|
||||
tmp_dir = tempfile.mkdtemp(prefix="tts_batch2_")
|
||||
dst = Path(tmp_dir) / "qwen3_tts_batch2.yaml"
|
||||
with open(dst, "w") as f:
|
||||
yaml.dump(cfg, f)
|
||||
return str(dst), tmp_dir
|
||||
|
||||
|
||||
def make_single_request(
|
||||
host: str,
|
||||
port: int,
|
||||
text: str,
|
||||
voice: str = "vivian",
|
||||
language: str = "English",
|
||||
timeout: float = 300.0,
|
||||
) -> httpx.Response:
|
||||
"""Make a request to the single /v1/audio/speech endpoint."""
|
||||
url = f"http://{host}:{port}/v1/audio/speech"
|
||||
payload = {"input": text, "voice": voice, "language": language}
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
return client.post(url, json=payload)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server_batch2():
|
||||
"""Start vLLM-Omni server with max_batch_size=2 config."""
|
||||
config_path, tmp_dir = _make_batch2_stage_config()
|
||||
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--stage-configs-path",
|
||||
config_path,
|
||||
"--stage-init-timeout",
|
||||
str(STAGE_INIT_TIMEOUT_S),
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
class TestSpeechBatchSize2:
|
||||
"""E2E tests with max_batch_size=2 to verify true batched inference."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch2_produces_valid_audio(self, omni_server_batch2) -> None:
|
||||
"""Batch of 2 items with batched engine produces valid audio."""
|
||||
items = [
|
||||
{"input": "Hello, this is the first sentence."},
|
||||
{"input": "And this is the second sentence."},
|
||||
]
|
||||
response = make_batch_request(
|
||||
host=omni_server_batch2.host,
|
||||
port=omni_server_batch2.port,
|
||||
items=items,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
data = response.json()
|
||||
|
||||
assert data["total"] == 2
|
||||
assert data["succeeded"] == 2
|
||||
assert data["failed"] == 0
|
||||
|
||||
# Save audio files for inspection
|
||||
output_dir = Path(tempfile.mkdtemp(prefix="tts_batch2_output_"))
|
||||
for result in data["results"]:
|
||||
assert result["status"] == "success"
|
||||
audio_bytes = base64.b64decode(result["audio_data"])
|
||||
assert verify_wav_bytes(audio_bytes), f"Item {result['index']}: invalid WAV"
|
||||
assert len(audio_bytes) > MIN_AUDIO_BYTES
|
||||
|
||||
wav_path = output_dir / f"batch_item_{result['index']}.wav"
|
||||
wav_path.write_bytes(audio_bytes)
|
||||
print(f" Item {result['index']}: {len(audio_bytes)} bytes -> {wav_path}")
|
||||
|
||||
print(f"\nBatch audio saved to: {output_dir}")
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_batch2_vs_sequential_timing(self, omni_server_batch2) -> None:
|
||||
"""Compare batch-of-2 vs 2 sequential single requests.
|
||||
|
||||
With max_batch_size=2 the engine can process both items
|
||||
concurrently, so the batch call should be faster than two
|
||||
sequential single calls (or at least comparable).
|
||||
"""
|
||||
text_a = "The quick brown fox jumps over the lazy dog."
|
||||
text_b = "A journey of a thousand miles begins with a single step."
|
||||
|
||||
host = omni_server_batch2.host
|
||||
port = omni_server_batch2.port
|
||||
|
||||
# --- Sequential: 2 single /v1/audio/speech calls ---
|
||||
t0 = time.perf_counter()
|
||||
resp_a = make_single_request(host, port, text_a)
|
||||
resp_b = make_single_request(host, port, text_b)
|
||||
sequential_time = time.perf_counter() - t0
|
||||
|
||||
assert resp_a.status_code == 200, f"Single A failed: {resp_a.text}"
|
||||
assert resp_b.status_code == 200, f"Single B failed: {resp_b.text}"
|
||||
|
||||
# --- Batch: 1 /v1/audio/speech/batch call with 2 items ---
|
||||
items = [{"input": text_a}, {"input": text_b}]
|
||||
t0 = time.perf_counter()
|
||||
resp_batch = make_batch_request(host, port, items)
|
||||
batch_time = time.perf_counter() - t0
|
||||
|
||||
assert resp_batch.status_code == 200, f"Batch failed: {resp_batch.text}"
|
||||
batch_data = resp_batch.json()
|
||||
assert batch_data["succeeded"] == 2
|
||||
|
||||
# Verify both produced valid audio
|
||||
for result in batch_data["results"]:
|
||||
audio_bytes = base64.b64decode(result["audio_data"])
|
||||
assert verify_wav_bytes(audio_bytes)
|
||||
assert len(audio_bytes) > MIN_AUDIO_BYTES
|
||||
|
||||
print(f"\n Sequential (2 singles): {sequential_time:.2f}s")
|
||||
print(f" Batch (2 items): {batch_time:.2f}s")
|
||||
print(f" Speedup: {sequential_time / batch_time:.2f}x")
|
||||
@@ -49,7 +49,7 @@ tts_server_params = [
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
@@ -75,7 +75,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_text_to_audio_002(omni_server, openai_client) -> None:
|
||||
|
||||
@@ -15,7 +15,7 @@ from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.tts]
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E tests for speaker_embedding API parameter.
|
||||
|
||||
Tests Base models (0.6B-Base and 1.7B-Base) which support voice cloning
|
||||
via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import struct
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
|
||||
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
STAGE_INIT_TIMEOUT_S = 120
|
||||
|
||||
# A synthetic 1024-dim speaker embedding (all 0.1 — not a real voice, but
|
||||
# exercises the full code path through the talker's _build_prompt_embeds).
|
||||
DUMMY_EMBEDDING_1024 = [0.1] * 1024
|
||||
DUMMY_EMBEDDING_2048 = [0.1] * 2048
|
||||
|
||||
SYN_TEXT = "Hello."
|
||||
MIN_AUDIO_BYTES = 2000
|
||||
# Limit generation to keep tests fast (dummy embeddings produce nonsensical
|
||||
# output that may never hit a natural stop token).
|
||||
MAX_NEW_TOKENS = 256
|
||||
|
||||
|
||||
def get_stage_config() -> str:
|
||||
return get_deploy_config_path("qwen3_tts.yaml")
|
||||
|
||||
|
||||
def _server_args():
|
||||
return [
|
||||
"--stage-configs-path",
|
||||
get_stage_config(),
|
||||
"--stage-init-timeout",
|
||||
str(STAGE_INIT_TIMEOUT_S),
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
]
|
||||
|
||||
|
||||
def verify_wav_audio(content: bytes) -> bool:
|
||||
if len(content) < 44:
|
||||
return False
|
||||
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
|
||||
|
||||
|
||||
def assert_not_silence(pcm_bytes: bytes):
|
||||
"""Assert PCM16 samples are not all identical (e.g. all-silence)."""
|
||||
samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
|
||||
unique = set(samples)
|
||||
assert len(unique) > 1, f"All-silence detected: {len(samples)} samples, unique values: {unique}"
|
||||
|
||||
|
||||
# ── 0.6B-Base model tests ──
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def base_server():
|
||||
"""Start vLLM-Omni server with 0.6B-Base model."""
|
||||
with OmniServer(MODEL_BASE, _server_args()) as server:
|
||||
yield server
|
||||
|
||||
|
||||
class TestSpeakerEmbeddingBase:
|
||||
"""Speaker embedding tests against the 0.6B-Base model (supports Base task)."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speaker_embedding_produces_audio(self, base_server) -> None:
|
||||
"""speaker_embedding with Base task produces valid WAV audio."""
|
||||
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": DUMMY_EMBEDDING_1024,
|
||||
"x_vector_only_mode": True,
|
||||
"response_format": "wav",
|
||||
"max_new_tokens": MAX_NEW_TOKENS,
|
||||
}
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert response.headers.get("content-type") == "audio/wav"
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speaker_embedding_pcm_not_silence(self, base_server) -> None:
|
||||
"""speaker_embedding PCM output contains real audio, not all-silence."""
|
||||
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": DUMMY_EMBEDDING_1024,
|
||||
"x_vector_only_mode": True,
|
||||
"response_format": "pcm",
|
||||
"max_new_tokens": MAX_NEW_TOKENS,
|
||||
}
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES
|
||||
assert_not_silence(response.content)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speaker_embedding_streaming(self, base_server) -> None:
|
||||
"""speaker_embedding works with streaming PCM output."""
|
||||
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": DUMMY_EMBEDDING_1024,
|
||||
"x_vector_only_mode": True,
|
||||
"response_format": "pcm",
|
||||
"stream": True,
|
||||
"max_new_tokens": MAX_NEW_TOKENS,
|
||||
}
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert "audio/pcm" in response.headers.get("content-type", "")
|
||||
assert len(response.content) > MIN_AUDIO_BYTES
|
||||
assert_not_silence(response.content)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speaker_embedding_mutually_exclusive_with_ref_audio(self, base_server) -> None:
|
||||
"""Sending both speaker_embedding and ref_audio returns 400."""
|
||||
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": DUMMY_EMBEDDING_1024,
|
||||
"ref_audio": "https://example.com/audio.wav",
|
||||
"response_format": "wav",
|
||||
}
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
|
||||
assert "mutually exclusive" in response.text
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_speaker_embedding_empty_rejected(self, base_server) -> None:
|
||||
"""Empty speaker_embedding list returns 400."""
|
||||
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": [],
|
||||
"response_format": "wav",
|
||||
}
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
|
||||
assert "non-empty" in response.text
|
||||
|
||||
|
||||
# ── 1.7B-Base model tests (2048-dim embeddings) ──
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def base_1_7b_server():
|
||||
"""Start vLLM-Omni server with 1.7B-Base model."""
|
||||
with OmniServer(MODEL_BASE_1_7B, _server_args()) as server:
|
||||
yield server
|
||||
|
||||
|
||||
class TestSpeakerEmbedding1_7B:
|
||||
"""Speaker embedding tests against the 1.7B-Base model (2048-dim embeddings)."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_2048_dim_embedding_produces_audio(self, base_1_7b_server) -> None:
|
||||
"""2048-dim speaker_embedding with 1.7B-Base model produces valid WAV audio."""
|
||||
url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE_1_7B,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": DUMMY_EMBEDDING_2048,
|
||||
"x_vector_only_mode": True,
|
||||
"response_format": "wav",
|
||||
"max_new_tokens": MAX_NEW_TOKENS,
|
||||
}
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert response.headers.get("content-type") == "audio/wav"
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_1024_dim_on_1_7b_model_rejected(self, base_1_7b_server) -> None:
|
||||
"""1024-dim speaker_embedding on a 1.7B model (expects 2048) should fail gracefully."""
|
||||
url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"model": MODEL_BASE_1_7B,
|
||||
"input": SYN_TEXT,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": DUMMY_EMBEDDING_1024,
|
||||
"x_vector_only_mode": True,
|
||||
"response_format": "wav",
|
||||
"max_new_tokens": MAX_NEW_TOKENS,
|
||||
}
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
response = client.post(url, json=payload)
|
||||
|
||||
# Wrong dimensions should produce an error, not silent garbage.
|
||||
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
|
||||
assert "speaker_embedding has 1024 dimensions" in response.text
|
||||
assert "expected 2048" in response.text
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online tests for Qwen3-TTS WebSocket streaming speech.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
||||
STAGE_INIT_TIMEOUT_S = 120
|
||||
|
||||
|
||||
def get_stage_config() -> str:
|
||||
return get_deploy_config_path("qwen3_tts.yaml")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server():
|
||||
stage_config_path = get_stage_config()
|
||||
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--stage-configs-path",
|
||||
stage_config_path,
|
||||
"--stage-init-timeout",
|
||||
str(STAGE_INIT_TIMEOUT_S),
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
env_dict={"VLLM_DISABLE_COMPILE_CACHE": "1"},
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
async def _run_ws_session(host: str, port: int) -> dict:
|
||||
uri = f"ws://{host}:{port}/v1/audio/speech/stream"
|
||||
starts: list[dict] = []
|
||||
dones: list[dict] = []
|
||||
chunk_lengths: dict[int, list[int]] = {}
|
||||
session_done: dict | None = None
|
||||
|
||||
async with websockets.connect(uri, max_size=None) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": MODEL,
|
||||
"voice": "vivian",
|
||||
"language": "English",
|
||||
"response_format": "pcm",
|
||||
"stream_audio": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": (
|
||||
"Hello, this is a websocket streaming test for Qwen three TTS, "
|
||||
"and this sentence is intentionally long enough to produce audio chunks. "
|
||||
"This is the second sentence."
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
|
||||
while True:
|
||||
message = await asyncio.wait_for(ws.recv(), timeout=180)
|
||||
if isinstance(message, bytes):
|
||||
if not starts:
|
||||
raise AssertionError("Received audio bytes before audio.start")
|
||||
sentence_index = starts[-1]["sentence_index"]
|
||||
chunk_lengths.setdefault(sentence_index, []).append(len(message))
|
||||
continue
|
||||
|
||||
payload = json.loads(message)
|
||||
msg_type = payload.get("type")
|
||||
if msg_type == "audio.start":
|
||||
starts.append(payload)
|
||||
chunk_lengths.setdefault(payload["sentence_index"], [])
|
||||
elif msg_type == "audio.done":
|
||||
dones.append(payload)
|
||||
elif msg_type == "session.done":
|
||||
session_done = payload
|
||||
break
|
||||
elif msg_type == "error":
|
||||
raise AssertionError(f"WebSocket error: {payload['message']}")
|
||||
else:
|
||||
raise AssertionError(f"Unexpected WebSocket message: {payload}")
|
||||
|
||||
return {
|
||||
"starts": starts,
|
||||
"dones": dones,
|
||||
"chunk_lengths": chunk_lengths,
|
||||
"session_done": session_done,
|
||||
}
|
||||
|
||||
|
||||
class TestQwen3TTSWebSocket:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
def test_streaming_pcm_output(self, omni_server) -> None:
|
||||
result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port))
|
||||
|
||||
starts = result["starts"]
|
||||
dones = result["dones"]
|
||||
chunk_lengths = result["chunk_lengths"]
|
||||
session_done = result["session_done"]
|
||||
|
||||
assert session_done is not None
|
||||
assert session_done["total_sentences"] == 2
|
||||
assert len(starts) == 2
|
||||
assert len(dones) == 2
|
||||
|
||||
for idx, start in enumerate(starts):
|
||||
assert start["type"] == "audio.start"
|
||||
assert start["sentence_index"] == idx
|
||||
assert start["format"] == "pcm"
|
||||
assert start["sample_rate"] == 24000
|
||||
assert start["sentence_text"]
|
||||
|
||||
for done in dones:
|
||||
sentence_index = done["sentence_index"]
|
||||
total_bytes = done["total_bytes"]
|
||||
assert done["error"] is False
|
||||
assert total_bytes > 0
|
||||
assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}"
|
||||
assert sum(chunk_lengths[sentence_index]) == total_bytes
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Online serving tests for ``Qwen/Qwen-Image`` (text-to-image).
|
||||
|
||||
- ``test_text_to_image_001``: single chat request, default server (``_get_default_case``).
|
||||
- ``test_batch_001``: concurrent prompts via ``send_diffusion_request([cfg0, cfg1, ...])`` — one
|
||||
dict per prompt; each entry carries its own ``messages`` / ``negative_prompt`` (see
|
||||
``TEST_PROMPTS``).
|
||||
- ``test_acceleration_feature_001``: ``advanced_model`` only — parametrized ``OmniServerParams``
|
||||
with cache/offload/parallel flags (see ``_get_diffusion_feature_cases``).
|
||||
|
||||
Full feature grids for Qwen-Image remain in ``test_qwen_image_expansion.py`` (L4).
|
||||
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_qwen_image.py -m "core_model and diffusion" --run-level=core_model
|
||||
pytest -s -v e2e/online_serving/test_qwen_image.py -m "advanced_model and diffusion" --run-level=advanced_model
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen-Image"
|
||||
T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
|
||||
NEGATIVE_PROMPT = "blurry, low quality"
|
||||
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
||||
|
||||
TEST_PROMPTS: list[dict[str, str]] = [
|
||||
{"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"},
|
||||
{"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": "cinematic, realistic"},
|
||||
{"prompt": "a futuristic city skyline at sunset", "negative_prompt": "blurry, foggy"},
|
||||
{"prompt": "a bowl of fresh strawberries", "negative_prompt": "low detail"},
|
||||
{"prompt": "a medieval knight standing in the rain", "negative_prompt": "modern clothing"},
|
||||
{"prompt": "a cat wearing sunglasses lounging in a garden", "negative_prompt": "dark lighting"},
|
||||
{"prompt": "a spaceship flying above a volcano", "negative_prompt": "low contrast"},
|
||||
{"prompt": "a watercolor painting of a mountain lake", "negative_prompt": "photo, realistic"},
|
||||
]
|
||||
|
||||
|
||||
def _get_default_case(model: str):
|
||||
"""Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(model=model),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
|
||||
def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
"""Default Qwen-Image T2I smoke (single ``default`` server config)."""
|
||||
messages = dummy_messages_from_mix_data(content_text=T2I_PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
|
||||
def test_batch_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
"""Concurrent T2I: one ``request_config`` dict per prompt (``send_diffusion_request`` list mode)."""
|
||||
request_config = [
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"messages": dummy_messages_from_mix_data(content_text=prompt["prompt"]),
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": prompt["negative_prompt"],
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
for prompt in TEST_PROMPTS
|
||||
]
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
|
||||
|
||||
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
|
||||
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2509`` — two reference images, fixed 512×512.
|
||||
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2509`` only, ``advanced_model`` — mixed input
|
||||
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
|
||||
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
|
||||
|
||||
``_get_diffusion_feature_cases`` registers a single ``default`` ``OmniServerParams`` row per model.
|
||||
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "core_model and diffusion" --run-level=core_model
|
||||
pytest -s -v e2e/online_serving/test_qwen_image_edit.py -m "advanced_model and diffusion" --run-level=advanced_model
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
"Transform the second image into a Vincent van Gogh style painting. "
|
||||
"Then juxtapose the two transformed images into a single artwork for visual contrast."
|
||||
)
|
||||
NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return one ``default`` ``OmniServerParams`` row for ``model`` (no extra ``server_args``)."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Single-reference edit smoke for ``Qwen/Qwen-Image-Edit`` (CFG when negative prompt + ``true_cfg_scale`` > 1)."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2509``."""
|
||||
|
||||
image_data_url_list = [f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" for _ in range(2)]
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url_list, content_text=MULTI_EDIT_PROMPT)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Multi-reference edit with distinct input resolutions; per-output ``width``/``height`` as lists (client splits into concurrent scalar-size calls)."""
|
||||
image_data_url_list = [
|
||||
f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}",
|
||||
f"data:image/jpeg;base64,{generate_synthetic_image(384, 256)['base64']}",
|
||||
]
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url_list, content_text=MULTI_EDIT_PROMPT)
|
||||
|
||||
# List height/width: client sends 512×512 and 768×512 in parallel, merges to two images; assertions use lists.
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": [512, 768],
|
||||
"width": [512, 512],
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Online serving tests for ``Qwen/Qwen-Image-Layered`` (layer decomposition).
|
||||
|
||||
- ``test_feature``: default single-GPU smoke via ``_get_diffusion_feature_cases`` (one ``default`` case).
|
||||
- ``test_layered_output_image_count``: guard for issue #1969 (multi-layer image count).
|
||||
- ``test_empty_prompt``: guard for issue #1966 (empty prompt).
|
||||
|
||||
Broader parallel / cache feature matrices previously lived in ``test_qwen_image_layered_expansion.py``;
|
||||
that module is now a shim that points here.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import generate_synthetic_image
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
MODEL = "Qwen/Qwen-Image-Layered"
|
||||
EDIT_PROMPT = "Decompose this image into layers."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
"Transform the second image into a Vincent van Gogh style painting. "
|
||||
"Then juxtapose the two transformed images into a single artwork for visual contrast."
|
||||
)
|
||||
NEGATIVE_PROMPT = "blurry, low quality, distorted"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(model=model),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
|
||||
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Default Qwen-Image-Layered smoke (single ``default`` server config)."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
Tests for Stable Diffusion 3.5 medium model.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
FOUR_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
|
||||
POSITIVE_PROMPT = "A serene mountain landscape at sunset"
|
||||
NEGATIVE_PROMPT = "blurry, low quality, distorted"
|
||||
|
||||
|
||||
# For now, we only test one or two good configurations for this model to keep
|
||||
# the CI light. The test cases focus on the most common feature combinations
|
||||
# that provide good performance improvements.
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
return [
|
||||
# Cache-DiT + CFG Parallel + Tensor Parallel
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
),
|
||||
id="default",
|
||||
marks=FOUR_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(
|
||||
model="stabilityai/stable-diffusion-3.5-medium",
|
||||
),
|
||||
indirect=True,
|
||||
)
|
||||
def test_text_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 1024,
|
||||
"width": 1024,
|
||||
"num_inference_steps": 28,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"guidance_scale": 4.5,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -5,7 +5,12 @@ Tests for Stable Diffusion 3.5 medium model.
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
from tests.helpers.runtime import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
@@ -45,9 +50,10 @@ def _get_diffusion_feature_cases(model: str):
|
||||
indirect=True,
|
||||
)
|
||||
def test_sd3_medium(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
messages = dummy_messages_from_mix_data(content_text=POSITIVE_PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": [{"role": "user", "content": POSITIVE_PROMPT}],
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 1024,
|
||||
"width": 1024,
|
||||
|
||||
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online serving tests for the async video generation API.
|
||||
|
||||
These tests exercise the real `/v1/videos` job lifecycle against a running
|
||||
diffusion model, including create, retrieve, list, download, and delete flows.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||
VIDEO_POLL_INTERVAL_S = 2.0
|
||||
VIDEO_TIMEOUT_S = 900.0
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def video_server():
|
||||
"""Start a real video generation server for async video API tests."""
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--num-gpus",
|
||||
"1",
|
||||
"--boundary-ratio",
|
||||
"0.875",
|
||||
"--flow-shift",
|
||||
"5.0",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
def _video_api_url(server: OmniServer, suffix: str = "") -> str:
|
||||
return f"http://{server.host}:{server.port}/v1/videos{suffix}"
|
||||
|
||||
|
||||
def _multipart_fields(payload: dict[str, Any]) -> list[tuple[str, tuple[None, str]]]:
|
||||
return [(key, (None, str(value))) for key, value in payload.items() if value is not None]
|
||||
|
||||
|
||||
def _create_video_job(
|
||||
server: OmniServer,
|
||||
*,
|
||||
prompt: str | None = None,
|
||||
**overrides: Any,
|
||||
) -> requests.Response:
|
||||
payload: dict[str, Any] = {
|
||||
"prompt": prompt or f"video api e2e {uuid.uuid4().hex[:8]}",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"num_frames": 5,
|
||||
"fps": 8,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 1.0,
|
||||
"guidance_scale_2": 1.0,
|
||||
"boundary_ratio": 0.875,
|
||||
"flow_shift": 5.0,
|
||||
"seed": 42,
|
||||
}
|
||||
payload.update(overrides)
|
||||
return requests.post(
|
||||
_video_api_url(server),
|
||||
files=_multipart_fields(payload),
|
||||
timeout=VIDEO_TIMEOUT_S,
|
||||
)
|
||||
|
||||
|
||||
def _retrieve_video_job(server: OmniServer, video_id: str) -> requests.Response:
|
||||
return requests.get(_video_api_url(server, f"/{video_id}"), timeout=VIDEO_TIMEOUT_S)
|
||||
|
||||
|
||||
def _wait_for_video_status(server: OmniServer, video_id: str, expected_status: str) -> dict[str, Any]:
|
||||
deadline = time.time() + VIDEO_TIMEOUT_S
|
||||
last_payload: dict[str, Any] | None = None
|
||||
|
||||
while time.time() < deadline:
|
||||
response = _retrieve_video_job(server, video_id)
|
||||
assert response.status_code == 200, response.text
|
||||
last_payload = response.json()
|
||||
status = last_payload["status"]
|
||||
if status == expected_status:
|
||||
return last_payload
|
||||
if status == "failed":
|
||||
raise AssertionError(f"Video job {video_id} failed unexpectedly: {last_payload}")
|
||||
time.sleep(VIDEO_POLL_INTERVAL_S)
|
||||
|
||||
raise AssertionError(
|
||||
f"Timed out waiting for video job {video_id} to reach status={expected_status}. Last payload: {last_payload}"
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_video_missing(server: OmniServer, video_id: str) -> None:
|
||||
deadline = time.time() + 30.0
|
||||
last_status: int | None = None
|
||||
|
||||
while time.time() < deadline:
|
||||
response = _retrieve_video_job(server, video_id)
|
||||
last_status = response.status_code
|
||||
if response.status_code == 404:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
|
||||
raise AssertionError(f"Timed out waiting for video job {video_id} to disappear. Last status={last_status}")
|
||||
|
||||
|
||||
def _delete_video_job(server: OmniServer, video_id: str) -> requests.Response:
|
||||
return requests.delete(_video_api_url(server, f"/{video_id}"), timeout=VIDEO_TIMEOUT_S)
|
||||
|
||||
|
||||
def _delete_video_job_with_retry(server: OmniServer, video_id: str) -> requests.Response:
|
||||
deadline = time.time() + 30.0
|
||||
last_response: requests.Response | None = None
|
||||
|
||||
while time.time() < deadline:
|
||||
response = _delete_video_job(server, video_id)
|
||||
last_response = response
|
||||
if response.status_code != 409:
|
||||
return response
|
||||
time.sleep(1.0)
|
||||
|
||||
raise AssertionError(
|
||||
f"Timed out waiting to delete video job {video_id}. "
|
||||
f"Last response: {None if last_response is None else last_response.text}"
|
||||
)
|
||||
|
||||
|
||||
def _best_effort_delete(server: OmniServer, video_id: str) -> None:
|
||||
try:
|
||||
response = _delete_video_job_with_retry(server, video_id)
|
||||
if response.status_code not in (200, 404):
|
||||
print(f"Cleanup delete for {video_id} returned {response.status_code}: {response.text}")
|
||||
except Exception as exc:
|
||||
print(f"Cleanup delete for {video_id} failed: {exc}")
|
||||
|
||||
|
||||
def _assert_mp4_payload(content: bytes) -> None:
|
||||
assert len(content) > 32, f"Downloaded video payload is unexpectedly small: {len(content)} bytes"
|
||||
assert content[4:8] == b"ftyp", "Downloaded payload does not look like an MP4 file."
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_create_list_retrieve_download_video(video_server: OmniServer, tmp_path: Path) -> None:
|
||||
video_id: str | None = None
|
||||
try:
|
||||
create_response = _create_video_job(video_server, prompt="A small paper airplane gliding over a lake at dawn.")
|
||||
assert create_response.status_code == 200, create_response.text
|
||||
|
||||
created = create_response.json()
|
||||
video_id = created["id"]
|
||||
assert created["object"] == "video"
|
||||
assert created["status"] == "queued"
|
||||
assert created["model"] == MODEL
|
||||
|
||||
completed = _wait_for_video_status(video_server, video_id, "completed")
|
||||
assert completed["file_name"] is not None
|
||||
assert completed["completed_at"] is not None
|
||||
assert completed["progress"] == 100
|
||||
|
||||
retrieve_response = _retrieve_video_job(video_server, video_id)
|
||||
assert retrieve_response.status_code == 200, retrieve_response.text
|
||||
retrieved = retrieve_response.json()
|
||||
assert retrieved["id"] == video_id
|
||||
assert retrieved["status"] == "completed"
|
||||
|
||||
list_response = requests.get(_video_api_url(video_server), timeout=VIDEO_TIMEOUT_S)
|
||||
assert list_response.status_code == 200, list_response.text
|
||||
listed = list_response.json()
|
||||
assert listed["object"] == "list"
|
||||
assert any(item["id"] == video_id for item in listed["data"])
|
||||
|
||||
download_response = requests.get(
|
||||
_video_api_url(video_server, f"/{video_id}/content"),
|
||||
timeout=VIDEO_TIMEOUT_S,
|
||||
)
|
||||
assert download_response.status_code == 200, download_response.text
|
||||
assert download_response.headers["content-type"].startswith("video/mp4")
|
||||
assert completed["file_name"] in download_response.headers.get("content-disposition", "")
|
||||
_assert_mp4_payload(download_response.content)
|
||||
|
||||
output_path = tmp_path / completed["file_name"]
|
||||
output_path.write_bytes(download_response.content)
|
||||
assert output_path.stat().st_size == len(download_response.content)
|
||||
finally:
|
||||
if video_id is not None:
|
||||
_best_effort_delete(video_server, video_id)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_delete_video_while_queued_or_in_progress(video_server: OmniServer) -> None:
|
||||
create_response = _create_video_job(
|
||||
video_server,
|
||||
prompt="A toy boat sailing through fog.",
|
||||
num_frames=13,
|
||||
num_inference_steps=4,
|
||||
)
|
||||
assert create_response.status_code == 200, create_response.text
|
||||
|
||||
created = create_response.json()
|
||||
video_id = created["id"]
|
||||
assert created["status"] == "queued"
|
||||
|
||||
# Delete immediately so the job is still queued or just entering execution.
|
||||
delete_response = _delete_video_job_with_retry(video_server, video_id)
|
||||
assert delete_response.status_code == 200, delete_response.text
|
||||
deleted = delete_response.json()
|
||||
assert deleted["id"] == video_id
|
||||
assert deleted["deleted"] is True
|
||||
assert deleted["object"] == "video.deleted"
|
||||
|
||||
_wait_for_video_missing(video_server, video_id)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
def test_delete_video_after_processing(video_server: OmniServer) -> None:
|
||||
create_response = _create_video_job(
|
||||
video_server,
|
||||
prompt="A lantern drifting above a quiet forest.",
|
||||
seed=43,
|
||||
)
|
||||
assert create_response.status_code == 200, create_response.text
|
||||
|
||||
video_id = create_response.json()["id"]
|
||||
completed = _wait_for_video_status(video_server, video_id, "completed")
|
||||
assert completed["file_name"] is not None
|
||||
|
||||
delete_response = _delete_video_job_with_retry(video_server, video_id)
|
||||
assert delete_response.status_code == 200, delete_response.text
|
||||
deleted = delete_response.json()
|
||||
assert deleted["id"] == video_id
|
||||
assert deleted["deleted"] is True
|
||||
assert deleted["object"] == "video.deleted"
|
||||
|
||||
_wait_for_video_missing(video_server, video_id)
|
||||
|
||||
download_response = requests.get(
|
||||
_video_api_url(video_server, f"/{video_id}/content"),
|
||||
timeout=VIDEO_TIMEOUT_S,
|
||||
)
|
||||
assert download_response.status_code == 404, download_response.text
|
||||
@@ -12,7 +12,6 @@ import os
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
@@ -26,36 +25,15 @@ EXTRA_ARGS = ["--trust-remote-code", "--enforce-eager", "--disable-log-stats"]
|
||||
TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, server_args=EXTRA_ARGS)]
|
||||
|
||||
|
||||
def make_speech_request(
|
||||
host: str,
|
||||
port: int,
|
||||
text: str,
|
||||
voice: str = "casual_female",
|
||||
language: str = "English",
|
||||
timeout: float = 120.0,
|
||||
) -> httpx.Response:
|
||||
"""Make a request to the /v1/audio/speech endpoint."""
|
||||
url = f"http://{host}:{port}/v1/audio/speech"
|
||||
payload = {
|
||||
"input": text,
|
||||
"voice": voice,
|
||||
"language": language,
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
return client.post(url, json=payload)
|
||||
|
||||
|
||||
def verify_wav_audio(content: bytes) -> bool:
|
||||
"""Verify that content is valid WAV audio data."""
|
||||
# WAV files start with "RIFF" header
|
||||
if len(content) < 44: # Minimum WAV header size
|
||||
if len(content) < 44:
|
||||
return False
|
||||
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
|
||||
|
||||
|
||||
# Minimum expected audio size for a short sentence (~1 second of 24kHz 16-bit mono WAV)
|
||||
MIN_AUDIO_BYTES = 10000
|
||||
_MIN_AUDIO_BYTES_BASIC = 10000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||
@@ -63,151 +41,71 @@ class TestVoxtralTTSFixedVoice:
|
||||
"""E2E tests for Voxtral TTS model."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_speech_english_basic(self, omni_server) -> None:
|
||||
def test_speech_english_basic(self, omni_server, openai_client) -> None:
|
||||
"""Test basic English TTS generation."""
|
||||
response = make_speech_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="Hello, how are you?",
|
||||
voice="casual_female",
|
||||
language="English",
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
assert response.headers.get("content-type") == "audio/wav"
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
|
||||
assert len(response.content) > MIN_AUDIO_BYTES, (
|
||||
f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
|
||||
openai_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "casual_female",
|
||||
"language": "English",
|
||||
"response_format": "wav",
|
||||
"timeout": 120.0,
|
||||
"min_audio_bytes": _MIN_AUDIO_BYTES_BASIC,
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_speech_english_streaming(self, omni_server) -> None:
|
||||
"""Test basic streaming English TTS generation."""
|
||||
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "casual_female",
|
||||
"language": "English",
|
||||
"stream": True,
|
||||
"response_format": "pcm",
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
with client.stream("POST", url, json=payload) as response:
|
||||
assert response.status_code == 200
|
||||
assert response.headers.get("content-type") == "audio/pcm"
|
||||
total = sum(len(c) for c in response.iter_bytes())
|
||||
|
||||
assert total > MIN_AUDIO_BYTES, (
|
||||
f"Streamed audio too small ({total} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
|
||||
def test_speech_english_streaming(self, omni_server, openai_client) -> None:
|
||||
"""Test basic streaming English TTS generation (PCM via streaming API)."""
|
||||
openai_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "casual_female",
|
||||
"language": "English",
|
||||
"stream": True,
|
||||
"response_format": "pcm",
|
||||
"timeout": 120.0,
|
||||
"min_audio_bytes": _MIN_AUDIO_BYTES_BASIC,
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_speech_different_voices(self, omni_server) -> None:
|
||||
def test_speech_different_voices(self, omni_server, openai_client) -> None:
|
||||
"""Test TTS with different voice presets."""
|
||||
voices = ["casual_female", "neutral_male"]
|
||||
for voice in voices:
|
||||
response = make_speech_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="Testing voice selection.",
|
||||
voice=voice,
|
||||
language="English",
|
||||
openai_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "Testing voice selection.",
|
||||
"voice": voice,
|
||||
"language": "English",
|
||||
"response_format": "wav",
|
||||
"timeout": 120.0,
|
||||
}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, f"Request failed for voice {voice}: {response.text}"
|
||||
assert verify_wav_audio(response.content), f"Invalid WAV for voice {voice}"
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_speech_invalid_voice_rejected(self, omni_server) -> None:
|
||||
"""Request with a non-existent voice should return an error response."""
|
||||
response = make_speech_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="This voice does not exist.",
|
||||
voice="nonexistent_voice_xyz",
|
||||
language="English",
|
||||
def test_speech_invalid_voice_rejected(self, omni_server, openai_client) -> None:
|
||||
"""Request with a non-existent voice should be rejected (HTTP 200 with JSON or 4xx, depending on server)."""
|
||||
openai_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "This voice does not exist.",
|
||||
"voice": "nonexistent_voice_xyz",
|
||||
"language": "English",
|
||||
"response_format": "wav",
|
||||
"timeout": 120.0,
|
||||
"err_message": "Invalid speaker",
|
||||
}
|
||||
)
|
||||
|
||||
# vLLM returns ErrorResponse as JSON with HTTP 200;
|
||||
# verify the body is a JSON error, not valid WAV audio.
|
||||
data = response.json()
|
||||
assert "error" in data or "message" in data, f"Expected error response for invalid voice, got: {data}"
|
||||
assert not verify_wav_audio(response.content), "Should not return valid audio for invalid voice"
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_speech_binary_response_not_utf8_error(self, omni_server) -> None:
|
||||
"""
|
||||
Regression test: Verify binary audio is returned, not UTF-8 error.
|
||||
|
||||
This test ensures the multimodal_output property correctly retrieves
|
||||
audio from completion outputs, preventing the "TTS model did not
|
||||
produce audio output" error.
|
||||
"""
|
||||
response = make_speech_request(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
text="This should return binary audio, not a JSON error.",
|
||||
voice="casual_female",
|
||||
language="English",
|
||||
)
|
||||
|
||||
# Should NOT be a JSON error response
|
||||
assert response.status_code == 200, f"Request failed: {response.text}"
|
||||
|
||||
# Verify it's binary audio, not JSON
|
||||
try:
|
||||
# If this succeeds and starts with {"error", it's a bug
|
||||
text = response.content.decode("utf-8")
|
||||
assert not text.startswith('{"error"'), f"Got error response instead of audio: {text}"
|
||||
except UnicodeDecodeError:
|
||||
# This is expected - binary audio can't be decoded as UTF-8
|
||||
pass
|
||||
|
||||
assert verify_wav_audio(response.content), "Response is not valid WAV audio"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
|
||||
class TestVoxtralTTSAPIEndpoints:
|
||||
"""Test API endpoint functionality."""
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_list_voices_endpoint(self, omni_server) -> None:
|
||||
"""Test the /v1/audio/voices endpoint returns available voices."""
|
||||
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/voices"
|
||||
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
response = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "voices" in data
|
||||
assert isinstance(data["voices"], list)
|
||||
assert len(data["voices"]) > 0
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_models_endpoint(self, omni_server) -> None:
|
||||
"""Test the /v1/models endpoint returns loaded model."""
|
||||
url = f"http://{omni_server.host}:{omni_server.port}/v1/models"
|
||||
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
response = client.get(url)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "data" in data
|
||||
assert len(data["data"]) > 0
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Online serving smoke for ``Wan-AI/Wan2.2-T2V-A14B-Diffusers`` (text-to-video via ``/v1/videos``).
|
||||
|
||||
Uses a single ``default`` ``OmniServerParams`` row via ``_get_diffusion_feature_cases`` (no extra
|
||||
``server_args``). Multi-variant / parallel coverage lives in ``test_wan22_expansion.py`` (L4).
|
||||
|
||||
From ``tests/``::
|
||||
|
||||
pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model
|
||||
pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "advanced_model and diffusion" --run-level=advanced_model
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||
PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
|
||||
NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
|
||||
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(model=model),
|
||||
id="default",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
|
||||
def test_text_to_video_001(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
"""Default Wan2.2 T2V smoke: async ``/v1/videos`` job completes and returns video bytes."""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"form_data": {
|
||||
"prompt": PROMPT,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_frames": 8,
|
||||
"fps": 8,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
openai_client.send_video_diffusion_request(request_config)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
Tests of common diffusion feature combinations in online serving mode
|
||||
for Z-Image.
|
||||
|
||||
Coverage is intentionally limited to the minimal 4xL4 cases that
|
||||
exercise Z-Image's supported parallel feature combinations:
|
||||
- CacheDiT + FP8 + Ring=2 + TP=2
|
||||
- TeaCache + FP8 + Ulysses=2 + Ring=2
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
MODEL = "Tongyi-MAI/Z-Image-Turbo"
|
||||
PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard."
|
||||
|
||||
FOUR_CARD_MARKS = hardware_marks(res={"cuda": "L4"}, num_cards=4)
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases():
|
||||
"""Return the common Z-Image feature combinations that fit L4 CI."""
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
),
|
||||
id="default",
|
||||
marks=FOUR_CARD_MARKS,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_basic_001(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
):
|
||||
"""Exercise supported Z-Image diffusion features in minimal CI cases."""
|
||||
messages = dummy_messages_from_mix_data(content_text=PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(),
|
||||
indirect=True,
|
||||
)
|
||||
def test_different_sizes_001(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
):
|
||||
"""Exercise supported Z-Image diffusion features in minimal CI cases."""
|
||||
messages = dummy_messages_from_mix_data(content_text=PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": [512, 512],
|
||||
"width": [512, 768],
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
@@ -1,79 +0,0 @@
|
||||
# stage config for running dynin_omni with a 3-stage architecture.
|
||||
# this config is intended for e2e smoke tests.
|
||||
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2text
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.5
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 4096
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2image
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 4096
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2text_to_token2image
|
||||
final_output: true
|
||||
final_output_type: image
|
||||
|
||||
- stage_id: 2
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: token2audio
|
||||
model_arch: DyninOmniForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 4096
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.dynin_omni.token2image_to_token2audio
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
- from: 1
|
||||
to: 2
|
||||
Reference in new issue
Block a user