Merge branch 'main' into dev/vllm-align
This commit is contained in:
commit
d3db476f2d
144 files changed
+17247
-1024
No files matched your search
@@ -6,7 +6,7 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor, hardware_test
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -23,9 +23,9 @@ models = ["riverclouds/qwen_image_random"]
|
||||
def inference(model_name: str, offload: bool = True):
|
||||
gc.collect()
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = torch.cuda.current_device()
|
||||
torch.cuda.reset_peak_memory_stats(device=device_index)
|
||||
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
device_index = current_omni_platform.current_device()
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
m = Omni(
|
||||
model=model_name,
|
||||
@@ -33,7 +33,7 @@ def inference(model_name: str, offload: bool = True):
|
||||
# cache_backend="cache_dit",
|
||||
enable_cpu_offload=offload,
|
||||
)
|
||||
torch.cuda.reset_peak_memory_stats(device=device_index)
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
height = 256
|
||||
width = 256
|
||||
|
||||
@@ -44,7 +44,7 @@ def inference(model_name: str, offload: bool = True):
|
||||
width=width,
|
||||
num_inference_steps=9,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
),
|
||||
)
|
||||
peak = monitor.peak_used_mb
|
||||
@@ -70,6 +70,11 @@ def test_cpu_offload_diffusion_model(model_name: str):
|
||||
pytest.fail("Inference failed")
|
||||
print(f"Offload peak memory: {offload_peak_memory} MB")
|
||||
print(f"No offload peak memory: {no_offload_peak_memory} MB")
|
||||
assert offload_peak_memory + 2500 < no_offload_peak_memory, (
|
||||
f"Offload peak memory {offload_peak_memory} MB should be less than no offload peak memory {no_offload_peak_memory} MB"
|
||||
# Set platform-specific VRAM saving thresholds to account
|
||||
# for varying runtime memory overhead and fragmentation between CUDA and ROCm.
|
||||
is_rocm = torch.version.hip is not None
|
||||
threshold = 2500 if not is_rocm else 2100
|
||||
assert offload_peak_memory + threshold < no_offload_peak_memory, (
|
||||
f"Offload peak memory {offload_peak_memory} MB should be less than "
|
||||
f"no offload peak memory {no_offload_peak_memory} MB by {threshold} MB"
|
||||
)
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor
|
||||
from tests.utils import DeviceMemoryMonitor
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -28,11 +28,9 @@ def run_inference(
|
||||
layerwise_offload: bool = False,
|
||||
num_inference_steps: int = 3,
|
||||
) -> float:
|
||||
# For now, only support on GPU, so apply torch.cuda operations here
|
||||
# NPU / ROCm platforms are expected to be detected and skipped this test function
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = torch.cuda.current_device()
|
||||
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
device_index = current_omni_platform.current_device()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
|
||||
m = Omni(
|
||||
@@ -44,7 +42,7 @@ def run_inference(
|
||||
flow_shift=5.0,
|
||||
)
|
||||
|
||||
torch.cuda.reset_peak_memory_stats(device=device_index)
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
|
||||
# Refer to tests/e2e/offline_inference/test_t2v_model.py
|
||||
# Use minimal settings for testing
|
||||
@@ -57,7 +55,7 @@ def run_inference(
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
|
||||
guidance_scale=1.0,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
|
||||
@@ -146,9 +146,6 @@ def _run_inference(
|
||||
# - warmup: whether to run warmup for this SP config
|
||||
# - is_perf_test: whether this is a performance test (show speedup metrics)
|
||||
SP_CONFIGS_L2 = [
|
||||
# Ulysses-2 - performance test
|
||||
(2, 1, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True),
|
||||
(1, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, True, True), # Ring-2 - performance test
|
||||
# Hybrid - correctness only
|
||||
(2, 2, DEFAULT_HEIGHT, DEFAULT_WIDTH, False, False),
|
||||
]
|
||||
|
||||
@@ -22,7 +22,7 @@ import torch
|
||||
from PIL import Image
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor, hardware_test
|
||||
from tests.utils import DeviceMemoryMonitor, hardware_test
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
@@ -54,12 +54,13 @@ def _pil_to_float_rgb_tensor(img: Image.Image) -> torch.Tensor:
|
||||
|
||||
|
||||
def _diff_metrics(a: Image.Image, b: Image.Image) -> tuple[float, float]:
|
||||
"""Return (mean_abs_diff, max_abs_diff) over RGB pixels in [0, 1]."""
|
||||
"""Return (mean_abs_diff, p99_abs_diff) over RGB pixels in [0, 1]."""
|
||||
ta = _pil_to_float_rgb_tensor(a)
|
||||
tb = _pil_to_float_rgb_tensor(b)
|
||||
assert ta.shape == tb.shape, f"Image shapes differ: {ta.shape} vs {tb.shape}"
|
||||
abs_diff = torch.abs(ta - tb)
|
||||
return abs_diff.mean().item(), abs_diff.max().item()
|
||||
p99_abs_diff = torch.quantile(abs_diff.flatten(), 0.99).item()
|
||||
return abs_diff.mean().item(), p99_abs_diff
|
||||
|
||||
|
||||
def _extract_single_image(outputs) -> Image.Image:
|
||||
@@ -93,9 +94,9 @@ def _run_zimage_generate(
|
||||
if num_requests < 2:
|
||||
raise ValueError("num_requests must be >= 2 (1 warmup + >=1 timed)")
|
||||
|
||||
torch.cuda.empty_cache()
|
||||
device_index = torch.cuda.current_device()
|
||||
monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
current_omni_platform.empty_cache()
|
||||
device_index = current_omni_platform.current_device()
|
||||
monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
|
||||
monitor.start()
|
||||
m = Omni(
|
||||
model=_get_zimage_model(),
|
||||
@@ -161,8 +162,8 @@ def _run_zimage_generate(
|
||||
def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
pytest.skip("Z-Image TP e2e test is only supported on CUDA for now.")
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
|
||||
pytest.skip("Z-Image TP=2 requires >= 2 CUDA devices.")
|
||||
if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
|
||||
pytest.skip("Z-Image TP=2 requires >= 2 devices.")
|
||||
|
||||
enforce_eager = False
|
||||
|
||||
@@ -196,18 +197,18 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
assert tp1_img.width == width and tp1_img.height == height
|
||||
assert tp2_img.width == width and tp2_img.height == height
|
||||
|
||||
mean_abs_diff, max_abs_diff = _diff_metrics(tp1_img, tp2_img)
|
||||
mean_abs_diff, p99_abs_diff = _diff_metrics(tp1_img, tp2_img)
|
||||
mean_threshold = 3e-2
|
||||
max_threshold = 5e-1
|
||||
p99_threshold = 2.5e-1
|
||||
print(
|
||||
"Z-Image TP image diff stats (TP=1 vs TP=2): "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
|
||||
f"tp1_img={tp1_path}, tp2_img={tp2_path}"
|
||||
)
|
||||
assert mean_abs_diff <= mean_threshold and max_abs_diff <= max_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e})"
|
||||
assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
|
||||
)
|
||||
|
||||
print(f"Z-Image TP perf (lower is better): tp1_time_s={tp1_time_s:.6f}, tp2_time_s={tp2_time_s:.6f}")
|
||||
@@ -219,59 +220,59 @@ def test_zimage_tensor_parallel_tp2(tmp_path: Path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
|
||||
if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA for now.")
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
|
||||
pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 CUDA devices.")
|
||||
# @pytest.mark.integration
|
||||
# def test_zimage_vae_patch_parallel_tp2(tmp_path: Path):
|
||||
# if current_omni_platform.is_npu() or current_omni_platform.is_rocm():
|
||||
# pytest.skip("Z-Image VAE patch parallel e2e test is only supported on CUDA for now.")
|
||||
# if not current_omni_platform.is_available() or current_omni_platform.device_count() < 2:
|
||||
# pytest.skip("Z-Image VAE patch parallel TP=2 requires >= 2 devices.")
|
||||
|
||||
enforce_eager = False
|
||||
# enforce_eager = False
|
||||
|
||||
# Use a larger image to ensure there are multiple VAE tiles.
|
||||
height = 1152
|
||||
width = 1152
|
||||
num_inference_steps = 2
|
||||
seed = 42
|
||||
# # Use a larger image to ensure there are multiple VAE tiles.
|
||||
# height = 1152
|
||||
# width = 1152
|
||||
# num_inference_steps = 2
|
||||
# seed = 42
|
||||
|
||||
baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=True,
|
||||
vae_patch_parallel_size=1,
|
||||
num_requests=2,
|
||||
)
|
||||
pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
|
||||
tp_size=2,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
seed=seed,
|
||||
enforce_eager=enforce_eager,
|
||||
vae_use_tiling=True,
|
||||
vae_patch_parallel_size=2,
|
||||
num_requests=2,
|
||||
)
|
||||
# baseline_img, _baseline_time_s, _baseline_peak_mem = _run_zimage_generate(
|
||||
# tp_size=2,
|
||||
# height=height,
|
||||
# width=width,
|
||||
# num_inference_steps=num_inference_steps,
|
||||
# seed=seed,
|
||||
# enforce_eager=enforce_eager,
|
||||
# vae_use_tiling=True,
|
||||
# vae_patch_parallel_size=1,
|
||||
# num_requests=2,
|
||||
# )
|
||||
# pp2_img, _pp2_time_s, _pp2_peak_mem = _run_zimage_generate(
|
||||
# tp_size=2,
|
||||
# height=height,
|
||||
# width=width,
|
||||
# num_inference_steps=num_inference_steps,
|
||||
# seed=seed,
|
||||
# enforce_eager=enforce_eager,
|
||||
# vae_use_tiling=True,
|
||||
# vae_patch_parallel_size=2,
|
||||
# num_requests=2,
|
||||
# )
|
||||
|
||||
baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
|
||||
pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
|
||||
baseline_img.save(baseline_path)
|
||||
pp2_img.save(pp2_path)
|
||||
# baseline_path = tmp_path / "zimage_tp2_vae_pp1.png"
|
||||
# pp2_path = tmp_path / "zimage_tp2_vae_pp2.png"
|
||||
# baseline_img.save(baseline_path)
|
||||
# pp2_img.save(pp2_path)
|
||||
|
||||
mean_abs_diff, max_abs_diff = _diff_metrics(baseline_img, pp2_img)
|
||||
mean_threshold = 5e-3
|
||||
max_threshold = 1e-1
|
||||
print(
|
||||
"Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
|
||||
f"mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e}; "
|
||||
f"thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e}; "
|
||||
f"pp1_img={baseline_path}, pp2_img={pp2_path}"
|
||||
)
|
||||
assert mean_abs_diff <= mean_threshold and max_abs_diff <= max_threshold, (
|
||||
f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, max_abs_diff={max_abs_diff:.6e} "
|
||||
f"(thresholds: mean<={mean_threshold:.6e}, max<={max_threshold:.6e})"
|
||||
)
|
||||
# mean_abs_diff, p99_abs_diff = _diff_metrics(baseline_img, pp2_img)
|
||||
# mean_threshold = 5e-3
|
||||
# p99_threshold = 1e-1
|
||||
# print(
|
||||
# "Z-Image VAE patch parallel image diff stats (TP=2, pp=1 vs pp=2): "
|
||||
# f"mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e}; "
|
||||
# f"thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e}; "
|
||||
# f"pp1_img={baseline_path}, pp2_img={pp2_path}"
|
||||
# )
|
||||
# assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, (
|
||||
# f"Image diff exceeded threshold: mean_abs_diff={mean_abs_diff:.6e}, p99_abs_diff={p99_abs_diff:.6e} "
|
||||
# f"(thresholds: mean<={mean_threshold:.6e}, p99<={p99_threshold:.6e})"
|
||||
# )
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServerParams,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
@@ -51,7 +52,9 @@ else:
|
||||
stage_configs = [get_chunk_config()]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
@@ -115,12 +118,11 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"stream": True,
|
||||
"key_words": {
|
||||
"audio": ["test"],
|
||||
"image": ["square", "quadrate"],
|
||||
},
|
||||
}
|
||||
|
||||
# Test single completion
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -146,4 +148,4 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -12,6 +12,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServerParams,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_audio,
|
||||
generate_synthetic_image,
|
||||
@@ -50,7 +51,9 @@ def get_chunk_config(default_path):
|
||||
default_path = str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
|
||||
stage_configs = [default_path, get_chunk_config(default_path)]
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
test_params = [
|
||||
OmniServerParams(model=model, stage_config_path=stage_config) for model in models for stage_config in stage_configs
|
||||
]
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
@@ -106,7 +109,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -128,7 +131,7 @@ def test_text_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"text": ["beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -153,7 +156,7 @@ def test_image_to_text_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -177,7 +180,7 @@ def test_image_to_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -201,7 +204,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -225,7 +228,7 @@ def test_video_to_text_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -249,7 +252,7 @@ def test_video_to_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -273,7 +276,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -298,7 +301,7 @@ def test_text_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"audio": AUDIO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -324,7 +327,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"image": IMAGE_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@@ -351,7 +354,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
}
|
||||
|
||||
openai_client.send_request(request_config)
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
|
||||
@@ -383,4 +386,4 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"stream": True,
|
||||
"key_words": {"audio": AUDIO_KEY, "image": IMAGE_KEY, "video": VIDEO_KEY},
|
||||
}
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E online tests for Qwen3-TTS WebSocket streaming speech.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.conftest import OmniServer
|
||||
from tests.utils import hardware_test
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
|
||||
|
||||
|
||||
def get_stage_config() -> str:
|
||||
return str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server():
|
||||
stage_config_path = get_stage_config()
|
||||
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--stage-configs-path",
|
||||
stage_config_path,
|
||||
"--stage-init-timeout",
|
||||
"120",
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--disable-log-stats",
|
||||
],
|
||||
env_dict={"VLLM_DISABLE_COMPILE_CACHE": "1"},
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
async def _run_ws_session(host: str, port: int) -> dict:
|
||||
uri = f"ws://{host}:{port}/v1/audio/speech/stream"
|
||||
starts: list[dict] = []
|
||||
dones: list[dict] = []
|
||||
chunk_lengths: dict[int, list[int]] = {}
|
||||
session_done: dict | None = None
|
||||
|
||||
async with websockets.connect(uri, max_size=None) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": MODEL,
|
||||
"voice": "vivian",
|
||||
"language": "English",
|
||||
"response_format": "pcm",
|
||||
"stream_audio": True,
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": (
|
||||
"Hello, this is a websocket streaming test for Qwen three TTS, "
|
||||
"and this sentence is intentionally long enough to produce audio chunks. "
|
||||
"This is the second sentence."
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
|
||||
while True:
|
||||
message = await asyncio.wait_for(ws.recv(), timeout=180)
|
||||
if isinstance(message, bytes):
|
||||
if not starts:
|
||||
raise AssertionError("Received audio bytes before audio.start")
|
||||
sentence_index = starts[-1]["sentence_index"]
|
||||
chunk_lengths.setdefault(sentence_index, []).append(len(message))
|
||||
continue
|
||||
|
||||
payload = json.loads(message)
|
||||
msg_type = payload.get("type")
|
||||
if msg_type == "audio.start":
|
||||
starts.append(payload)
|
||||
chunk_lengths.setdefault(payload["sentence_index"], [])
|
||||
elif msg_type == "audio.done":
|
||||
dones.append(payload)
|
||||
elif msg_type == "session.done":
|
||||
session_done = payload
|
||||
break
|
||||
elif msg_type == "error":
|
||||
raise AssertionError(f"WebSocket error: {payload['message']}")
|
||||
else:
|
||||
raise AssertionError(f"Unexpected WebSocket message: {payload}")
|
||||
|
||||
return {
|
||||
"starts": starts,
|
||||
"dones": dones,
|
||||
"chunk_lengths": chunk_lengths,
|
||||
"session_done": session_done,
|
||||
}
|
||||
|
||||
|
||||
class TestQwen3TTSWebSocket:
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=4)
|
||||
def test_streaming_pcm_output(self, omni_server) -> None:
|
||||
result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port))
|
||||
|
||||
starts = result["starts"]
|
||||
dones = result["dones"]
|
||||
chunk_lengths = result["chunk_lengths"]
|
||||
session_done = result["session_done"]
|
||||
|
||||
assert session_done is not None
|
||||
assert session_done["total_sentences"] == 2
|
||||
assert len(starts) == 2
|
||||
assert len(dones) == 2
|
||||
|
||||
for idx, start in enumerate(starts):
|
||||
assert start["type"] == "audio.start"
|
||||
assert start["sentence_index"] == idx
|
||||
assert start["format"] == "pcm"
|
||||
assert start["sample_rate"] == 24000
|
||||
assert start["sentence_text"]
|
||||
|
||||
for done in dones:
|
||||
sentence_index = done["sentence_index"]
|
||||
total_bytes = done["total_bytes"]
|
||||
assert done["error"] is False
|
||||
assert total_bytes > 0
|
||||
assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}"
|
||||
assert sum(chunk_lengths[sentence_index]) == total_bytes
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Comprehensive tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following models:
|
||||
- Qwen-Image-Edit: single image input
|
||||
- Qwen-Image-Edit-2509: two image inputs
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
OmniServer,
|
||||
OmniServerParams,
|
||||
OpenAIClientHandler,
|
||||
dummy_messages_from_mix_data,
|
||||
generate_synthetic_image,
|
||||
)
|
||||
from tests.utils import hardware_marks
|
||||
|
||||
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
"Transform the second image into a Vincent van Gogh style painting. "
|
||||
"Then juxtapose the two transformed images into a single artwork for visual contrast."
|
||||
)
|
||||
NEGATIVE_PROMPT = "blurry, low quality, modern, geometrist"
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
|
||||
|
||||
|
||||
# This test file targets two models, so I write a helper function.
|
||||
# If a similar test only involves one model, one can just define a global list variable.
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
return [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together.
|
||||
"--enable-layerwise-offload",
|
||||
],
|
||||
),
|
||||
id="single_card_001",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"--ulysses-degree",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_001",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"--ring",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_002",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"tea_cache",
|
||||
"--cfg-parallel-size",
|
||||
"2",
|
||||
],
|
||||
),
|
||||
id="parallel_003",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--vae-patch-parallel-size",
|
||||
"2",
|
||||
"--vae-use-tiling",
|
||||
],
|
||||
),
|
||||
id="parallel_004",
|
||||
marks=PARALLEL_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test all diffusion features with Qwen-Image-Edit in regular end-user scenarios."""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=EDIT_PROMPT)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit_2509(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test all diffusion features with Qwen-Image-Edit-2509 in regular end-user scenarios."""
|
||||
image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(
|
||||
image_data_url=[image_data_url_1, image_data_url_2], content_text=MULTI_EDIT_PROMPT
|
||||
)
|
||||
|
||||
# CFG parallel is only activated when a negative prompt and true_cfg_scale > 1.0 are both present
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 2,
|
||||
"negative_prompt": NEGATIVE_PROMPT,
|
||||
"true_cfg_scale": 4.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
Reference in new issue
Block a user