[Test] Cleanup for helpers (#6523)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
wangyuandamy-why-3459 authored and GitHub committed 2026-08-25 09:10:17 +08:00
1 parent 8360346a66
commit 4af0ddc1c2
140 files changed
+4121 -4194

No files matched your search

+25 -17
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import json
import os
import re
@@ -14,7 +17,7 @@ from hashlib import sha1
from io import BytesIO
from mimetypes import guess_type
from pathlib import Path
from typing import Any
from typing import IO, Any
from urllib.parse import urlparse
import numpy as np
@@ -35,11 +38,14 @@ class IdentityFtfy:
def apply_ftfy_mock(*, wan_i2v_module: Any | None = None) -> None:
"""Mock ftfy library for test cases in the main process"""
if wan_i2v_module is None:
from diffusers.pipelines.wan import pipeline_wan_i2v as wan_i2v_module
module: Any = wan_i2v_module
if module is None:
from diffusers.pipelines.wan import pipeline_wan_i2v as wan_pipeline
if not hasattr(wan_i2v_module, "ftfy"):
wan_i2v_module.ftfy = IdentityFtfy()
module = wan_pipeline
if not hasattr(module, "ftfy"):
module.ftfy = IdentityFtfy()
print("ftfy (text encoding sanitizer) is not installed. Using mock ftfy implementation (identity function)")
else:
print("ftfy (text encoding sanitizer) is installed. Using actual ftfy implementation.")
@@ -118,6 +124,7 @@ def probe_binary(binary: str, *, test_name: str = "video similarity e2e test") -
resolved = shutil.which(binary)
if resolved is None:
pytest.skip(f"{binary} is required for {test_name}.")
assert resolved is not None
return resolved
@@ -235,7 +242,7 @@ def video_artifact_dir(result_root: Path, source: str) -> Path:
def send_video_request_with_timeout(
openai_client,
online_client,
request_config: dict[str, Any],
*,
timeout_seconds: int,
@@ -266,7 +273,7 @@ def send_video_request_with_timeout(
normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference})
start_time = time.perf_counter()
create_url = openai_client._build_url("/v1/videos")
create_url = online_client._build_url("/v1/videos")
response = requests.post(
create_url,
data=normalized_form_data,
@@ -280,8 +287,8 @@ def send_video_request_with_timeout(
raise requests.HTTPError(f"{exc}; response body: {response.text}", response=response) from exc
job_data = response.json()
video_id = job_data["id"]
openai_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds)
video_content = openai_client._download_video_content(video_id)
online_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds)
video_content = online_client._download_video_content(video_id)
print(f"online_video_e2e_latency_s={time.perf_counter() - start_time:.3f}")
return video_content
@@ -571,11 +578,11 @@ class SemanticSimilarityScorer:
raise ImportError("FlagEmbedding not installed. Install with: pip install FlagEmbedding")
return self._text_model
def text_similarity(self, text1: str, text2: str) -> float:
def text_similarity(self, text1: str, text2: str) -> dict[str, float | int]:
"""
Compute semantic similarity between two texts using BGE-M3
"""
results = {}
results: dict[str, float | int] = {}
# 1. Text prefix matching (maximum matching character length)
match_count = 0
for char_vllm, char_ref in zip(text1, text2):
@@ -653,7 +660,7 @@ def dreamzero_cfg_parallel_size() -> int:
def _dreamzero_parity_vllm_env() -> dict[str, str]:
from tests.helpers.env import pick_least_used_device_indices
from tests.helpers.clean import pick_least_used_device_indices
from tests.helpers.runtime import get_open_port
parallel_size = dreamzero_cfg_parallel_size()
@@ -683,8 +690,9 @@ def dreamzero_parity_vllm_params(model: str | None = None):
]
if parallel_size > 1:
server_args.extend(["--cfg-parallel-size", str(parallel_size)])
resolved_model = model or os.environ.get("VLLM_DREAMZERO_MODEL") or _DREAMZERO_DEFAULT_MODEL
return OmniServerParams(
model=model or os.environ.get("VLLM_DREAMZERO_MODEL", _DREAMZERO_DEFAULT_MODEL),
model=resolved_model,
stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"),
server_args=server_args,
env_dict=_dreamzero_parity_vllm_env(),
@@ -725,9 +733,9 @@ def dreamzero_parity_vllm_client(
def _dreamzero_create_openpi_vllm_client(host: str, port: int) -> Any:
from openpi_client import msgpack_numpy
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
_, websocket_client_policy_cls = load_dreamzero_upstream_modules()
class _OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
class _OpenPIWebsocketClientPolicy(websocket_client_policy_cls): # type: ignore[misc, valid-type]
def __init__(
self,
host: str = "127.0.0.1",
@@ -869,7 +877,7 @@ class _DreamZeroParityServer(ABC):
self.port = get_open_port() if port is None else port
self.ready_timeout_s = ready_timeout_s
self.proc: subprocess.Popen[str] | None = None
self._log_file = None
self._log_file: IO[str] | None = None
@abstractmethod
def _build_argv(self) -> list[str]:
@@ -932,7 +940,7 @@ class DreamZeroUpstreamServer(_DreamZeroParityServer):
]
def _build_env(self) -> dict[str, str]:
from tests.helpers.env import pick_least_used_device_indices
from tests.helpers.clean import pick_least_used_device_indices
dreamzero_repo, _ = require_dreamzero_parity_env()
env = os.environ.copy()
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import os
@@ -149,7 +152,7 @@ def _generate_offline_video(*, image_source: str) -> Path:
def _generate_online_video(
*,
omni_server,
openai_client,
online_client,
image_source: str,
timeout_seconds: int,
) -> Path:
@@ -171,7 +174,7 @@ def _generate_online_video(
"image_reference": build_online_image_reference(online_image_source),
}
online_video_bytes = send_video_request_with_timeout(
openai_client,
online_client,
request_config,
timeout_seconds=timeout_seconds,
)
@@ -204,7 +207,7 @@ def test_hunyuanvideo15_i2v_diffusers_offline_generates_video(
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_hunyuanvideo15_i2v_online_serving_generates_video(
omni_server,
openai_client,
online_client,
hunyuanvideo15_i2v_image_source: str | None,
hunyuanvideo15_online_timeout_seconds: int,
) -> None:
@@ -216,7 +219,7 @@ def test_hunyuanvideo15_i2v_online_serving_generates_video(
validate_image_source(image_source)
online_path = _generate_online_video(
omni_server=omni_server,
openai_client=openai_client,
online_client=online_client,
image_source=image_source,
timeout_seconds=hunyuanvideo15_online_timeout_seconds,
)
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import os
@@ -115,7 +118,7 @@ def _generate_offline_video() -> Path:
return offline_path
def _generate_online_video(*, omni_server, openai_client, timeout_seconds: int) -> Path:
def _generate_online_video(*, omni_server, online_client, timeout_seconds: int) -> Path:
online_path, _ = _artifact_paths()
request_config = {
"model": omni_server.model,
@@ -132,7 +135,7 @@ def _generate_online_video(*, omni_server, openai_client, timeout_seconds: int)
},
}
online_video_bytes = send_video_request_with_timeout(
openai_client,
online_client,
request_config,
timeout_seconds=timeout_seconds,
)
@@ -161,7 +164,7 @@ def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None:
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_hunyuanvideo15_t2v_online_serving_generates_video(
omni_server,
openai_client,
online_client,
hunyuanvideo15_online_timeout_seconds: int,
) -> None:
if not torch.cuda.is_available():
@@ -170,7 +173,7 @@ def test_hunyuanvideo15_t2v_online_serving_generates_video(
probe_binary("ffprobe")
online_path = _generate_online_video(
omni_server=omni_server,
openai_client=openai_client,
online_client=online_client,
timeout_seconds=hunyuanvideo15_online_timeout_seconds,
)
assert online_path.exists(), f"Expected online video artifact at {online_path}"
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
@@ -28,7 +29,7 @@ DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
class OmniBenchServerEndpoint(Protocol):
"""Anything with ``host`` / ``port`` / ``model`` (e.g. :class:`tests.conftest.OmniServer`)."""
"""Anything with ``host`` / ``port`` / ``model`` (e.g. :class:`tests.helpers.runtime.OmniServer`)."""
host: str
port: int
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-Omni accuracy benchmarks (Daily-Omni MCQ + Seed-TTS WER) via ``vllm bench serve --omni``.
Starts a **module-scoped** Omni OpenAI-compatible server (same pattern as ``tests/dfx/perf`` and
@@ -67,12 +68,8 @@ def get_chunk_config(config_path: str | None = None):
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
if config_path is None:
config_path = _CI_DEPLOY
# TODO: remove this workaround once legacy `stage_args` path is deleted.
# The pipeline (qwen3_omni/pipeline.py) already wires
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
# so only async_chunk needs flipping. Writing nested `engine_args:` into
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
# drops flat fields (load_format, max_num_seqs, ...).
# Pipeline already wires thinker2talker_async_chunk / talker2code2wav_async_chunk
# on stage 0/1; only the top-level async_chunk flag needs flipping here.
return modify_stage_config(config_path, updates={"async_chunk": True})
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Run vLLM-Omni with diffusers backend, then run diffusers directly. Compare the output similarity.
@@ -39,9 +39,9 @@ from tests.e2e.accuracy.helpers import (
from tests.e2e.accuracy.helpers import (
run_ffmpeg_similarity as _run_ffmpeg_similarity,
)
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.clean import cleanup_test_environment
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OnlineOmniClient
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import (
CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS,
)
@@ -128,7 +128,7 @@ def _run_vllm_omni_wan22_i2v(
"seed": SEED,
}
with OmniServer(model, server_args, env_dict=env_to_apply_ftfy_mock_in_subproc(), use_omni=True) as omni_server:
client = OpenAIClientHandler(
client = OnlineOmniClient(
host=omni_server.host,
port=omni_server.port,
run_level="full_model",
@@ -139,16 +139,20 @@ def _run_vllm_omni_wan22_i2v(
"image_reference": f"data:image/png;base64,{pil_to_base64(conditioning_image, 'png')}",
}
result = client.send_video_diffusion_request(request_config)[0]
video_bytes = result.videos[0] # pyright: ignore[reportOptionalSubscript] # Guaranteed not None
videos = result.videos
assert videos is not None
video_bytes = videos[0]
output_path.write_bytes(video_bytes)
return result.e2e_latency # pyright: ignore[reportReturnType] # Guaranteed not None
latency = result.e2e_latency
assert latency is not None
return latency
def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_image: Image.Image) -> float:
from diffusers import WanImageToVideoPipeline # pyright: ignore[reportPrivateImportUsage]
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
run_pre_test_cleanup()
cleanup_test_environment()
apply_ftfy_mock()
pipe: WanImageToVideoPipeline | None = None
try:
@@ -190,7 +194,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
@@ -248,7 +252,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
run_pre_test_cleanup()
cleanup_test_environment()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -288,7 +292,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
@pytest.mark.benchmark
+8 -5
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import base64
@@ -13,7 +16,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from PIL import Image
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.clean import cleanup_test_environment
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
@@ -149,7 +152,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> Image.Image:
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
run_pre_test_cleanup()
cleanup_test_environment()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -179,7 +182,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
def _run_vllm_omni_qwen_image_2512(*, model: str, output_path: Path) -> Image.Image:
@@ -211,7 +214,7 @@ def _run_vllm_omni_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Image:
run_pre_test_cleanup()
cleanup_test_environment()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -241,7 +244,7 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
@pytest.mark.benchmark
+8 -5
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import gc
@@ -12,7 +15,7 @@ from PIL import Image
from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.clean import cleanup_test_environment
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer
@@ -109,7 +112,7 @@ def _run_diffusers_image_edit(
input_images: list[Image.Image],
output_path: Path,
) -> Image.Image:
run_pre_test_cleanup()
cleanup_test_environment()
pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None
try:
images = input_images[0] if len(input_images) == 1 else input_images
@@ -140,7 +143,7 @@ def _run_diffusers_image_edit(
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
def _load_edit_2511_input_images() -> list[Image.Image]:
@@ -189,7 +192,7 @@ def _run_diffusers_image_edit_2511(
input_images: list[Image.Image],
output_path: Path,
) -> Image.Image:
run_pre_test_cleanup()
cleanup_test_environment()
pipe: QwenImageEditPlusPipeline | None = None
try:
pipe = QwenImageEditPlusPipeline.from_pretrained(
@@ -222,7 +225,7 @@ def _run_diffusers_image_edit_2511(
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
def _vllm_omni_output_single_image(
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import base64
@@ -13,7 +16,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from PIL import Image
from tests.e2e.accuracy.helpers import assert_image_sequence_similarity, model_output_dir
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
from tests.helpers.clean import cleanup_test_environment
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer
@@ -93,7 +96,7 @@ def _run_vllm_omni_qwen_image_layered(*, model: str, input_image: Image.Image, o
def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, output_dir: Path) -> list[Image.Image]:
run_pre_test_cleanup()
cleanup_test_environment()
pipe: DiffusionPipeline | None = None
try:
pipe = DiffusionPipeline.from_pretrained(
@@ -126,7 +129,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o
gc.collect()
if torch.cuda.is_available():
torch.accelerator.empty_cache()
run_post_test_cleanup()
cleanup_test_environment()
@pytest.mark.benchmark
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Online serving smoke test for streaming video output similarity.
"""
@@ -9,7 +12,7 @@ import pytest
from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_video
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler
from tests.helpers.runtime import DiffusionResponse, OmniServer, OnlineOmniClient, get_model_prefix
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
@@ -67,8 +70,8 @@ def _request_config(model: str, *, serialize_extra_params: bool) -> dict:
}
def _client_for(server: OmniServer, *, run_level: str) -> OpenAIClientHandler:
return OpenAIClientHandler(
def _client_for(server: OmniServer, *, run_level: str) -> OnlineOmniClient:
return OnlineOmniClient(
host=server.host,
port=server.port,
api_key="EMPTY",
@@ -95,12 +98,11 @@ def _write_video_artifact(response: DiffusionResponse, output_path: Path) -> Pat
)
def test_helios_streaming_output_matches_non_streaming(
model: str,
model_prefix: str,
run_level: str,
tmp_path: Path,
record_property,
):
model = model_prefix + model
model = get_model_prefix() + model
with OmniServer(model, _server_args(streaming_output=True), use_omni=True) as server:
client = _client_for(server, run_level=run_level)
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import os
@@ -338,7 +341,7 @@ def _artifact_paths(image_source: str) -> tuple[Path, Path, Path]:
def _generate_online_video(
*,
omni_server,
openai_client,
online_client,
image_source: str,
online_timeout_seconds: int,
) -> Path:
@@ -360,7 +363,7 @@ def _generate_online_video(
"image_reference": _build_online_image_reference(image_source),
}
online_video_bytes = _send_video_request_with_timeout(
openai_client,
online_client,
request_config,
timeout_seconds=_online_timeout_seconds(online_timeout_seconds),
)
@@ -412,7 +415,7 @@ def test_wan22_i2v_diffusers_offline_generates_video(
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
def test_wan22_i2v_online_serving_generates_video(
omni_server,
openai_client,
online_client,
wan22_i2v_image_source: str | None,
wan22_i2v_online_timeout_seconds: int,
) -> None:
@@ -424,7 +427,7 @@ def test_wan22_i2v_online_serving_generates_video(
_validate_image_source(image_source)
online_path = _generate_online_video(
omni_server=omni_server,
openai_client=openai_client,
online_client=online_client,
image_source=image_source,
online_timeout_seconds=wan22_i2v_online_timeout_seconds,
)