[Test] Cleanup for helpers (#6523)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] <your.email@example.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
8360346a66
commit
4af0ddc1c2
140 files changed
+4121
-4194
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -14,7 +17,7 @@ from hashlib import sha1
|
||||
from io import BytesIO
|
||||
from mimetypes import guess_type
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import IO, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import numpy as np
|
||||
@@ -35,11 +38,14 @@ class IdentityFtfy:
|
||||
|
||||
def apply_ftfy_mock(*, wan_i2v_module: Any | None = None) -> None:
|
||||
"""Mock ftfy library for test cases in the main process"""
|
||||
if wan_i2v_module is None:
|
||||
from diffusers.pipelines.wan import pipeline_wan_i2v as wan_i2v_module
|
||||
module: Any = wan_i2v_module
|
||||
if module is None:
|
||||
from diffusers.pipelines.wan import pipeline_wan_i2v as wan_pipeline
|
||||
|
||||
if not hasattr(wan_i2v_module, "ftfy"):
|
||||
wan_i2v_module.ftfy = IdentityFtfy()
|
||||
module = wan_pipeline
|
||||
|
||||
if not hasattr(module, "ftfy"):
|
||||
module.ftfy = IdentityFtfy()
|
||||
print("ftfy (text encoding sanitizer) is not installed. Using mock ftfy implementation (identity function)")
|
||||
else:
|
||||
print("ftfy (text encoding sanitizer) is installed. Using actual ftfy implementation.")
|
||||
@@ -118,6 +124,7 @@ def probe_binary(binary: str, *, test_name: str = "video similarity e2e test") -
|
||||
resolved = shutil.which(binary)
|
||||
if resolved is None:
|
||||
pytest.skip(f"{binary} is required for {test_name}.")
|
||||
assert resolved is not None
|
||||
return resolved
|
||||
|
||||
|
||||
@@ -235,7 +242,7 @@ def video_artifact_dir(result_root: Path, source: str) -> Path:
|
||||
|
||||
|
||||
def send_video_request_with_timeout(
|
||||
openai_client,
|
||||
online_client,
|
||||
request_config: dict[str, Any],
|
||||
*,
|
||||
timeout_seconds: int,
|
||||
@@ -266,7 +273,7 @@ def send_video_request_with_timeout(
|
||||
normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference})
|
||||
|
||||
start_time = time.perf_counter()
|
||||
create_url = openai_client._build_url("/v1/videos")
|
||||
create_url = online_client._build_url("/v1/videos")
|
||||
response = requests.post(
|
||||
create_url,
|
||||
data=normalized_form_data,
|
||||
@@ -280,8 +287,8 @@ def send_video_request_with_timeout(
|
||||
raise requests.HTTPError(f"{exc}; response body: {response.text}", response=response) from exc
|
||||
job_data = response.json()
|
||||
video_id = job_data["id"]
|
||||
openai_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds)
|
||||
video_content = openai_client._download_video_content(video_id)
|
||||
online_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds)
|
||||
video_content = online_client._download_video_content(video_id)
|
||||
print(f"online_video_e2e_latency_s={time.perf_counter() - start_time:.3f}")
|
||||
return video_content
|
||||
|
||||
@@ -571,11 +578,11 @@ class SemanticSimilarityScorer:
|
||||
raise ImportError("FlagEmbedding not installed. Install with: pip install FlagEmbedding")
|
||||
return self._text_model
|
||||
|
||||
def text_similarity(self, text1: str, text2: str) -> float:
|
||||
def text_similarity(self, text1: str, text2: str) -> dict[str, float | int]:
|
||||
"""
|
||||
Compute semantic similarity between two texts using BGE-M3
|
||||
"""
|
||||
results = {}
|
||||
results: dict[str, float | int] = {}
|
||||
# 1. Text prefix matching (maximum matching character length)
|
||||
match_count = 0
|
||||
for char_vllm, char_ref in zip(text1, text2):
|
||||
@@ -653,7 +660,7 @@ def dreamzero_cfg_parallel_size() -> int:
|
||||
|
||||
|
||||
def _dreamzero_parity_vllm_env() -> dict[str, str]:
|
||||
from tests.helpers.env import pick_least_used_device_indices
|
||||
from tests.helpers.clean import pick_least_used_device_indices
|
||||
from tests.helpers.runtime import get_open_port
|
||||
|
||||
parallel_size = dreamzero_cfg_parallel_size()
|
||||
@@ -683,8 +690,9 @@ def dreamzero_parity_vllm_params(model: str | None = None):
|
||||
]
|
||||
if parallel_size > 1:
|
||||
server_args.extend(["--cfg-parallel-size", str(parallel_size)])
|
||||
resolved_model = model or os.environ.get("VLLM_DREAMZERO_MODEL") or _DREAMZERO_DEFAULT_MODEL
|
||||
return OmniServerParams(
|
||||
model=model or os.environ.get("VLLM_DREAMZERO_MODEL", _DREAMZERO_DEFAULT_MODEL),
|
||||
model=resolved_model,
|
||||
stage_config_path=get_deploy_config_path("dreamzero_tp1_cfg2.yaml"),
|
||||
server_args=server_args,
|
||||
env_dict=_dreamzero_parity_vllm_env(),
|
||||
@@ -725,9 +733,9 @@ def dreamzero_parity_vllm_client(
|
||||
def _dreamzero_create_openpi_vllm_client(host: str, port: int) -> Any:
|
||||
from openpi_client import msgpack_numpy
|
||||
|
||||
_, WebsocketClientPolicy = load_dreamzero_upstream_modules()
|
||||
_, websocket_client_policy_cls = load_dreamzero_upstream_modules()
|
||||
|
||||
class _OpenPIWebsocketClientPolicy(WebsocketClientPolicy):
|
||||
class _OpenPIWebsocketClientPolicy(websocket_client_policy_cls): # type: ignore[misc, valid-type]
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "127.0.0.1",
|
||||
@@ -869,7 +877,7 @@ class _DreamZeroParityServer(ABC):
|
||||
self.port = get_open_port() if port is None else port
|
||||
self.ready_timeout_s = ready_timeout_s
|
||||
self.proc: subprocess.Popen[str] | None = None
|
||||
self._log_file = None
|
||||
self._log_file: IO[str] | None = None
|
||||
|
||||
@abstractmethod
|
||||
def _build_argv(self) -> list[str]:
|
||||
@@ -932,7 +940,7 @@ class DreamZeroUpstreamServer(_DreamZeroParityServer):
|
||||
]
|
||||
|
||||
def _build_env(self) -> dict[str, str]:
|
||||
from tests.helpers.env import pick_least_used_device_indices
|
||||
from tests.helpers.clean import pick_least_used_device_indices
|
||||
|
||||
dreamzero_repo, _ = require_dreamzero_parity_env()
|
||||
env = os.environ.copy()
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
@@ -149,7 +152,7 @@ def _generate_offline_video(*, image_source: str) -> Path:
|
||||
def _generate_online_video(
|
||||
*,
|
||||
omni_server,
|
||||
openai_client,
|
||||
online_client,
|
||||
image_source: str,
|
||||
timeout_seconds: int,
|
||||
) -> Path:
|
||||
@@ -171,7 +174,7 @@ def _generate_online_video(
|
||||
"image_reference": build_online_image_reference(online_image_source),
|
||||
}
|
||||
online_video_bytes = send_video_request_with_timeout(
|
||||
openai_client,
|
||||
online_client,
|
||||
request_config,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
@@ -204,7 +207,7 @@ def test_hunyuanvideo15_i2v_diffusers_offline_generates_video(
|
||||
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
|
||||
def test_hunyuanvideo15_i2v_online_serving_generates_video(
|
||||
omni_server,
|
||||
openai_client,
|
||||
online_client,
|
||||
hunyuanvideo15_i2v_image_source: str | None,
|
||||
hunyuanvideo15_online_timeout_seconds: int,
|
||||
) -> None:
|
||||
@@ -216,7 +219,7 @@ def test_hunyuanvideo15_i2v_online_serving_generates_video(
|
||||
validate_image_source(image_source)
|
||||
online_path = _generate_online_video(
|
||||
omni_server=omni_server,
|
||||
openai_client=openai_client,
|
||||
online_client=online_client,
|
||||
image_source=image_source,
|
||||
timeout_seconds=hunyuanvideo15_online_timeout_seconds,
|
||||
)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
@@ -115,7 +118,7 @@ def _generate_offline_video() -> Path:
|
||||
return offline_path
|
||||
|
||||
|
||||
def _generate_online_video(*, omni_server, openai_client, timeout_seconds: int) -> Path:
|
||||
def _generate_online_video(*, omni_server, online_client, timeout_seconds: int) -> Path:
|
||||
online_path, _ = _artifact_paths()
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
@@ -132,7 +135,7 @@ def _generate_online_video(*, omni_server, openai_client, timeout_seconds: int)
|
||||
},
|
||||
}
|
||||
online_video_bytes = send_video_request_with_timeout(
|
||||
openai_client,
|
||||
online_client,
|
||||
request_config,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
@@ -161,7 +164,7 @@ def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None:
|
||||
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
|
||||
def test_hunyuanvideo15_t2v_online_serving_generates_video(
|
||||
omni_server,
|
||||
openai_client,
|
||||
online_client,
|
||||
hunyuanvideo15_online_timeout_seconds: int,
|
||||
) -> None:
|
||||
if not torch.cuda.is_available():
|
||||
@@ -170,7 +173,7 @@ def test_hunyuanvideo15_t2v_online_serving_generates_video(
|
||||
probe_binary("ffprobe")
|
||||
online_path = _generate_online_video(
|
||||
omni_server=omni_server,
|
||||
openai_client=openai_client,
|
||||
online_client=online_client,
|
||||
timeout_seconds=hunyuanvideo15_online_timeout_seconds,
|
||||
)
|
||||
assert online_path.exists(), f"Expected online video artifact at {online_path}"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
|
||||
|
||||
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
|
||||
@@ -28,7 +29,7 @@ DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
|
||||
|
||||
|
||||
class OmniBenchServerEndpoint(Protocol):
|
||||
"""Anything with ``host`` / ``port`` / ``model`` (e.g. :class:`tests.conftest.OmniServer`)."""
|
||||
"""Anything with ``host`` / ``port`` / ``model`` (e.g. :class:`tests.helpers.runtime.OmniServer`)."""
|
||||
|
||||
host: str
|
||||
port: int
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Qwen3-Omni accuracy benchmarks (Daily-Omni MCQ + Seed-TTS WER) via ``vllm bench serve --omni``.
|
||||
|
||||
Starts a **module-scoped** Omni OpenAI-compatible server (same pattern as ``tests/dfx/perf`` and
|
||||
@@ -67,12 +68,8 @@ def get_chunk_config(config_path: str | None = None):
|
||||
"""Load the qwen3_omni CI deploy yaml with async_chunk modifications for streaming mode."""
|
||||
if config_path is None:
|
||||
config_path = _CI_DEPLOY
|
||||
# TODO: remove this workaround once legacy `stage_args` path is deleted.
|
||||
# The pipeline (qwen3_omni/pipeline.py) already wires
|
||||
# thinker2talker_async_chunk / talker2code2wav_async_chunk on stage 0/1,
|
||||
# so only async_chunk needs flipping. Writing nested `engine_args:` into
|
||||
# the new-schema overlay trips _parse_stage_deploy's legacy branch and
|
||||
# drops flat fields (load_format, max_num_seqs, ...).
|
||||
# Pipeline already wires thinker2talker_async_chunk / talker2code2wav_async_chunk
|
||||
# on stage 0/1; only the top-level async_chunk flag needs flipping here.
|
||||
return modify_stage_config(config_path, updates={"async_chunk": True})
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Run vLLM-Omni with diffusers backend, then run diffusers directly. Compare the output similarity.
|
||||
@@ -39,9 +39,9 @@ from tests.e2e.accuracy.helpers import (
|
||||
from tests.e2e.accuracy.helpers import (
|
||||
run_ffmpeg_similarity as _run_ffmpeg_similarity,
|
||||
)
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.clean import cleanup_test_environment
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer, OpenAIClientHandler
|
||||
from tests.helpers.runtime import OmniServer, OnlineOmniClient
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter import (
|
||||
CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS,
|
||||
)
|
||||
@@ -128,7 +128,7 @@ def _run_vllm_omni_wan22_i2v(
|
||||
"seed": SEED,
|
||||
}
|
||||
with OmniServer(model, server_args, env_dict=env_to_apply_ftfy_mock_in_subproc(), use_omni=True) as omni_server:
|
||||
client = OpenAIClientHandler(
|
||||
client = OnlineOmniClient(
|
||||
host=omni_server.host,
|
||||
port=omni_server.port,
|
||||
run_level="full_model",
|
||||
@@ -139,16 +139,20 @@ def _run_vllm_omni_wan22_i2v(
|
||||
"image_reference": f"data:image/png;base64,{pil_to_base64(conditioning_image, 'png')}",
|
||||
}
|
||||
result = client.send_video_diffusion_request(request_config)[0]
|
||||
video_bytes = result.videos[0] # pyright: ignore[reportOptionalSubscript] # Guaranteed not None
|
||||
videos = result.videos
|
||||
assert videos is not None
|
||||
video_bytes = videos[0]
|
||||
output_path.write_bytes(video_bytes)
|
||||
return result.e2e_latency # pyright: ignore[reportReturnType] # Guaranteed not None
|
||||
latency = result.e2e_latency
|
||||
assert latency is not None
|
||||
return latency
|
||||
|
||||
|
||||
def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_image: Image.Image) -> float:
|
||||
from diffusers import WanImageToVideoPipeline # pyright: ignore[reportPrivateImportUsage]
|
||||
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
|
||||
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
apply_ftfy_mock()
|
||||
pipe: WanImageToVideoPipeline | None = None
|
||||
try:
|
||||
@@ -190,7 +194,7 @@ def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_imag
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
|
||||
@@ -248,7 +252,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]:
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -288,7 +292,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
@@ -13,7 +16,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.clean import cleanup_test_environment
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
@@ -149,7 +152,7 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -179,7 +182,7 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image:
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
def _run_vllm_omni_qwen_image_2512(*, model: str, output_path: Path) -> Image.Image:
|
||||
@@ -211,7 +214,7 @@ def _run_vllm_omni_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Image:
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -241,7 +244,7 @@ def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Im
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
@@ -12,7 +15,7 @@ from PIL import Image
|
||||
|
||||
from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes
|
||||
from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.clean import cleanup_test_environment
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniServer
|
||||
@@ -109,7 +112,7 @@ def _run_diffusers_image_edit(
|
||||
input_images: list[Image.Image],
|
||||
output_path: Path,
|
||||
) -> Image.Image:
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None
|
||||
try:
|
||||
images = input_images[0] if len(input_images) == 1 else input_images
|
||||
@@ -140,7 +143,7 @@ def _run_diffusers_image_edit(
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
def _load_edit_2511_input_images() -> list[Image.Image]:
|
||||
@@ -189,7 +192,7 @@ def _run_diffusers_image_edit_2511(
|
||||
input_images: list[Image.Image],
|
||||
output_path: Path,
|
||||
) -> Image.Image:
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
pipe: QwenImageEditPlusPipeline | None = None
|
||||
try:
|
||||
pipe = QwenImageEditPlusPipeline.from_pretrained(
|
||||
@@ -222,7 +225,7 @@ def _run_diffusers_image_edit_2511(
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
def _vllm_omni_output_single_image(
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
@@ -13,7 +16,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
from PIL import Image
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_image_sequence_similarity, model_output_dir
|
||||
from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
|
||||
from tests.helpers.clean import cleanup_test_environment
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
@@ -93,7 +96,7 @@ def _run_vllm_omni_qwen_image_layered(*, model: str, input_image: Image.Image, o
|
||||
|
||||
|
||||
def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, output_dir: Path) -> list[Image.Image]:
|
||||
run_pre_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
pipe: DiffusionPipeline | None = None
|
||||
try:
|
||||
pipe = DiffusionPipeline.from_pretrained(
|
||||
@@ -126,7 +129,7 @@ def _run_diffusers_qwen_image_layered(*, model: str, input_image: Image.Image, o
|
||||
gc.collect()
|
||||
if torch.cuda.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
run_post_test_cleanup()
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Online serving smoke test for streaming video output similarity.
|
||||
"""
|
||||
@@ -9,7 +12,7 @@ import pytest
|
||||
|
||||
from tests.e2e.accuracy.helpers import assert_video_similarity_metrics, probe_video
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import DiffusionResponse, OmniServer, OpenAIClientHandler
|
||||
from tests.helpers.runtime import DiffusionResponse, OmniServer, OnlineOmniClient, get_model_prefix
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
|
||||
|
||||
@@ -67,8 +70,8 @@ def _request_config(model: str, *, serialize_extra_params: bool) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _client_for(server: OmniServer, *, run_level: str) -> OpenAIClientHandler:
|
||||
return OpenAIClientHandler(
|
||||
def _client_for(server: OmniServer, *, run_level: str) -> OnlineOmniClient:
|
||||
return OnlineOmniClient(
|
||||
host=server.host,
|
||||
port=server.port,
|
||||
api_key="EMPTY",
|
||||
@@ -95,12 +98,11 @@ def _write_video_artifact(response: DiffusionResponse, output_path: Path) -> Pat
|
||||
)
|
||||
def test_helios_streaming_output_matches_non_streaming(
|
||||
model: str,
|
||||
model_prefix: str,
|
||||
run_level: str,
|
||||
tmp_path: Path,
|
||||
record_property,
|
||||
):
|
||||
model = model_prefix + model
|
||||
model = get_model_prefix() + model
|
||||
|
||||
with OmniServer(model, _server_args(streaming_output=True), use_omni=True) as server:
|
||||
client = _client_for(server, run_level=run_level)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
@@ -338,7 +341,7 @@ def _artifact_paths(image_source: str) -> tuple[Path, Path, Path]:
|
||||
def _generate_online_video(
|
||||
*,
|
||||
omni_server,
|
||||
openai_client,
|
||||
online_client,
|
||||
image_source: str,
|
||||
online_timeout_seconds: int,
|
||||
) -> Path:
|
||||
@@ -360,7 +363,7 @@ def _generate_online_video(
|
||||
"image_reference": _build_online_image_reference(image_source),
|
||||
}
|
||||
online_video_bytes = _send_video_request_with_timeout(
|
||||
openai_client,
|
||||
online_client,
|
||||
request_config,
|
||||
timeout_seconds=_online_timeout_seconds(online_timeout_seconds),
|
||||
)
|
||||
@@ -412,7 +415,7 @@ def test_wan22_i2v_diffusers_offline_generates_video(
|
||||
@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
|
||||
def test_wan22_i2v_online_serving_generates_video(
|
||||
omni_server,
|
||||
openai_client,
|
||||
online_client,
|
||||
wan22_i2v_image_source: str | None,
|
||||
wan22_i2v_online_timeout_seconds: int,
|
||||
) -> None:
|
||||
@@ -424,7 +427,7 @@ def test_wan22_i2v_online_serving_generates_video(
|
||||
_validate_image_source(image_source)
|
||||
online_path = _generate_online_video(
|
||||
omni_server=omni_server,
|
||||
openai_client=openai_client,
|
||||
online_client=online_client,
|
||||
image_source=image_source,
|
||||
online_timeout_seconds=wan22_i2v_online_timeout_seconds,
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user