[Test] Cleanup for helpers (#6523)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
wangyuandamy-why-3459 authored and GitHub committed 2026-08-25 09:10:17 +08:00
1 parent 8360346a66
commit 4af0ddc1c2
140 files changed
+4121 -4194

No files matched your search

@@ -1,13 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""``POST /v1/audio/generate`` validation (live Stable Audio Open server)."""
from __future__ import annotations
from typing import Any
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
@@ -95,20 +97,20 @@ _PARAMS = [
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
def test_audio_generate_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
body = {"model": omni_server.model, "input": "ambient electronic pad"}
body: dict[str, Any] = {"model": omni_server.model, "input": "ambient electronic pad"}
body.update(overrides)
openai_client.send_audio_generate_http_request(
online_client.send_audio_generate_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
def test_audio_generate_missing_input(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_audio_generate_http_request(
def test_audio_generate_missing_input(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_audio_generate_http_request(
{
"json": {"model": omni_server.model},
"timeout": 120,
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""HTTP + WebSocket validation for Qwen3-TTS Base, Higgs Audio V3 and Audex TTA ``/v1/audio/speech`` (and related routes)."""
from __future__ import annotations
@@ -13,8 +13,8 @@ from typing import Any
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.media import load_test_audio_data_url
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
from tests.helpers.stage_config import get_deploy_config_path
# Speech / voice-upload numeric caps for these tests only (not exported from vllm_omni).
@@ -26,7 +26,7 @@ _SPEECH_API_MAX_NEW_TOKENS = 4096
_VOICE_UPLOAD_MAX_CONSENT_LEN = 1024
_VOICE_UPLOAD_MAX_REF_TEXT_CHARS = 8192
_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS = 2048
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
REF_AUDIO_URL = get_asset_path("qwen3_tts/clone_2.wav", as_data_url=True)
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
# Qwen3-TTS 0.6B expects 1024-dim speaker embeddings (see serving_speech / talker hidden_size).
_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM = 1024
@@ -57,6 +57,7 @@ _QWEN3_TTS_SPEECH = [
),
]
_HIGGS_AUDIO_V3_SPEECH = [
pytest.param(
OmniServerParams(
@@ -84,7 +85,7 @@ _AUDEX_TTA_SPEECH = [
]
def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
def _apply_batch_overrides(body: dict[str, Any], *, loc: str, overrides: dict[str, object]) -> None:
if loc == "batch":
body.update(overrides)
return
@@ -112,8 +113,8 @@ def _pcm_wav_mono_bytes(*, duration_s: float = 1.1, sample_rate: int = 44100) ->
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_malformed_json(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_audio_speech_http_request(
def test_speech_malformed_json(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_audio_speech_http_request(
{
"raw_body": "{",
"timeout": 120,
@@ -124,8 +125,8 @@ def test_speech_malformed_json(omni_server: OmniServer, openai_client: OpenAICli
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_missing_required_fields(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_audio_speech_http_request(
def test_speech_missing_required_fields(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_audio_speech_http_request(
{
"json": {"model": omni_server.model},
"timeout": 120,
@@ -242,13 +243,13 @@ def test_speech_missing_required_fields(omni_server: OmniServer, openai_client:
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
ov = dict(overrides)
embedding_only = bool(ov.pop(_SPEECH_INVALID_EMBEDDING_ONLY_BODY, False))
body = {
body: dict[str, Any] = {
"model": omni_server.model,
"input": "Hello.",
"voice": "clone",
@@ -261,7 +262,7 @@ def test_speech_invalid_field_values(
body.pop("ref_audio", None)
body.pop("ref_text", None)
body.update(ov)
openai_client.send_audio_speech_http_request(
online_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -328,12 +329,12 @@ def test_speech_invalid_field_values(
@pytest.mark.parametrize("omni_server", _HIGGS_AUDIO_V3_SPEECH, indirect=True)
def test_speech_higgs_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
extra_json: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
body = {"model": omni_server.model, **extra_json}
openai_client.send_audio_speech_http_request(
online_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -359,12 +360,12 @@ def test_speech_higgs_invalid_field_values(
@pytest.mark.parametrize("omni_server", _AUDEX_TTA_SPEECH, indirect=True)
def test_speech_audex_tta_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
body = {"model": omni_server.model, "input": _AUDEX_TTA_CAPTION, **overrides}
openai_client.send_audio_speech_http_request(
online_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -375,15 +376,15 @@ def test_speech_audex_tta_invalid_field_values(
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_batch_empty_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_audio_speech_batch_http_request(
def test_speech_batch_empty_items(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_audio_speech_batch_http_request(
{"json": {"items": []}, "timeout": 120, "err_code": (400, 422), "err_message": ("items", "least 1 item")}
)
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_batch_missing_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_audio_speech_batch_http_request(
def test_speech_batch_missing_items(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_audio_speech_batch_http_request(
{
"json": {"model": omni_server.model},
"timeout": 120,
@@ -618,14 +619,14 @@ def test_speech_batch_missing_items(omni_server: OmniServer, openai_client: Open
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_batch_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
loc: str,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
ov = dict(overrides)
strip_clone = bool(ov.pop(_SPEECH_BATCH_NO_CLONE_FIELDS, False))
body = {
body: dict[str, Any] = {
"model": omni_server.model,
"voice": "clone",
"ref_audio": REF_AUDIO_URL,
@@ -638,7 +639,7 @@ def test_speech_batch_invalid_field_values(
body.pop("ref_audio", None)
body.pop("ref_text", None)
_apply_batch_overrides(body, loc=loc, overrides=ov)
openai_client.send_audio_speech_batch_http_request(
online_client.send_audio_speech_batch_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -670,7 +671,7 @@ _SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES = object()
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_speech_stream_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
send_frames_spec: Any,
err_message: str | tuple[str, ...],
) -> None:
@@ -687,7 +688,7 @@ def test_speech_stream_invalid_requests(
else:
send_frames = send_frames_spec
assert isinstance(send_frames, str)
openai_client.send_audio_speech_stream_ws_request(
online_client.send_audio_speech_stream_ws_request(
{
"send_frames": send_frames,
"timeout": 120,
@@ -893,7 +894,7 @@ def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_voices_create_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
multipart_kind: str | None,
form_template: dict[str, Any],
err_message: str | tuple[str, ...],
@@ -909,7 +910,7 @@ def test_voices_create_invalid_requests(
files = _voices_upload_multipart_files(multipart_kind)
if files is not None:
cfg["files"] = files
openai_client.send_audio_voices_create_http_request(cfg)
online_client.send_audio_voices_create_http_request(cfg)
@pytest.mark.parametrize(
@@ -962,13 +963,13 @@ def test_voices_create_invalid_requests(
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
def test_voices_delete_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
voice_name: str,
err_code: int | tuple[int, ...],
err_message: str | tuple[str, ...],
) -> None:
"""DELETE ``/v1/audio/voices/{name}``: missing voice (404), odd segments (often 404), ``.``paths (405)."""
openai_client.send_audio_voices_delete_http_request(
online_client.send_audio_voices_delete_http_request(
{
"name": voice_name,
"timeout": 120,
@@ -1021,7 +1022,7 @@ _MINIMAX_MUSIC3_MAX_FRAMES = 9000
@pytest.mark.parametrize("omni_server", _MINIMAX_MUSIC3_SPEECH, indirect=True)
def test_minimax_music3_speech_invalid_field_values(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
overrides: dict[str, object],
err_message: str | tuple[str, ...],
) -> None:
@@ -1038,6 +1039,6 @@ def test_minimax_music3_speech_invalid_field_values(
body.pop(key, None)
else:
body[key] = value
openai_client.send_audio_speech_http_request(
online_client.send_audio_speech_http_request(
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""``POST /v1/images/edits`` multipart validation (live ``Qwen/Qwen-Image-Edit``)."""
from __future__ import annotations
@@ -9,7 +9,7 @@ from typing import Any
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
@@ -72,7 +72,7 @@ def _finalize_edits_form_data(template: dict[str, Any], server_model: str) -> di
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
def test_images_edits_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
tiny_png_bytes: bytes,
include_image: bool,
data_template: dict[str, Any],
@@ -87,4 +87,4 @@ def test_images_edits_invalid_requests(
}
if include_image:
cfg["files"] = {"image": ("x.png", tiny_png_bytes, "image/png")}
openai_client.send_images_edits_http_request(cfg)
online_client.send_images_edits_http_request(cfg)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""``POST /v1/images/generations`` validation (live ``Qwen/Qwen-Image``)."""
from __future__ import annotations
@@ -9,7 +9,7 @@ from typing import Any
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
@@ -126,7 +126,7 @@ def _minimal_images_gen_json(omni_server: OmniServer) -> dict[str, object]:
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
def test_images_generations_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
body_spec: Any,
err_message: str | tuple[str, ...],
) -> None:
@@ -137,6 +137,6 @@ def test_images_generations_invalid_requests(
assert isinstance(body_spec, dict)
body = _minimal_images_gen_json(omni_server)
body.update(body_spec)
openai_client.send_images_generations_http_request(
online_client.send_images_generations_http_request(
{"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Invalid inputs on Qwen3-Omni: ``POST /v1/chat/completions``, ``WS /v1/video/chat/stream``, ``WS /v1/realtime``."""
from __future__ import annotations
@@ -10,7 +10,7 @@ from typing import Any
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
from tests.helpers.stage_config import get_deploy_config_path
pytestmark = [pytest.mark.slow, pytest.mark.omni]
@@ -140,7 +140,7 @@ def _chat_completions_request_without_expectations(omni_server: OmniServer, case
@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
def test_chat_completions_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
case_id: str,
err_code: int | tuple[int, ...],
err_message: str | tuple[str, ...],
@@ -148,7 +148,7 @@ def test_chat_completions_invalid_requests(
cfg = _chat_completions_request_without_expectations(omni_server, case_id)
cfg["err_code"] = err_code
cfg["err_message"] = err_message
openai_client.send_chat_completions_http_request(cfg)[0]
online_client.send_chat_completions_http_request(cfg)[0]
# ─────────────────────────────────────────────────────────────────────────────
@@ -174,7 +174,7 @@ _VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR = object()
@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
def test_video_chat_stream_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
send_frames_spec: Any,
err_message: str,
) -> None:
@@ -183,7 +183,7 @@ def test_video_chat_stream_invalid_requests(
else:
send_frames = send_frames_spec
assert isinstance(send_frames, str)
openai_client.send_video_chat_stream_ws_request(
online_client.send_video_chat_stream_ws_request(
{
"send_frames": send_frames,
"timeout": 120,
@@ -243,11 +243,12 @@ _REALTIME_WS_INVALID_AUDIO_APPEND = object()
@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
def test_realtime_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
send_frames_spec: Any,
ws_error_code: str,
err_message: str | tuple[str, ...],
) -> None:
send_frames: Any
if send_frames_spec is _REALTIME_WS_MODEL_MISMATCH:
send_frames = json.dumps(
{
@@ -262,7 +263,7 @@ def test_realtime_invalid_requests(
]
else:
send_frames = send_frames_spec
openai_client.send_realtime_ws_request(
online_client.send_realtime_ws_request(
{
"send_frames": send_frames,
"timeout": 120,
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Omni sleep/wakeup JSON validation and level-2 wake contract (live ``Qwen/Qwen-Image``)."""
from __future__ import annotations
@@ -9,7 +9,7 @@ from typing import Any, Literal
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
pytestmark = [pytest.mark.slow, pytest.mark.diffusion, pytest.mark.omni]
@@ -86,7 +86,7 @@ _QWEN_IMAGE = [
@pytest.mark.parametrize("omni_server", _QWEN_IMAGE, indirect=True)
def test_omni_sleep_wakeup_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
endpoint: Literal["sleep", "wakeup"],
json_body: dict[str, Any],
err_message: str | tuple[str, ...],
@@ -98,9 +98,9 @@ def test_omni_sleep_wakeup_invalid_requests(
"err_message": err_message,
}
if endpoint == "sleep":
openai_client.send_omni_sleep_http_request(cfg)
online_client.send_omni_sleep_http_request(cfg)
else:
openai_client.send_omni_wakeup_http_request(cfg)
online_client.send_omni_wakeup_http_request(cfg)
# Keep this after the invalid-JSON cases: sleep(level=2) discards weights and the
@@ -108,7 +108,7 @@ def test_omni_sleep_wakeup_invalid_requests(
@pytest.mark.parametrize("omni_server", _QWEN_IMAGE, indirect=True)
def test_wakeup_after_level2_sleep_fails(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
) -> None:
"""Regression for #4473 Repro A via HTTP: wake after sleep(level=2) → 501.
@@ -116,7 +116,7 @@ def test_wakeup_after_level2_sleep_fails(
Live serve surfaces ``NotImplementedError`` as OpenAI-style JSON with code 501.
"""
assert omni_server is not None
sleep_resps = openai_client.send_omni_sleep_http_request(
sleep_resps = online_client.send_omni_sleep_http_request(
{
"json": {"stage_ids": [0], "level": 2},
"timeout": 180,
@@ -124,10 +124,11 @@ def test_wakeup_after_level2_sleep_fails(
)
sleep_resp = sleep_resps[0]
assert sleep_resp.status_code == 200, sleep_resp
assert sleep_resp.json_body is not None
assert sleep_resp.json_body.get("status") == "SUCCESS", sleep_resp.json_body
json_body = sleep_resp.json_body
assert isinstance(json_body, dict)
assert json_body.get("status") == "SUCCESS", json_body
openai_client.send_omni_wakeup_http_request(
online_client.send_omni_wakeup_http_request(
{
"json": {"stage_ids": [0]},
"timeout": 120,
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""HTTP video job routes (live ``Wan-AI/Wan2.2-T2V-A14B-Diffusers``).
WebSocket ``/v1/video/chat/stream`` and ``/v1/realtime`` invalid-input checks live in
@@ -13,7 +13,7 @@ from typing import Any
import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
@@ -54,12 +54,12 @@ def _minimal_video_form_data() -> dict[str, str]:
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
def test_post_videos_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
form_extra: dict[str, str] | None,
err_message: str | tuple[str, ...],
) -> None:
data: dict[str, str] = {} if form_extra is None else {**_minimal_video_form_data(), **form_extra}
openai_client.send_videos_create_http_request(
online_client.send_videos_create_http_request(
{"data": data, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -85,12 +85,12 @@ def test_post_videos_invalid_requests(
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
def test_post_videos_sync_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
form_extra: dict[str, str] | None,
err_message: str | tuple[str, ...],
) -> None:
data: dict[str, str] = {} if form_extra is None else {**_minimal_video_form_data(), **form_extra}
openai_client.send_videos_sync_http_request(
online_client.send_videos_sync_http_request(
{"data": data, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -111,11 +111,11 @@ def test_post_videos_sync_invalid_requests(
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
def test_get_videos_list_invalid_requests(
omni_server: OmniServer,
openai_client: OpenAIClientHandler,
online_client: OnlineOmniClient,
params: dict[str, Any],
err_message: str | tuple[str, ...],
) -> None:
openai_client.send_videos_list_http_request(
online_client.send_videos_list_http_request(
{"params": params, "timeout": 120, "err_code": 400, "err_message": err_message}
)
@@ -126,8 +126,8 @@ def test_get_videos_list_invalid_requests(
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
def test_get_video_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_video_retrieve_http_request(
def test_get_video_invalid_requests(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_video_retrieve_http_request(
{
"video_id": "does-not-exist",
"timeout": 120,
@@ -143,8 +143,8 @@ def test_get_video_invalid_requests(omni_server: OmniServer, openai_client: Open
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
def test_delete_video_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_video_delete_http_request(
def test_delete_video_invalid_requests(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_video_delete_http_request(
{
"video_id": "does-not-exist",
"timeout": 120,
@@ -160,8 +160,8 @@ def test_delete_video_invalid_requests(omni_server: OmniServer, openai_client: O
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
def test_get_video_content_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
openai_client.send_video_content_http_request(
def test_get_video_content_invalid_requests(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
online_client.send_video_content_http_request(
{
"video_id": "does-not-exist",
"timeout": 120,
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
HunyuanImage-3.0-Instruct reliability integration tests (DiT-only deploy).
@@ -35,7 +38,7 @@ from tests.dfx.reliability.helpers import (
worker_residual_timeout_after_kill_signal,
)
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OpenAIClientHandler
from tests.helpers.runtime import OnlineOmniClient
RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
{
@@ -133,8 +136,8 @@ def _image_request_config(omni_server: Any) -> dict[str, Any]:
}
def _submit_image_generation(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
responses = openai_client.send_images_generations_http_request(_image_request_config(omni_server))
def _submit_image_generation(online_client: OnlineOmniClient, omni_server: Any) -> None:
responses = online_client.send_images_generations_http_request(_image_request_config(omni_server))
resp = responses[0]
if resp.status_code >= 400:
raise RuntimeError(
@@ -189,7 +192,7 @@ def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
def test_reliability_fault_gpu_oom_image_large_request_failure(
omni_server_function,
openai_client_function,
online_client_function,
) -> None:
stage_config_path = getattr(omni_server_function, "stage_config_path", None)
device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
@@ -202,7 +205,7 @@ def test_reliability_fault_gpu_oom_image_large_request_failure(
)
try:
try:
_submit_image_generation(openai_client_function, omni_server_function)
_submit_image_generation(online_client_function, omni_server_function)
except Exception as exc:
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
else:
@@ -222,10 +225,10 @@ def test_reliability_fault_gpu_oom_image_large_request_failure(
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_request_failure(
omni_server_after_fault_function,
openai_client_function,
online_client_function,
) -> None:
try:
_submit_image_generation(openai_client_function, omni_server_after_fault_function)
_submit_image_generation(online_client_function, omni_server_after_fault_function)
except Exception as exc:
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
else:
@@ -244,7 +247,7 @@ def test_reliability_fault_process_kill_request_failure(
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
omni_server_after_fault_function,
openai_client_function,
online_client_function,
) -> None:
host = omni_server_after_fault_function.host
port = omni_server_after_fault_function.port
@@ -280,7 +283,7 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
start = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
futures = [
executor.submit(_submit_image_generation, openai_client_function, omni_server_after_fault_function)
executor.submit(_submit_image_generation, online_client_function, omni_server_after_fault_function)
for _ in range(3)
]
done, pending = concurrent.futures.wait(
@@ -335,12 +338,12 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_worker_with_load_request_failure(
omni_server_function,
openai_client_function,
online_client_function,
fault_injector: FaultInjector,
) -> None:
scenario = "kill_worker_with_load"
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
submit_request=lambda: _submit_image_generation(online_client_function, omni_server_function),
inject_fault=lambda: fault_injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -363,13 +366,13 @@ def test_reliability_fault_process_kill_worker_with_load_request_failure(
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
omni_server_function,
openai_client_function,
online_client_function,
signal_name: str,
) -> None:
scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
submit_request=lambda: _submit_image_generation(online_client_function, omni_server_function),
inject_fault=lambda: injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -448,7 +451,7 @@ def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
omni_server_function,
openai_client_function,
online_client_function,
signal_name: str,
) -> None:
scenario = f"kill_serve_tree_with_load_{signal_name.lower()}"
@@ -458,7 +461,7 @@ def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
inter_kill_wait_seconds=0.1,
)
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
submit_request=lambda: _submit_image_generation(online_client_function, omni_server_function),
inject_fault=lambda: injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Qwen3-Omni reliability integration tests.
"""
@@ -63,7 +66,7 @@ DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vll
def _default_oom_device_spec() -> str:
"""Use currently visible CUDA ordinals to avoid invalid device index in sidecar."""
count = torch.accelerator.device_count()
count = torch.accelerator.device_count() or 0
if count <= 0:
return "0"
return ",".join(str(i) for i in range(count))
@@ -414,7 +417,7 @@ def test_reliability_fault_gpu_oom_error_contract_consistent_chat_speech(
@pytest.mark.slow
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
def test_reliability_fault_gpu_oom_chat_large_payload_failure(omni_server_function, openai_client_function) -> None:
def test_reliability_fault_gpu_oom_chat_large_payload_failure(omni_server_function, online_client_function) -> None:
device_spec = resolve_oom_device_spec(
OOM_INJECTION_CONFIG,
_deploy_config_path_from_omni_server(omni_server_function),
@@ -444,7 +447,7 @@ def test_reliability_fault_gpu_oom_chat_large_payload_failure(omni_server_functi
"key_words": {"audio": ["test"]},
}
try:
openai_client_function.send_omni_request(request_config, request_num=1)
online_client_function.send_omni_request(request_config, request_num=1)
except Exception as exc:
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
else:
@@ -456,7 +459,7 @@ def test_reliability_fault_gpu_oom_chat_large_payload_failure(omni_server_functi
@pytest.mark.slow
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
def test_reliability_fault_gpu_oom_concurrent_pressure_failure(omni_server_function, openai_client_function) -> None:
def test_reliability_fault_gpu_oom_concurrent_pressure_failure(omni_server_function, online_client_function) -> None:
device_spec = resolve_oom_device_spec(
OOM_INJECTION_CONFIG,
_deploy_config_path_from_omni_server(omni_server_function),
@@ -487,7 +490,7 @@ def test_reliability_fault_gpu_oom_concurrent_pressure_failure(omni_server_funct
"key_words": {"audio": ["test"]},
}
try:
openai_client_function.send_omni_request(request_config, request_num=4)
online_client_function.send_omni_request(request_config, request_num=4)
except Exception as exc:
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
else:
@@ -505,7 +508,7 @@ def test_reliability_fault_gpu_oom_concurrent_pressure_failure(omni_server_funct
)
@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_request_failure(
omni_server_after_fault_function, openai_client_function
omni_server_after_fault_function, online_client_function
) -> None:
messages = dummy_messages_from_mix_data(
system_prompt=_get_system_prompt(),
@@ -519,7 +522,7 @@ def test_reliability_fault_process_kill_request_failure(
"key_words": {"text": ["beijing"]},
}
try:
openai_client_function.send_omni_request(request_config, request_num=1)
online_client_function.send_omni_request(request_config, request_num=1)
except Exception as exc:
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
else:
@@ -567,7 +570,7 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
assert elapsed <= _PROCESS_KILL_WORKER_CHAT_FF_MAX_ELAPSED_SEC + 1.0, (
f"[process_kill fast_fail] request did not fail fast after fault: {elapsed:.2f}s"
)
assert ff_status >= 500, (
assert ff_status is not None and ff_status >= 500, (
f"[process_kill fast_fail] expected server-side failure after fault, "
f"got status={ff_status}, body={ff_body[:200]!r}"
)
@@ -652,13 +655,13 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_worker_with_load_request_failure(
omni_server_function,
openai_client_function,
online_client_function,
fault_injector: FaultInjector,
) -> None:
request_config = _chat_request_config(omni_server_function)
scenario = "kill_worker_with_load"
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: openai_client_function.send_omni_request(request_config, request_num=1),
submit_request=lambda: online_client_function.send_omni_request(request_config, request_num=1),
inject_fault=lambda: fault_injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -704,14 +707,14 @@ def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup
@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
omni_server_function,
openai_client_function,
online_client_function,
signal_name: str,
) -> None:
request_config = _chat_request_config(omni_server_function)
scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: openai_client_function.send_omni_request(request_config, request_num=1),
submit_request=lambda: online_client_function.send_omni_request(request_config, request_num=1),
inject_fault=lambda: injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -764,7 +767,7 @@ def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
@pytest.mark.parametrize("omni_server_function", QWEN_PARAMS, indirect=True)
def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
omni_server_function,
openai_client_function,
online_client_function,
signal_name: str,
) -> None:
request_config = _chat_request_config(omni_server_function)
@@ -775,7 +778,7 @@ def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
inter_kill_wait_seconds=0.1,
)
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: openai_client_function.send_omni_request(request_config, request_num=1),
submit_request=lambda: online_client_function.send_omni_request(request_config, request_num=1),
inject_fault=lambda: injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+15 -12
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Wan2.2 reliability integration tests.
"""
@@ -183,7 +186,7 @@ def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -
@pytest.mark.slow
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_function, openai_client_function) -> None:
def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_function, online_client_function) -> None:
stage_config_path = getattr(omni_server_function, "stage_config_path", None)
device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
handle = inject_gpu_oom(
@@ -211,7 +214,7 @@ def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_funct
"stream": False,
}
try:
openai_client_function.send_video_diffusion_request(request_config, request_num=1)
online_client_function.send_video_diffusion_request(request_config, request_num=1)
except Exception as exc:
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
else:
@@ -230,11 +233,11 @@ def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_funct
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
def test_reliability_fault_process_kill_request_failure(
omni_server_after_fault_function,
openai_client_function,
online_client_function,
) -> None:
request_config = _video_request_config()
try:
openai_client_function.send_video_diffusion_request(request_config, request_num=1)
online_client_function.send_video_diffusion_request(request_config, request_num=1)
except Exception as exc:
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
else:
@@ -251,7 +254,7 @@ def test_reliability_fault_process_kill_request_failure(
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
omni_server_after_fault_function,
openai_client_function,
online_client_function,
) -> None:
"""Black-box: after process kill, /v1/videos fails fast and concurrent calls don't hang.
@@ -308,7 +311,7 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
start = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
futures = [
executor.submit(openai_client_function.send_video_diffusion_request, request_config, 1) for _ in range(3)
executor.submit(online_client_function.send_video_diffusion_request, request_config, 1) for _ in range(3)
]
done, pending = concurrent.futures.wait(
futures,
@@ -367,13 +370,13 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
def test_reliability_fault_process_kill_worker_with_load_request_failure(
omni_server_function,
openai_client_function,
online_client_function,
fault_injector: FaultInjector,
) -> None:
request_config = _video_request_config()
scenario = "kill_worker_with_load"
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: openai_client_function.send_video_diffusion_request(request_config, request_num=1),
submit_request=lambda: online_client_function.send_video_diffusion_request(request_config, request_num=1),
inject_fault=lambda: fault_injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -516,7 +519,7 @@ def test_reliability_video_oom_recovers_after_fault_removed(
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
omni_server_function,
openai_client_function,
online_client_function,
signal_name: str,
) -> None:
"""Black-box: during in-flight video requests, kill serve root and verify post-fault behavior/cleanup."""
@@ -524,7 +527,7 @@ def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_clean
scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: openai_client_function.send_video_diffusion_request(request_config, request_num=1),
submit_request=lambda: online_client_function.send_video_diffusion_request(request_config, request_num=1),
inject_fault=lambda: injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
@@ -601,7 +604,7 @@ def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
omni_server_function,
openai_client_function,
online_client_function,
signal_name: str,
) -> None:
"""Black-box: during in-flight video requests, kill server process tree and verify post-fault behavior/cleanup."""
@@ -613,7 +616,7 @@ def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
inter_kill_wait_seconds=0.1,
)
load_result = run_fault_injection_with_rate_load(
submit_request=lambda: openai_client_function.send_video_diffusion_request(request_config, request_num=1),
submit_request=lambda: online_client_function.send_video_diffusion_request(request_config, request_num=1),
inject_fault=lambda: injector(omni_server_function),
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,