[CI] invalid_param reliability suite and weekly http_invalid jobs (#3652)
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed
1 parent
e277feacaf
commit
bd83ac9b4b
18 files changed
+2647
-184
No files matched your search
@@ -0,0 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared env for ``http_invalid`` (real :func:`omni_server`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
# Match ``tests/e2e/online_serving/*`` module-level env for subprocess serve.
|
||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
os.environ.setdefault("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tiny_png_bytes() -> bytes:
|
||||
img = Image.new("RGB", (32, 32), color="gray")
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``POST /v1/audio/generate`` validation (live Stable Audio Open server)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
|
||||
|
||||
# https://github.com/vllm-project/vllm-omni/issues/3649 — invalid inputs still HTTP 200 / timeout / missing validation.
|
||||
_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
|
||||
|
||||
_PARAMS = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="stabilityai/stable-audio-open-1.0",
|
||||
server_args=[
|
||||
"--trust-remote-code",
|
||||
"--enforce-eager",
|
||||
"--model-class-name",
|
||||
"StableAudioPipeline",
|
||||
],
|
||||
),
|
||||
id="stable_audio_open",
|
||||
marks=hardware_marks(res={"cuda": "L4"}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, err_message",
|
||||
[
|
||||
pytest.param({"input": ""}, ("cannot be empty", "input"), id="input_empty", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
{"input": " "}, ("cannot be empty", "input"), id="input_whitespace_only", marks=_SKIP_ISSUE_3649
|
||||
),
|
||||
pytest.param({"input": 123}, ("input", "a valid string"), id="input_wrong_type"),
|
||||
pytest.param(
|
||||
{"response_format": "mpeg"},
|
||||
("response_format", "literal_error", "Input should be"),
|
||||
id="response_format_invalid",
|
||||
),
|
||||
pytest.param({"speed": 0.1}, ("speed", "greater_than_equal", "0.25"), id="speed_too_low"),
|
||||
pytest.param({"speed": 99.0}, ("speed", "less_than_equal", "4"), id="speed_too_high"),
|
||||
pytest.param({"stream_format": 1}, ("stream_format", "literal_error", "audio"), id="stream_format_wrong_type"),
|
||||
pytest.param(
|
||||
{"stream_format": "sse"},
|
||||
("sse", "stream_format", "value_error", "not a supported stream_format"),
|
||||
id="stream_format_sse_blocked",
|
||||
),
|
||||
pytest.param(
|
||||
{"audio_length": -1.0}, ("audio_end_in_s", "higher than", "audio_start_in_s"), id="audio_length_negative"
|
||||
),
|
||||
pytest.param({"audio_length": 0}, "audio_length", id="audio_length_zero", marks=_SKIP_ISSUE_3649),
|
||||
# ``OmniOpenAIServingAudioGenerate`` only forwards ``audio_start`` when ``audio_length`` is set.
|
||||
pytest.param(
|
||||
{"audio_start": -0.5, "audio_length": 5.0},
|
||||
("audio_start_in_s", "between", "0", "512"),
|
||||
id="audio_start_negative",
|
||||
),
|
||||
pytest.param(
|
||||
{"negative_prompt": ["noise"]}, ("negative_prompt", "a valid string"), id="negative_prompt_wrong_type"
|
||||
),
|
||||
pytest.param({"guidance_scale": -1.0}, "guidance_scale", id="guidance_scale_negative", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param({"guidance_scale": 0}, "guidance_scale", id="guidance_scale_zero", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
{"num_inference_steps": 0}, "num_inference_steps", id="num_inference_steps_zero", marks=_SKIP_ISSUE_3649
|
||||
),
|
||||
pytest.param(
|
||||
{"num_inference_steps": -1}, ("number of steps", "non-negative"), id="num_inference_steps_negative"
|
||||
),
|
||||
pytest.param(
|
||||
{"num_inference_steps": 6000},
|
||||
"num_inference_steps",
|
||||
id="num_inference_steps_above_max",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
{"guidance_scale": 1001.0}, "guidance_scale", id="guidance_scale_above_max", marks=_SKIP_ISSUE_3649
|
||||
),
|
||||
pytest.param(
|
||||
{"audio_length": 86401.0}, ("Requested audio length", "exceeds maximum"), id="audio_length_above_max"
|
||||
),
|
||||
pytest.param(
|
||||
{"audio_start": 86401.0, "audio_length": 1.0},
|
||||
("audio_start_in_s", "between", "0", "512"),
|
||||
id="audio_start_above_max",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
|
||||
def test_audio_generate_invalid_field_values(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
overrides: dict[str, object],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
body = {"model": omni_server.model, "input": "ambient electronic pad"}
|
||||
body.update(overrides)
|
||||
openai_client.send_audio_generate_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
|
||||
def test_audio_generate_missing_input(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_audio_generate_http_request(
|
||||
{
|
||||
"json": {"model": omni_server.model},
|
||||
"timeout": 120,
|
||||
"err_code": 400,
|
||||
"err_message": ("missing", "input", "field required"),
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,862 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HTTP + WebSocket validation for Qwen3-TTS Base: ``/v1/audio/speech``, ``/v1/audio/speech/stream``, batch, voices."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import uuid
|
||||
import wave
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.media import load_test_audio_data_url
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
# Speech / voice-upload numeric caps for these tests only (not exported from vllm_omni).
|
||||
# Align where possible with serving_speech defaults: _TTS_MAX_INSTRUCTIONS_LENGTH (500),
|
||||
# _TTS_MAX_NEW_TOKENS_MAX (4096). Voice form max lengths are not public constants in prod;
|
||||
# values below keep tests self-contained and deterministic.
|
||||
_SPEECH_API_MAX_INSTRUCTIONS_CHARS = 500
|
||||
_SPEECH_API_MAX_NEW_TOKENS = 4096
|
||||
_VOICE_UPLOAD_MAX_CONSENT_LEN = 1024
|
||||
_VOICE_UPLOAD_MAX_REF_TEXT_CHARS = 8192
|
||||
_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS = 2048
|
||||
REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
|
||||
REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
|
||||
# Qwen3-TTS 0.6B expects 1024-dim speaker embeddings (see serving_speech / talker hidden_size).
|
||||
_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM = 1024
|
||||
# Pop-only marker for ``test_speech_invalid_field_values``: omit ``ref_audio`` / ``ref_text`` so Base
|
||||
# ``speaker_embedding`` cases mirror standalone embedding-only JSON (no ref_audio clash).
|
||||
_SPEECH_INVALID_EMBEDDING_ONLY_BODY = "__speech_invalid_embedding_only_body__"
|
||||
# Batch-only: merged items inherit batch ``ref_audio`` / ``ref_text`` because ``_pick`` treats item ``None``
|
||||
# as "unset". Strip clone fields from the batch envelope before asserting CustomVoice preset errors.
|
||||
_SPEECH_BATCH_NO_CLONE_FIELDS = "__speech_batch_no_clone_fields__"
|
||||
|
||||
_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
|
||||
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
|
||||
_L4_SPEECH_HW = hardware_marks(res={"cuda": "L4"})
|
||||
_SPEECH_SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
|
||||
|
||||
_QWEN3_TTS_SPEECH = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="Qwen/Qwen3-TTS-12Hz-0.6B-Base",
|
||||
stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
|
||||
server_args=_SPEECH_SERVER_ARGS,
|
||||
),
|
||||
id="qwen3_tts",
|
||||
marks=_L4_SPEECH_HW,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
|
||||
if loc == "batch":
|
||||
body.update(overrides)
|
||||
return
|
||||
items = body["items"]
|
||||
assert isinstance(items, list) and len(items) >= 1
|
||||
first = items[0]
|
||||
assert isinstance(first, dict)
|
||||
body["items"] = [{**first, **overrides}]
|
||||
|
||||
|
||||
def _pcm_wav_mono_bytes(*, duration_s: float = 1.1, sample_rate: int = 44100) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with wave.open(buf, "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(sample_rate)
|
||||
nframes = max(1, int(sample_rate * duration_s))
|
||||
wf.writeframes(b"\x00\x00" * nframes)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/audio/speech (JSON)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_malformed_json(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_audio_speech_http_request(
|
||||
{
|
||||
"raw_body": "{",
|
||||
"timeout": 120,
|
||||
"err_code": 400,
|
||||
"err_message": ("json", "decode", "invalid"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_missing_required_fields(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_audio_speech_http_request(
|
||||
{
|
||||
"json": {"model": omni_server.model},
|
||||
"timeout": 120,
|
||||
"err_code": 400,
|
||||
"err_message": ("input", "field required", "missing"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, err_message",
|
||||
[
|
||||
pytest.param({"input": ""}, ("input", "empty"), id="input_empty"),
|
||||
pytest.param({"input": " "}, ("input", "empty"), id="input_whitespace_only"),
|
||||
pytest.param({"voice": ""}, "Invalid voice", id="voice_empty", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
{"instructions": 123}, ("instructions", "string_type", "valid string"), id="instructions_wrong_type"
|
||||
),
|
||||
pytest.param(
|
||||
{"response_format": "mpeg"}, ("response_format", "literal_error", "wav"), id="response_format_invalid"
|
||||
),
|
||||
pytest.param({"stream_format": 1}, ("stream_format", "literal_error", "audio"), id="stream_format_wrong_type"),
|
||||
pytest.param(
|
||||
{"stream_format": "sse"}, ("stream_format", "value_error", "audio"), id="stream_format_sse_blocked"
|
||||
),
|
||||
pytest.param({"stream": "wrong_type"}, ("stream", "bool_parsing", "validation error"), id="stream_wrong_type"),
|
||||
pytest.param(
|
||||
{
|
||||
"voice": "vivian",
|
||||
"task_type": "CustomVoice",
|
||||
"ref_audio": None,
|
||||
"ref_text": None,
|
||||
},
|
||||
("CustomVoice", "does not support"),
|
||||
id="customvoice_task_not_supported",
|
||||
),
|
||||
pytest.param(
|
||||
{"task_type": "InvalidEnum"}, ("task_type", "literal_error", "CustomVoice"), id="task_type_invalid"
|
||||
),
|
||||
pytest.param({"language": ""}, ("language", "invalid language", "chinese"), id="language_empty"),
|
||||
pytest.param({"ref_audio": "ftp://example.com/a.wav"}, ("ref_audio", "URL"), id="ref_audio_bad_scheme"),
|
||||
pytest.param(
|
||||
{"ref_audio": "not_a_valid_uri"},
|
||||
("ref_audio", "url"),
|
||||
id="ref_audio_invalid_uri",
|
||||
),
|
||||
pytest.param({"ref_text": ["x"]}, ("ref_text", "string_type", "valid string"), id="ref_text_wrong_type"),
|
||||
pytest.param(
|
||||
{"x_vector_only_mode": "wrong"},
|
||||
("x_vector_only_mode", "valid boolean"),
|
||||
id="x_vector_only_mode_string_not_boolean",
|
||||
),
|
||||
pytest.param(
|
||||
{"speaker_embedding": [0.01] * _QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM},
|
||||
("mutually exclusive", "speaker_embedding", "ref_audio"),
|
||||
id="speaker_embedding_with_ref_audio_mutually_exclusive",
|
||||
),
|
||||
pytest.param({"max_new_tokens": 0}, ("max_new_tokens", "least 1"), id="max_new_tokens_below_min"),
|
||||
pytest.param(
|
||||
{"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
|
||||
("max_new_tokens", "exceed 4096"),
|
||||
id="max_new_tokens_above_max",
|
||||
),
|
||||
pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
|
||||
pytest.param({"seed": 2**63}, ("seed", "less_than_equal", "9223372036854775807"), id="seed_above_max"),
|
||||
pytest.param(
|
||||
{"initial_codec_chunk_frames": -1},
|
||||
("initial_codec_chunk_frames", "greater_than_equal", "0"),
|
||||
id="initial_codec_chunk_frames_negative",
|
||||
),
|
||||
pytest.param(
|
||||
{"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
|
||||
("instructions", "too long"),
|
||||
id="instructions_exceed_api_limit",
|
||||
),
|
||||
# Base clone: whitespace-only ref_text fails serving validation (not pydantic).
|
||||
pytest.param(
|
||||
{
|
||||
"task_type": "Base",
|
||||
"ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
|
||||
"ref_text": " ",
|
||||
},
|
||||
("ref_text", "base task", "non-empty"),
|
||||
id="ref_text_whitespace_only_base_clone",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
_SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
|
||||
"task_type": "Base",
|
||||
"speaker_embedding": [],
|
||||
},
|
||||
("speaker_embedding", "non-empty"),
|
||||
id="base_speaker_embedding_empty",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
_SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
|
||||
"task_type": "Base",
|
||||
"x_vector_only_mode": True,
|
||||
"speaker_embedding": [0.01] * 512,
|
||||
},
|
||||
("speaker_embedding", "dimensions", str(_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM)),
|
||||
id="base_speaker_embedding_wrong_dim_short",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
_SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
|
||||
"task_type": "Base",
|
||||
"x_vector_only_mode": True,
|
||||
"speaker_embedding": [0.01] * 2048,
|
||||
},
|
||||
("speaker_embedding", "2048", "dimensions", str(_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM)),
|
||||
id="base_speaker_embedding_wrong_dim_long",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_invalid_field_values(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
overrides: dict[str, object],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
ov = dict(overrides)
|
||||
embedding_only = bool(ov.pop(_SPEECH_INVALID_EMBEDDING_ONLY_BODY, False))
|
||||
body = {
|
||||
"model": omni_server.model,
|
||||
"input": "Hello.",
|
||||
"voice": "clone",
|
||||
"language": "English",
|
||||
"response_format": "wav",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"ref_text": REF_TEXT,
|
||||
}
|
||||
if embedding_only:
|
||||
body.pop("ref_audio", None)
|
||||
body.pop("ref_text", None)
|
||||
body.update(ov)
|
||||
openai_client.send_audio_speech_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/audio/speech/batch (JSON)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_batch_empty_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_audio_speech_batch_http_request(
|
||||
{"json": {"items": []}, "timeout": 120, "err_code": (400, 422), "err_message": ("items", "least 1 item")}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_batch_missing_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_audio_speech_batch_http_request(
|
||||
{
|
||||
"json": {"model": omni_server.model},
|
||||
"timeout": 120,
|
||||
"err_code": (400, 422),
|
||||
"err_message": ("items", "field required", "missing"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# Align field-level checks with ``POST /v1/audio/speech`` where ``BatchSpeechRequest`` /
|
||||
# ``SpeechBatchItem`` expose the same knobs. Not on batch schema (single-speech only):
|
||||
# ``stream``, ``stream_format``, ``seed``, ``speaker_embedding`` — see dedicated speech tests.
|
||||
@pytest.mark.parametrize(
|
||||
"loc, overrides, err_message",
|
||||
[
|
||||
pytest.param("item", {"input": ""}, ("input", "empty"), id="item_input_empty", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"input": " "},
|
||||
("input", "empty"),
|
||||
id="item_input_whitespace_only",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param("batch", {"voice": ""}, "Invalid voice", id="batch_voice_empty", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param("item", {"voice": ""}, "Invalid voice", id="item_voice_empty", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{
|
||||
"voice": "nonexistent_voice_xyz",
|
||||
"task_type": "CustomVoice",
|
||||
"ref_audio": None,
|
||||
"ref_text": None,
|
||||
},
|
||||
"Invalid voice",
|
||||
id="batch_voice_unknown_preset",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{
|
||||
_SPEECH_BATCH_NO_CLONE_FIELDS: True,
|
||||
"voice": "nonexistent_voice_xyz",
|
||||
"task_type": "CustomVoice",
|
||||
},
|
||||
"Invalid voice",
|
||||
id="item_voice_unknown_preset",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"instructions": 123},
|
||||
("instructions", "string_type", "valid string"),
|
||||
id="batch_instructions_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"instructions": 123},
|
||||
("instructions", "string_type", "valid string"),
|
||||
id="item_instructions_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
|
||||
("instructions", "too long"),
|
||||
id="batch_instructions_exceed_api_limit",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
|
||||
("instructions", "too long"),
|
||||
id="item_instructions_exceed_api_limit",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"response_format": "mpeg"},
|
||||
("response_format", "literal_error", "wav"),
|
||||
id="batch_response_format_invalid",
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"response_format": "mpeg"},
|
||||
("response_format", "literal_error", "wav"),
|
||||
id="item_response_format_invalid",
|
||||
),
|
||||
pytest.param("batch", {"speed": 0.1}, ("speed", "greater_than_equal", "0.25"), id="batch_speed_too_low"),
|
||||
pytest.param("item", {"speed": 10.0}, ("speed", "less_than_equal", "4"), id="item_speed_too_high"),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"task_type": "InvalidEnum"},
|
||||
("task_type", "literal_error", "CustomVoice"),
|
||||
id="batch_task_type_invalid",
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"task_type": "InvalidEnum"},
|
||||
("task_type", "literal_error", "CustomVoice"),
|
||||
id="item_task_type_invalid",
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"language": ""},
|
||||
("language", "invalid language", "chinese"),
|
||||
id="batch_language_empty",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"language": ""},
|
||||
("language", "invalid language", "chinese"),
|
||||
id="item_language_empty",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"ref_audio": "ftp://example.com/a.wav"},
|
||||
("ref_audio", "URL"),
|
||||
id="batch_ref_audio_bad_scheme",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"ref_audio": "ftp://example.com/a.wav"},
|
||||
("ref_audio", "URL"),
|
||||
id="item_ref_audio_bad_scheme",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"ref_audio": "not_a_valid_uri"},
|
||||
("ref_audio", "url"),
|
||||
id="batch_ref_audio_invalid_uri",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"ref_audio": "not_a_valid_uri"},
|
||||
("ref_audio", "url"),
|
||||
id="item_ref_audio_invalid_uri",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"ref_text": ["x"]},
|
||||
("ref_text", "string_type", "valid string"),
|
||||
id="batch_ref_text_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"ref_text": ["x"]},
|
||||
("ref_text", "string_type", "valid string"),
|
||||
id="item_ref_text_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"x_vector_only_mode": "wrong_type"},
|
||||
("x_vector_only_mode", "bool_parsing", "validation error"),
|
||||
id="batch_x_vector_only_mode_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"x_vector_only_mode": "wrong_type"},
|
||||
("x_vector_only_mode", "bool_parsing", "validation error"),
|
||||
id="item_x_vector_only_mode_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"max_new_tokens": 0},
|
||||
("max_new_tokens", "least 1"),
|
||||
id="batch_max_new_tokens_below_min",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"max_new_tokens": 0},
|
||||
("max_new_tokens", "least 1"),
|
||||
id="item_max_new_tokens_below_min",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
|
||||
("max_new_tokens", "exceed 4096"),
|
||||
id="batch_max_new_tokens_above_max",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
|
||||
("max_new_tokens", "exceed 4096"),
|
||||
id="item_max_new_tokens_above_max",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"batch",
|
||||
{"initial_codec_chunk_frames": -1},
|
||||
("initial_codec_chunk_frames", "greater_than_equal", "0"),
|
||||
id="batch_initial_codec_chunk_frames_negative",
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{"initial_codec_chunk_frames": -1},
|
||||
("initial_codec_chunk_frames", "greater_than_equal", "0"),
|
||||
id="item_initial_codec_chunk_frames_negative",
|
||||
),
|
||||
# Base clone: whitespace-only ref_text (same serving check as single speech).
|
||||
pytest.param(
|
||||
"batch",
|
||||
{
|
||||
"task_type": "Base",
|
||||
"ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
|
||||
"ref_text": " ",
|
||||
},
|
||||
("ref_text", "base task", "non-empty"),
|
||||
id="batch_ref_text_whitespace_only_base_clone",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"item",
|
||||
{
|
||||
"task_type": "Base",
|
||||
"ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
|
||||
"ref_text": " ",
|
||||
},
|
||||
("ref_text", "base task", "non-empty"),
|
||||
id="item_ref_text_whitespace_only_base_clone",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_batch_invalid_field_values(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
loc: str,
|
||||
overrides: dict[str, object],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
ov = dict(overrides)
|
||||
strip_clone = bool(ov.pop(_SPEECH_BATCH_NO_CLONE_FIELDS, False))
|
||||
body = {
|
||||
"model": omni_server.model,
|
||||
"voice": "clone",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"ref_text": REF_TEXT,
|
||||
"language": "English",
|
||||
"response_format": "wav",
|
||||
"items": [{"input": "Hello."}],
|
||||
}
|
||||
if strip_clone:
|
||||
body.pop("ref_audio", None)
|
||||
body.pop("ref_text", None)
|
||||
_apply_batch_overrides(body, loc=loc, overrides=ov)
|
||||
openai_client.send_audio_speech_batch_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# WebSocket /v1/audio/speech/stream
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ``test_speech_stream_invalid_requests``: build ``session.config`` JSON with ``omni_server.model``.
|
||||
_SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES = object()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"send_frames_spec, err_message",
|
||||
[
|
||||
pytest.param("{not-json", "Invalid JSON", id="invalid_json"),
|
||||
pytest.param(
|
||||
json.dumps({"type": "input.text", "text": "hello"}),
|
||||
"Expected session.config",
|
||||
id="wrong_first_message_type",
|
||||
),
|
||||
pytest.param(
|
||||
_SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES,
|
||||
("invalid session config", "response_format", "pcm"),
|
||||
id="session_config_stream_audio_requires_pcm",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_speech_stream_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
send_frames_spec: Any,
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
"""Bad first WebSocket text frames: non-JSON, wrong ``type``, or invalid ``session.config``."""
|
||||
if send_frames_spec is _SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES:
|
||||
send_frames = json.dumps(
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": omni_server.model,
|
||||
"response_format": "wav",
|
||||
"stream_audio": True,
|
||||
}
|
||||
)
|
||||
else:
|
||||
send_frames = send_frames_spec
|
||||
assert isinstance(send_frames, str)
|
||||
openai_client.send_audio_speech_stream_ws_request(
|
||||
{
|
||||
"send_frames": send_frames,
|
||||
"timeout": 120,
|
||||
"ws_max_size": None,
|
||||
"ws_json_type": "error",
|
||||
"err_message": err_message,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/audio/voices (multipart + embedding-only) and DELETE /v1/audio/voices/{name}
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Templates for ``test_voices_create_invalid_requests``: resolved in ``_finalize_voices_form_data``.
|
||||
_VF_LONG_CONSENT = object()
|
||||
_VF_LONG_REF_TEXT = object()
|
||||
_VF_LONG_SPEAKER_DESC = object()
|
||||
_VF_EMBEDDING_NAN_JSON = object()
|
||||
|
||||
|
||||
def _finalize_voices_form_data(template: dict[str, Any], uuid_hex: str) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {}
|
||||
for key, val in template.items():
|
||||
if val is _VF_LONG_CONSENT:
|
||||
out[key] = "x" * (_VOICE_UPLOAD_MAX_CONSENT_LEN + 1)
|
||||
elif val is _VF_LONG_REF_TEXT:
|
||||
out[key] = "y" * (_VOICE_UPLOAD_MAX_REF_TEXT_CHARS + 1)
|
||||
elif val is _VF_LONG_SPEAKER_DESC:
|
||||
out[key] = "z" * (_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS + 1)
|
||||
elif val is _VF_EMBEDDING_NAN_JSON:
|
||||
out[key] = json.dumps([0.1] * 1023 + [float("nan")])
|
||||
elif isinstance(val, str):
|
||||
out[key] = val.replace("{uuid}", uuid_hex)
|
||||
else:
|
||||
out[key] = val
|
||||
return out
|
||||
|
||||
|
||||
def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
|
||||
"""``None`` => no multipart file (embedding-only or neither)."""
|
||||
if kind is None:
|
||||
return None
|
||||
if kind == "wav_ok":
|
||||
return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(), "audio/wav")}
|
||||
if kind == "wav_short":
|
||||
return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(duration_s=0.3), "audio/wav")}
|
||||
if kind == "wav_bad_body":
|
||||
return {"audio_sample": ("clip.wav", b"not valid wav content", "audio/wav")}
|
||||
if kind == "wav_pdf_type":
|
||||
return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(), "application/pdf")}
|
||||
raise AssertionError(f"unknown multipart kind {kind!r}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"multipart_kind, form_template, err_message",
|
||||
[
|
||||
pytest.param(
|
||||
None,
|
||||
{"consent": "c1", "name": "v1"},
|
||||
("audio_sample", "speaker_embedding", "provided"),
|
||||
id="neither_audio_nor_embedding",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"name": "v_miss_consent_{uuid}"},
|
||||
("missing", "consent", "field required"),
|
||||
id="missing_consent",
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": "consent_ok"},
|
||||
("missing", "name", "field required"),
|
||||
id="missing_name",
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": "", "name": "v_empty_consent_{uuid}"},
|
||||
("missing", "consent", "field required"),
|
||||
id="empty_consent",
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": " ", "name": "v_ws_consent_{uuid}"},
|
||||
"consent",
|
||||
id="whitespace_consent",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": "bad/consent", "name": "v_bad_cons_{uuid}"},
|
||||
"consent",
|
||||
id="consent_path_sep",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": _VF_LONG_CONSENT, "name": "v_long_cons_{uuid}"},
|
||||
("consent", "too long", "Failed to save"),
|
||||
id="consent_too_long",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": "consent_ok", "name": ""},
|
||||
("missing", "name", "field required"),
|
||||
id="empty_name",
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": "consent_ok", "name": "evil/name"},
|
||||
("voice name", "invalid voice name"),
|
||||
id="invalid_name_path_sep",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{"consent": "consent_ok", "name": "v_long_ref_{uuid}", "ref_text": _VF_LONG_REF_TEXT},
|
||||
"ref_text",
|
||||
id="ref_text_too_long",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{
|
||||
"consent": "consent_ok",
|
||||
"name": "v_long_desc_{uuid}",
|
||||
"speaker_description": _VF_LONG_SPEAKER_DESC,
|
||||
},
|
||||
"speaker_description",
|
||||
id="speaker_description_too_long",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_pdf_type",
|
||||
{"consent": "consent_ok", "name": "v_bad_mime_{uuid}"},
|
||||
("mime", "unsupported", "audio/mp4"),
|
||||
id="audio_unsupported_mime",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_short",
|
||||
{"consent": "consent_ok", "name": "v_short_{uuid}"},
|
||||
("too short", "reference audio"),
|
||||
id="audio_too_short",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_bad_body",
|
||||
{"consent": "consent_ok", "name": "v_bad_wav_{uuid}"},
|
||||
("decode", "audio", "Format not recognised"),
|
||||
id="audio_decode_error",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
{
|
||||
"speaker_embedding": "not valid json [[[",
|
||||
"consent": "consent_emb",
|
||||
"name": "v_bad_json_{uuid}",
|
||||
},
|
||||
("speaker_embedding", "valid JSON"),
|
||||
id="speaker_embedding_invalid_json",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
{
|
||||
"speaker_embedding": json.dumps([]),
|
||||
"consent": "consent_emb",
|
||||
"name": "v_empty_emb_{uuid}",
|
||||
},
|
||||
("speaker_embedding", "non-empty"),
|
||||
id="speaker_embedding_empty",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
{
|
||||
"speaker_embedding": _VF_EMBEDDING_NAN_JSON,
|
||||
"consent": "consent_emb",
|
||||
"name": "v_nan_emb_{uuid}",
|
||||
},
|
||||
("speaker_embedding", "finite"),
|
||||
id="speaker_embedding_nan",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
{
|
||||
"speaker_embedding": json.dumps([0.01] * 512),
|
||||
"consent": "consent_emb",
|
||||
"name": "v_wrong_dim_{uuid}",
|
||||
},
|
||||
("expected", "dimensions", "1024"),
|
||||
id="speaker_embedding_wrong_dim",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"wav_ok",
|
||||
{
|
||||
"consent": "consent_emb",
|
||||
"name": "v_both_{uuid}",
|
||||
"speaker_embedding": json.dumps([0.01] * 1024),
|
||||
},
|
||||
("mutually exclusive", "audio_sample", "speaker_embedding"),
|
||||
id="audio_and_embedding_mutually_exclusive",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_voices_create_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
multipart_kind: str | None,
|
||||
form_template: dict[str, Any],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
"""Invalid ``POST /v1/audio/voices`` bodies (multipart or embedding-only)."""
|
||||
uid = uuid.uuid4().hex[:8]
|
||||
cfg: dict[str, Any] = {
|
||||
"data": _finalize_voices_form_data(form_template, uid),
|
||||
"timeout": 120,
|
||||
"err_code": 400,
|
||||
"err_message": err_message,
|
||||
}
|
||||
files = _voices_upload_multipart_files(multipart_kind)
|
||||
if files is not None:
|
||||
cfg["files"] = files
|
||||
openai_client.send_audio_voices_create_http_request(cfg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"voice_name, err_code, err_message",
|
||||
[
|
||||
pytest.param(
|
||||
"missing-voice-404",
|
||||
404,
|
||||
"not found",
|
||||
id="not_found",
|
||||
),
|
||||
pytest.param(
|
||||
" ",
|
||||
404,
|
||||
"not found",
|
||||
id="whitespace_only",
|
||||
),
|
||||
pytest.param(
|
||||
"evil/name",
|
||||
404,
|
||||
"not found",
|
||||
id="path_separator",
|
||||
),
|
||||
pytest.param(
|
||||
".",
|
||||
405,
|
||||
("Method Not Allowed", "detail"),
|
||||
id="dot_single",
|
||||
),
|
||||
pytest.param(
|
||||
"..",
|
||||
404,
|
||||
"not found",
|
||||
id="dot_dot",
|
||||
),
|
||||
pytest.param(
|
||||
"bad\x00voice",
|
||||
404,
|
||||
"not found",
|
||||
id="nul_byte",
|
||||
),
|
||||
pytest.param(
|
||||
"",
|
||||
405,
|
||||
"Method Not Allowed",
|
||||
id="empty_name_path",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
|
||||
def test_voices_delete_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
voice_name: str,
|
||||
err_code: int | tuple[int, ...],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
"""DELETE ``/v1/audio/voices/{name}``: missing voice (404), odd segments (often 404), ``.``paths (405)."""
|
||||
openai_client.send_audio_voices_delete_http_request(
|
||||
{
|
||||
"name": voice_name,
|
||||
"timeout": 120,
|
||||
"err_code": err_code,
|
||||
"err_message": err_message,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``POST /v1/images/edits`` multipart validation (live ``Qwen/Qwen-Image-Edit``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
|
||||
|
||||
_PARAMS = [
|
||||
pytest.param(
|
||||
OmniServerParams(model="Qwen/Qwen-Image-Edit"),
|
||||
id="qwen_image_edit",
|
||||
marks=hardware_marks(res={"cuda": "H100"}),
|
||||
),
|
||||
]
|
||||
|
||||
# Populated in ``test_images_edits_invalid_requests`` from ``omni_server.model``.
|
||||
_IMAGE_EDITS_SERVER_MODEL = object()
|
||||
|
||||
|
||||
def _finalize_edits_form_data(template: dict[str, Any], server_model: str) -> dict[str, Any]:
|
||||
return {k: (server_model if v is _IMAGE_EDITS_SERVER_MODEL else v) for k, v in template.items()}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"include_image, data_template, err_message, err_code",
|
||||
[
|
||||
pytest.param(
|
||||
False,
|
||||
{"prompt": "make it brighter", "model": _IMAGE_EDITS_SERVER_MODEL},
|
||||
("image", "required", "Unprocessable Entity"),
|
||||
422,
|
||||
id="missing_image",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{"prompt": "edit", "model": "wrong-model-id"},
|
||||
("model", "mismatch"),
|
||||
400,
|
||||
id="model_mismatch",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{"prompt": "edit", "response_format": "url"},
|
||||
("response_format", "b64_json", "supported"),
|
||||
400,
|
||||
id="unsupported_response_format",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "output_compression": "101"},
|
||||
("less_than_equal", "output_compression", "100"),
|
||||
400,
|
||||
id="output_compression_above_max",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
{"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "layers": "2"},
|
||||
("Invalid layers", "3", "1"),
|
||||
400,
|
||||
id="invalid_layers",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
|
||||
def test_images_edits_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
tiny_png_bytes: bytes,
|
||||
include_image: bool,
|
||||
data_template: dict[str, Any],
|
||||
err_message: str | tuple[str, ...],
|
||||
err_code: int | tuple[int, ...],
|
||||
) -> None:
|
||||
cfg: dict[str, Any] = {
|
||||
"data": _finalize_edits_form_data(data_template, omni_server.model),
|
||||
"timeout": 300,
|
||||
"err_code": err_code,
|
||||
"err_message": err_message,
|
||||
}
|
||||
if include_image:
|
||||
cfg["files"] = {"image": ("x.png", tiny_png_bytes, "image/png")}
|
||||
openai_client.send_images_edits_http_request(cfg)
|
||||
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``POST /v1/images/generations`` validation (live ``Qwen/Qwen-Image``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
|
||||
|
||||
_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
|
||||
|
||||
_PARAMS = [
|
||||
pytest.param(
|
||||
OmniServerParams(model="Qwen/Qwen-Image"),
|
||||
id="qwen_image",
|
||||
marks=hardware_marks(res={"cuda": "H100"}),
|
||||
),
|
||||
]
|
||||
|
||||
# Full replacement body for ``test_images_generations_invalid_requests`` (no minimal prompt/model merge).
|
||||
_IMAGES_GEN_MISSING_PROMPT_JSON = object()
|
||||
|
||||
|
||||
def _minimal_images_gen_json(omni_server: OmniServer) -> dict[str, object]:
|
||||
return {"prompt": "a simple red apple icon", "model": omni_server.model}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/images/generations (JSON)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body_spec, err_message",
|
||||
[
|
||||
pytest.param(_IMAGES_GEN_MISSING_PROMPT_JSON, ("prompt", "Field required", "Missing"), id="missing_prompt"),
|
||||
pytest.param(
|
||||
{"prompt": "cat", "model": "not-the-loaded-checkpoint"},
|
||||
("model", "mismatch"),
|
||||
id="model_mismatch",
|
||||
),
|
||||
pytest.param({"prompt": ""}, ("prompt", "empty"), id="prompt_empty", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param({"prompt": " "}, ("prompt", "empty"), id="prompt_whitespace_only", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param({"prompt": 123}, ("prompt", "string_type", "valid string"), id="prompt_wrong_type"),
|
||||
pytest.param({"n": 0}, ("n", "greater_than_equal", "1"), id="n_below_min"),
|
||||
pytest.param({"n": 11}, ("n", "less_than_equal", "10"), id="n_above_max"),
|
||||
pytest.param({"size": "1024"}, ("size", "value_error", "WIDTHxHEIGHT"), id="size_missing_height_token"),
|
||||
pytest.param({"size": "not-a-size"}, ("size", "value_error", "WIDTHxHEIGHT"), id="size_missing_separator"),
|
||||
pytest.param({"size": "1024xabc"}, ("Invalid size format", "integers"), id="size_non_integer_height"),
|
||||
pytest.param({"size": "1024x"}, ("Invalid size format", "integers"), id="size_incomplete_height"),
|
||||
pytest.param({"size": "0x1024"}, ("Invalid size", "positive integers"), id="size_non_positive_width"),
|
||||
pytest.param(
|
||||
{"response_format": "url"},
|
||||
("response_format", "value_error", "b64_json"),
|
||||
id="response_format_not_b64_json",
|
||||
),
|
||||
pytest.param(
|
||||
{"use_system_prompt": "bogus"},
|
||||
("use_system_prompt", "value_error", "dynamic"),
|
||||
id="use_system_prompt_invalid",
|
||||
),
|
||||
pytest.param(
|
||||
{"num_inference_steps": 0},
|
||||
("num_inference_steps", "'greater_than_equal", "1"),
|
||||
id="num_inference_steps_below_min",
|
||||
),
|
||||
pytest.param(
|
||||
{"num_inference_steps": 201},
|
||||
("num_inference_steps", "less_than_equal", "200"),
|
||||
id="num_inference_steps_above_max",
|
||||
),
|
||||
pytest.param(
|
||||
{"guidance_scale": -1.0}, ("guidance_scale", "greater_than_equal", "0"), id="guidance_scale_negative"
|
||||
),
|
||||
pytest.param(
|
||||
{"guidance_scale": 21.0}, ("guidance_scale", "less_than_equal", "20"), id="guidance_scale_above_max"
|
||||
),
|
||||
pytest.param(
|
||||
{"true_cfg_scale": -1.0}, ("true_cfg_scale", "greater_than_equal", "0"), id="true_cfg_scale_negative"
|
||||
),
|
||||
pytest.param(
|
||||
{"true_cfg_scale": 21.0}, ("true_cfg_scale", "less_than_equal", "20"), id="true_cfg_scale_above_max"
|
||||
),
|
||||
pytest.param({"layers": 2}, ("layers", "value_error", "3", "10"), id="layers_below_min"),
|
||||
pytest.param({"layers": 11}, ("layers", "value_error", "3", "10"), id="layers_above_max"),
|
||||
pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
{"seed": 2**32},
|
||||
("seed", "less_than_equal", "4294967295"),
|
||||
id="seed_above_uint32",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
{"output_format": "gif"},
|
||||
("output_format", "value_error", "b64_json"),
|
||||
id="output_format_invalid",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
{"vae_use_slicing": "wrong_type"},
|
||||
("vae_use_slicing", "bool_parsing", "validation error"),
|
||||
id="vae_use_slicing_wrong_type",
|
||||
),
|
||||
pytest.param({"user": 123}, ("user", "string_type", "valid string"), id="user_wrong_type"),
|
||||
pytest.param(
|
||||
{"negative_prompt": ["noise"]},
|
||||
("negative_prompt", "string_type", "valid string"),
|
||||
id="negative_prompt_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
{"generator_device": 1},
|
||||
("generator_device", "string_type", "valid string"),
|
||||
id="generator_device_wrong_type",
|
||||
),
|
||||
pytest.param(
|
||||
{"lora": {"foo": "bar"}}, ("lora", "both name and path", "required"), id="lora_missing_required_fields"
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
|
||||
def test_images_generations_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
body_spec: Any,
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
"""Invalid JSON bodies for ``POST /v1/images/generations`` (missing prompt, model mismatch, bad fields)."""
|
||||
if body_spec is _IMAGES_GEN_MISSING_PROMPT_JSON:
|
||||
body: dict[str, object] = {"size": "1024x1024"}
|
||||
else:
|
||||
assert isinstance(body_spec, dict)
|
||||
body = _minimal_images_gen_json(omni_server)
|
||||
body.update(body_spec)
|
||||
openai_client.send_images_generations_http_request(
|
||||
{"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
@@ -0,0 +1,216 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Invalid inputs on Qwen3-Omni: ``POST /v1/chat/completions``, ``WS /v1/video/chat/stream``, ``WS /v1/realtime``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.omni]
|
||||
|
||||
_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
|
||||
|
||||
|
||||
def _minimal_chat_json(omni_server: OmniServer) -> dict[str, object]:
|
||||
"""Minimal valid chat body; individual tests override one offending field."""
|
||||
return {
|
||||
"model": omni_server.model,
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
|
||||
_QWEN3_OMNI_SERVER = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
stage_config_path=get_deploy_config_path("qwen3_omni_moe.yaml"),
|
||||
server_args=["--async-chunk"],
|
||||
),
|
||||
id="qwen3_omni",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _chat_completions_request_without_expectations(omni_server: OmniServer, case_id: str) -> dict[str, Any]:
|
||||
"""Build ``send_chat_completions_http_request`` kwargs excluding ``err_code`` / ``err_message``."""
|
||||
if case_id == "malformed_json":
|
||||
return {"raw_body": "{not-json", "timeout": 120}
|
||||
if case_id == "missing_messages":
|
||||
return {"json": {"model": omni_server.model, "stream": False}, "timeout": 120}
|
||||
if case_id == "model_mismatch":
|
||||
return {
|
||||
"json": {
|
||||
"model": "this-model-is-not-served-on-this-instance",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream": False,
|
||||
},
|
||||
"timeout": 120,
|
||||
}
|
||||
body = _minimal_chat_json(omni_server)
|
||||
if case_id == "stream_wrong_type":
|
||||
body["stream"] = "not-a-boolean" # type: ignore[assignment]
|
||||
elif case_id == "temperature_wrong_type":
|
||||
body["temperature"] = "hot" # type: ignore[assignment]
|
||||
elif case_id == "max_tokens_wrong_type":
|
||||
body["max_tokens"] = "not-an-int" # type: ignore[assignment] # type: ignore[assignment]
|
||||
elif case_id == "modalities_list_bad":
|
||||
body["modalities"] = [123] # type: ignore[assignment]
|
||||
elif case_id == "response_format_json_schema_incomplete":
|
||||
body["response_format"] = {"type": "json_schema"}
|
||||
elif case_id == "logprobs_wrong_type":
|
||||
body["logprobs"] = "yes" # type: ignore[assignment]
|
||||
elif case_id == "logprobs_top_without_enabled":
|
||||
body["logprobs"] = False
|
||||
body["top_logprobs"] = 5
|
||||
elif case_id == "speaker_unknown":
|
||||
body["speaker"] = "zz_invalid_qwen3_omni_chat_speaker_xyz"
|
||||
else:
|
||||
raise AssertionError(f"unknown chat completions invalid case_id {case_id!r}")
|
||||
return {"json": body, "timeout": 120}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/chat/completions
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize(
|
||||
"case_id, err_code, err_message",
|
||||
[
|
||||
pytest.param(
|
||||
"malformed_json",
|
||||
400,
|
||||
("json_invalid", "decode error"),
|
||||
id="malformed_json",
|
||||
),
|
||||
pytest.param("missing_messages", 400, ("messages", "missing", "Field required"), id="missing_messages"),
|
||||
pytest.param(
|
||||
"model_mismatch",
|
||||
404,
|
||||
("model", "does not exist", "NotFoundError"),
|
||||
id="model_mismatch",
|
||||
),
|
||||
pytest.param("stream_wrong_type", 400, ("stream", "bool_parsing", "not-a-boolean"), id="invalid_stream_type"),
|
||||
pytest.param(
|
||||
"temperature_wrong_type",
|
||||
400,
|
||||
("temperature", "float_parsing", "a valid number"),
|
||||
id="invalid_temperature_type",
|
||||
),
|
||||
pytest.param(
|
||||
"max_tokens_wrong_type", 400, ("max_tokens", "int_parsing", "integer"), id="invalid_max_tokens_type"
|
||||
),
|
||||
pytest.param(
|
||||
"modalities_list_bad",
|
||||
400,
|
||||
("modalities", "value_error", ""),
|
||||
id="modalities_list_bad_element",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"response_format_json_schema_incomplete",
|
||||
400,
|
||||
("response_format", "value_error", "json_schema"),
|
||||
id="invalid_response_format_json_schema",
|
||||
),
|
||||
pytest.param("logprobs_wrong_type", 400, "logprobs", id="logprobs_wrong_type", marks=_SKIP_ISSUE_3649),
|
||||
pytest.param(
|
||||
"logprobs_top_without_enabled",
|
||||
400,
|
||||
("top_logprobs", "logprobs", "True"),
|
||||
id="logprobs_top_without_logprobs_true",
|
||||
),
|
||||
pytest.param(
|
||||
"speaker_unknown",
|
||||
400,
|
||||
("Invalid speaker", "Supported"),
|
||||
id="speaker_unknown_preset",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
|
||||
def test_chat_completions_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
case_id: str,
|
||||
err_code: int | tuple[int, ...],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
cfg = _chat_completions_request_without_expectations(omni_server, case_id)
|
||||
cfg["err_code"] = err_code
|
||||
cfg["err_message"] = err_message
|
||||
openai_client.send_chat_completions_http_request(cfg)[0]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# WS /v1/video/chat/stream
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# First-frame JSON with ``omni_server.model`` for ``test_video_chat_stream_invalid_requests``.
|
||||
_VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR = object()
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize(
|
||||
"send_frames_spec, err_message",
|
||||
[
|
||||
pytest.param("{not-json", "Invalid JSON", id="invalid_json"),
|
||||
pytest.param(
|
||||
_VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR,
|
||||
"Invalid session config",
|
||||
id="session_modalities_scalar",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
|
||||
def test_video_chat_stream_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
send_frames_spec: Any,
|
||||
err_message: str,
|
||||
) -> None:
|
||||
if send_frames_spec is _VIDEO_CHAT_WS_SESSION_MODALITIES_SCALAR:
|
||||
send_frames = json.dumps({"type": "session.config", "model": omni_server.model, "modalities": "text"})
|
||||
else:
|
||||
send_frames = send_frames_spec
|
||||
assert isinstance(send_frames, str)
|
||||
openai_client.send_video_chat_stream_ws_request(
|
||||
{
|
||||
"send_frames": send_frames,
|
||||
"timeout": 120,
|
||||
"ws_max_size": None,
|
||||
"ws_json_type": "error",
|
||||
"err_message": err_message,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# WS /v1/realtime
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True)
|
||||
def test_realtime_rejected_when_async_chunk_enabled(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
) -> None:
|
||||
openai_client.send_realtime_ws_request(
|
||||
{
|
||||
"timeout": 120,
|
||||
"ws_max_size": None,
|
||||
"ws_json_type": "error",
|
||||
"ws_error_code": "unsupported",
|
||||
"err_message": "async_chunk",
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Omni sleep/wakeup JSON validation (live ``Qwen/Qwen-Image`` process)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion, pytest.mark.omni]
|
||||
|
||||
_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
|
||||
|
||||
_QWEN_IMAGE = [
|
||||
pytest.param(
|
||||
OmniServerParams(model="Qwen/Qwen-Image"),
|
||||
id="qwen_image",
|
||||
marks=hardware_marks(res={"cuda": "H100"}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST omni sleep / omni wakeup (invalid JSON)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint, json_body, err_message",
|
||||
[
|
||||
pytest.param("sleep", {"level": 2}, ("stage_ids", "Field required", "Missing"), id="sleep_missing_stage_ids"),
|
||||
pytest.param(
|
||||
"sleep",
|
||||
{"stage_ids": [], "level": 2},
|
||||
("stage_ids", "Field required", "Missing"),
|
||||
id="sleep_empty_stage_ids",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param(
|
||||
"sleep",
|
||||
{"stage_ids": "not-a-list", "level": 2},
|
||||
("stage_ids", "list_type", "a valid list"),
|
||||
id="sleep_invalid_stage_ids_type",
|
||||
),
|
||||
pytest.param(
|
||||
"sleep",
|
||||
{"stage_ids": [0], "level": "high"},
|
||||
("level", "int_parsing", "a valid integer"),
|
||||
id="sleep_invalid_level_type",
|
||||
),
|
||||
pytest.param(
|
||||
"sleep",
|
||||
{"stage_ids": [0], "level": -1},
|
||||
("level", "greater"),
|
||||
id="sleep_negative_level",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
pytest.param("wakeup", {}, ("stage_ids", "missing", "Field required"), id="wakeup_missing_stage_ids"),
|
||||
pytest.param(
|
||||
"wakeup",
|
||||
{"stage_ids": None},
|
||||
("stage_ids", "list_type", "a valid list"),
|
||||
id="wakeup_invalid_stage_ids_type",
|
||||
),
|
||||
pytest.param(
|
||||
"wakeup",
|
||||
{"stage_ids": []},
|
||||
("stage_ids", "length"),
|
||||
id="wakeup_empty_stage_ids",
|
||||
marks=_SKIP_ISSUE_3649,
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _QWEN_IMAGE, indirect=True)
|
||||
def test_omni_sleep_wakeup_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
endpoint: Literal["sleep", "wakeup"],
|
||||
json_body: dict[str, Any],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
cfg: dict[str, Any] = {
|
||||
"json": json_body,
|
||||
"timeout": 120,
|
||||
"err_code": 400,
|
||||
"err_message": err_message,
|
||||
}
|
||||
if endpoint == "sleep":
|
||||
openai_client.send_omni_sleep_http_request(cfg)
|
||||
else:
|
||||
openai_client.send_omni_wakeup_http_request(cfg)
|
||||
@@ -0,0 +1,171 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HTTP video job routes (live ``Wan-AI/Wan2.2-T2V-A14B-Diffusers``).
|
||||
|
||||
WebSocket ``/v1/video/chat/stream`` and ``/v1/realtime`` invalid-input checks live in
|
||||
``test_invalid_omni_chat.py`` with the same Qwen3-Omni server module fixture (Wan cannot serve those paths).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
|
||||
|
||||
_WAN_T2V = [
|
||||
pytest.param(
|
||||
OmniServerParams(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers"),
|
||||
id="wan22_t2v",
|
||||
marks=hardware_marks(res={"cuda": "H100"}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _minimal_video_form_data() -> dict[str, str]:
|
||||
return {"prompt": "a cat walking on grass"}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/videos (async)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"form_extra, err_message",
|
||||
[
|
||||
pytest.param(None, ("prompt", "Field required", "Missing"), id="missing_prompt"),
|
||||
pytest.param({"seconds": "0"}, ("seconds", "string_pattern_mismatch", "1"), id="seconds_zero"),
|
||||
pytest.param({"size": "bad"}, ("size", "string_pattern_mismatch"), id="size_bad"),
|
||||
pytest.param(
|
||||
{"num_inference_steps": "0"},
|
||||
("num_inference_steps", "greater_than_equal", "1"),
|
||||
id="num_inference_steps_zero",
|
||||
),
|
||||
pytest.param(
|
||||
{"guidance_scale": "25"}, ("guidance_scale", "less_than_equal", "20"), id="guidance_scale_above_max"
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
|
||||
def test_post_videos_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
form_extra: dict[str, str] | None,
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
data: dict[str, str] = {} if form_extra is None else {**_minimal_video_form_data(), **form_extra}
|
||||
openai_client.send_videos_create_http_request(
|
||||
{"data": data, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/videos/sync
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"form_extra, err_message",
|
||||
[
|
||||
pytest.param(None, ("prompt", "Field required", "Missing"), id="missing_prompt"),
|
||||
pytest.param({"seconds": "0"}, ("seconds", "string_pattern_mismatch", "1"), id="seconds_zero"),
|
||||
pytest.param({"size": "bad"}, ("size", "string_pattern_mismatch"), id="size_bad"),
|
||||
pytest.param(
|
||||
{"num_inference_steps": "201"},
|
||||
("num_inference_steps", "less_than_equal", "200"),
|
||||
id="num_inference_steps_above_max",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
|
||||
def test_post_videos_sync_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
form_extra: dict[str, str] | None,
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
data: dict[str, str] = {} if form_extra is None else {**_minimal_video_form_data(), **form_extra}
|
||||
openai_client.send_videos_sync_http_request(
|
||||
{"data": data, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# GET /v1/videos (list)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params, err_message",
|
||||
[
|
||||
pytest.param({"limit": -1}, ("limit", "greater_than_equal", "0"), id="invalid_limit_negative"),
|
||||
pytest.param({"limit": 101}, ("limit", "less_than_equal", "100"), id="limit_above_max"),
|
||||
pytest.param({"order": "newest"}, ("order", "literal_error", "asc", "desc"), id="invalid_order"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
|
||||
def test_get_videos_list_invalid_requests(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
params: dict[str, Any],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
openai_client.send_videos_list_http_request(
|
||||
{"params": params, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# GET /v1/videos/{video_id}
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
|
||||
def test_get_video_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_video_retrieve_http_request(
|
||||
{
|
||||
"video_id": "does-not-exist",
|
||||
"timeout": 120,
|
||||
"err_code": 404,
|
||||
"err_message": ("not found", "video"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# DELETE /v1/videos/{video_id}
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
|
||||
def test_delete_video_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_video_delete_http_request(
|
||||
{
|
||||
"video_id": "does-not-exist",
|
||||
"timeout": 120,
|
||||
"err_code": 404,
|
||||
"err_message": ("not found", "video"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# GET /v1/videos/{video_id}/content
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", _WAN_T2V, indirect=True)
|
||||
def test_get_video_content_invalid_requests(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
|
||||
openai_client.send_video_content_http_request(
|
||||
{
|
||||
"video_id": "does-not-exist",
|
||||
"timeout": 120,
|
||||
"err_code": 404,
|
||||
"err_message": ("not found", "video"),
|
||||
}
|
||||
)
|
||||
Reference in new issue
Block a user