[Test] Unify L2/L3 test layout, Buildkite steps, and test helpers (#2556)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Co-authored-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>

tests/entrypoints/openai_api/test_openai_audio_list_endpoints.py

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+GET ``/v1/audio/voices`` and ``/v1/models`` against a running omni server (TTS model).
+
+Migrated from ``tests/e2e/online_serving/test_voxtral_tts.py`` to colocate with other OpenAPI entrypoint tests.
+"""
+
+import os
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+import httpx
+import pytest
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path
+
+MODEL = "mistralai/Voxtral-4B-TTS-2603"
+STAGE_CONFIG = get_deploy_config_path("voxtral_tts.yaml")
+EXTRA_ARGS = ["--trust-remote-code", "--enforce-eager", "--disable-log-stats"]
+TEST_PARAMS = [OmniServerParams(model=MODEL, stage_config_path=STAGE_CONFIG, server_args=EXTRA_ARGS)]
+
+
+@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
+class TestOpenAIAudioListEndpoints:
+    """HTTP surface checks for OpenAI-compatible meta routes (voices, models)."""
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_list_voices_endpoint(self, omni_server) -> None:
+        """``GET /v1/audio/voices`` returns a non-empty ``voices`` list."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/voices"
+
+        with httpx.Client(timeout=30.0) as client:
+            response = client.get(url)
+
+        assert response.status_code == 200
+        data = response.json()
+        assert "voices" in data
+        assert isinstance(data["voices"], list)
+        assert len(data["voices"]) > 0
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    def test_models_endpoint(self, omni_server) -> None:
+        """``GET /v1/models`` returns at least one model."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/models"
+
+        with httpx.Client(timeout=30.0) as client:
+            response = client.get(url)
+
+        assert response.status_code == 200
+        data = response.json()
+        assert "data" in data
+        assert len(data["data"]) > 0

tests/entrypoints/openai_api/test_qwen3_tts_speaker_embedding.py

@@ -0,0 +1,242 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E tests for speaker_embedding API parameter.
+
+Tests Base models (0.6B-Base and 1.7B-Base) which support voice cloning
+via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction.
+"""
+
+import os
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+import struct
+
+import httpx
+import pytest
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path
+
+MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
+MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+
+# A synthetic 1024-dim speaker embedding (all 0.1 — not a real voice, but
+# exercises the full code path through the talker's _build_prompt_embeds).
+DUMMY_EMBEDDING_1024 = [0.1] * 1024
+DUMMY_EMBEDDING_2048 = [0.1] * 2048
+
+SYN_TEXT = "Hello."
+MIN_AUDIO_BYTES = 2000
+# Limit generation to keep tests fast (dummy embeddings produce nonsensical
+# output that may never hit a natural stop token).
+MAX_NEW_TOKENS = 256
+
+
+base_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL_BASE,
+            stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
+            server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"],
+        ),
+        id="qwen3-tts-0.6b-base",
+    )
+]
+
+base_1_7b_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL_BASE_1_7B,
+            stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
+            server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"],
+        ),
+        id="qwen3-tts-1.7b-base",
+    )
+]
+
+
+def verify_wav_audio(content: bytes) -> bool:
+    if len(content) < 44:
+        return False
+    return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
+
+
+def assert_not_silence(pcm_bytes: bytes):
+    """Assert PCM16 samples are not all identical (e.g. all-silence)."""
+    samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
+    unique = set(samples)
+    assert len(unique) > 1, f"All-silence detected: {len(samples)} samples, unique values: {unique}"
+
+
+# ── 0.6B-Base model tests ──
+
+
+@pytest.mark.parametrize("omni_server", base_server_params, indirect=True)
+class TestSpeakerEmbeddingBase:
+    """Speaker embedding tests against the 0.6B-Base model (supports Base task)."""
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_speaker_embedding_produces_audio(self, omni_server) -> None:
+        """speaker_embedding with Base task produces valid WAV audio."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": DUMMY_EMBEDDING_1024,
+            "x_vector_only_mode": True,
+            "response_format": "wav",
+            "max_new_tokens": MAX_NEW_TOKENS,
+        }
+        with httpx.Client(timeout=120.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV"
+        assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_speaker_embedding_pcm_not_silence(self, omni_server) -> None:
+        """speaker_embedding PCM output contains real audio, not all-silence."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": DUMMY_EMBEDDING_1024,
+            "x_vector_only_mode": True,
+            "response_format": "pcm",
+            "max_new_tokens": MAX_NEW_TOKENS,
+        }
+        with httpx.Client(timeout=120.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert len(response.content) > MIN_AUDIO_BYTES
+        assert_not_silence(response.content)
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_speaker_embedding_streaming(self, omni_server) -> None:
+        """speaker_embedding works with streaming PCM output."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": DUMMY_EMBEDDING_1024,
+            "x_vector_only_mode": True,
+            "response_format": "pcm",
+            "stream": True,
+            "max_new_tokens": MAX_NEW_TOKENS,
+        }
+        with httpx.Client(timeout=120.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert "audio/pcm" in response.headers.get("content-type", "")
+        assert len(response.content) > MIN_AUDIO_BYTES
+        assert_not_silence(response.content)
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_speaker_embedding_mutually_exclusive_with_ref_audio(self, omni_server) -> None:
+        """Sending both speaker_embedding and ref_audio returns 400."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": DUMMY_EMBEDDING_1024,
+            "ref_audio": "https://example.com/audio.wav",
+            "response_format": "wav",
+        }
+        with httpx.Client(timeout=30.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
+        assert "mutually exclusive" in response.text
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_speaker_embedding_empty_rejected(self, omni_server) -> None:
+        """Empty speaker_embedding list returns 400."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": [],
+            "response_format": "wav",
+        }
+        with httpx.Client(timeout=30.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
+        assert "non-empty" in response.text
+
+
+# ── 1.7B-Base model tests (2048-dim embeddings) ──
+
+
+@pytest.mark.parametrize("omni_server", base_1_7b_server_params, indirect=True)
+class TestSpeakerEmbedding1_7B:
+    """Speaker embedding tests against the 1.7B-Base model (2048-dim embeddings)."""
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_2048_dim_embedding_produces_audio(self, omni_server) -> None:
+        """2048-dim speaker_embedding with 1.7B-Base model produces valid WAV audio."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": DUMMY_EMBEDDING_2048,
+            "x_vector_only_mode": True,
+            "response_format": "wav",
+            "max_new_tokens": MAX_NEW_TOKENS,
+        }
+        with httpx.Client(timeout=120.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV"
+        assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=1)
+    def test_1024_dim_on_1_7b_model_rejected(self, omni_server) -> None:
+        """1024-dim speaker_embedding on a 1.7B model (expects 2048) should fail gracefully."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
+        payload = {
+            "model": omni_server.model,
+            "input": SYN_TEXT,
+            "task_type": "Base",
+            "speaker_embedding": DUMMY_EMBEDDING_1024,
+            "x_vector_only_mode": True,
+            "response_format": "wav",
+            "max_new_tokens": MAX_NEW_TOKENS,
+        }
+        with httpx.Client(timeout=120.0) as client:
+            response = client.post(url, json=payload)
+
+        # Wrong dimensions should produce an error, not silent garbage.
+        assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
+        assert "speaker_embedding has 1024 dimensions" in response.text
+        assert "expected 2048" in response.text

tests/entrypoints/openai_api/test_qwen3_tts_websocket.py

@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E online tests for Qwen3-TTS WebSocket streaming speech.
+"""
+
+import asyncio
+import json
+import os
+
+import pytest
+import websockets
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
+
+
+tts_ws_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL,
+            stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
+            server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"],
+            env_dict={"VLLM_DISABLE_COMPILE_CACHE": "1"},
+        ),
+        id="qwen3-tts-ws-customvoice",
+    )
+]
+
+
+async def _run_ws_session(host: str, port: int, model: str) -> dict:
+    uri = f"ws://{host}:{port}/v1/audio/speech/stream"
+    starts: list[dict] = []
+    dones: list[dict] = []
+    chunk_lengths: dict[int, list[int]] = {}
+    session_done: dict | None = None
+
+    async with websockets.connect(uri, max_size=None) as ws:
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "session.config",
+                    "model": model,
+                    "voice": "vivian",
+                    "language": "English",
+                    "response_format": "pcm",
+                    "stream_audio": True,
+                }
+            )
+        )
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "input.text",
+                    "text": (
+                        "Hello, this is a websocket streaming test for Qwen three TTS, "
+                        "and this sentence is intentionally long enough to produce audio chunks. "
+                        "This is the second sentence."
+                    ),
+                }
+            )
+        )
+        await ws.send(json.dumps({"type": "input.done"}))
+
+        while True:
+            message = await asyncio.wait_for(ws.recv(), timeout=180)
+            if isinstance(message, bytes):
+                if not starts:
+                    raise AssertionError("Received audio bytes before audio.start")
+                sentence_index = starts[-1]["sentence_index"]
+                chunk_lengths.setdefault(sentence_index, []).append(len(message))
+                continue
+
+            payload = json.loads(message)
+            msg_type = payload.get("type")
+            if msg_type == "audio.start":
+                starts.append(payload)
+                chunk_lengths.setdefault(payload["sentence_index"], [])
+            elif msg_type == "audio.done":
+                dones.append(payload)
+            elif msg_type == "session.done":
+                session_done = payload
+                break
+            elif msg_type == "error":
+                raise AssertionError(f"WebSocket error: {payload['message']}")
+            else:
+                raise AssertionError(f"Unexpected WebSocket message: {payload}")
+
+    return {
+        "starts": starts,
+        "dones": dones,
+        "chunk_lengths": chunk_lengths,
+        "session_done": session_done,
+    }
+
+
+class TestQwen3TTSWebSocket:
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    @pytest.mark.parametrize("omni_server", tts_ws_server_params, indirect=True)
+    def test_streaming_pcm_output(self, omni_server) -> None:
+        result = asyncio.run(_run_ws_session(omni_server.host, omni_server.port, omni_server.model))
+
+        starts = result["starts"]
+        dones = result["dones"]
+        chunk_lengths = result["chunk_lengths"]
+        session_done = result["session_done"]
+
+        assert session_done is not None
+        assert session_done["total_sentences"] == 2
+        assert len(starts) == 2
+        assert len(dones) == 2
+
+        for idx, start in enumerate(starts):
+            assert start["type"] == "audio.start"
+            assert start["sentence_index"] == idx
+            assert start["format"] == "pcm"
+            assert start["sample_rate"] == 24000
+            assert start["sentence_text"]
+
+        for done in dones:
+            sentence_index = done["sentence_index"]
+            total_bytes = done["total_bytes"]
+            assert done["error"] is False
+            assert total_bytes > 0
+            assert chunk_lengths[sentence_index], f"Expected binary PCM frames for sentence {sentence_index}"
+            assert sum(chunk_lengths[sentence_index]) == total_bytes

tests/entrypoints/openai_api/test_serving_speech_batch.py

@@ -0,0 +1,414 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E tests for the /v1/audio/speech/batch endpoint.
+
+Validates bulk synthesis via the batch API with actual model inference.
+"""
+
+import os
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+import base64
+import struct
+import tempfile
+import time
+from pathlib import Path
+
+import httpx
+import pytest
+import yaml
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.media import convert_audio_file_to_text, cosine_similarity_text
+from tests.helpers.runtime import OmniServerParams
+from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
+
+MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
+
+
+default_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL,
+            stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
+            server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"],
+        ),
+        id="qwen3-tts-batch-default",
+    )
+]
+
+
+def make_batch_request(
+    host: str,
+    port: int,
+    items: list[dict],
+    voice: str | None = "vivian",
+    language: str | None = "English",
+    response_format: str = "wav",
+    timeout: float = 300.0,
+    **kwargs,
+) -> httpx.Response:
+    """Make a request to the /v1/audio/speech/batch endpoint."""
+    url = f"http://{host}:{port}/v1/audio/speech/batch"
+    payload: dict = {
+        "items": items,
+        "response_format": response_format,
+    }
+    if voice is not None:
+        payload["voice"] = voice
+    if language is not None:
+        payload["language"] = language
+    payload.update(kwargs)
+
+    with httpx.Client(timeout=timeout) as client:
+        return client.post(url, json=payload)
+
+
+def verify_wav_bytes(audio_bytes: bytes) -> bool:
+    """Verify decoded audio bytes form a valid WAV."""
+    if len(audio_bytes) < 44:
+        return False
+    return audio_bytes[:4] == b"RIFF" and audio_bytes[8:12] == b"WAVE"
+
+
+def assert_not_silence(pcm_bytes: bytes):
+    """Assert PCM16 samples are not all identical."""
+    samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
+    unique = set(samples)
+    assert len(unique) > 1, f"All-silence: {len(samples)} samples, unique={unique}"
+
+
+MIN_AUDIO_BYTES = 10000
+
+
+class TestSpeechBatchE2E:
+    """E2E tests for /v1/audio/speech/batch endpoint."""
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_basic_two_items(self, omni_server) -> None:
+        """Batch with two items returns two successful base64-encoded results."""
+        items = [
+            {"input": "Hello, how are you today?"},
+            {"input": "The weather is nice outside."},
+        ]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+
+        assert data["total"] == 2
+        assert data["succeeded"] == 2
+        assert data["failed"] == 0
+        assert data["id"].startswith("speech-batch-")
+        assert len(data["results"]) == 2
+
+        for i, result in enumerate(data["results"]):
+            assert result["index"] == i
+            assert result["status"] == "success"
+            assert result["audio_data"] is not None
+            assert result["error"] is None
+            audio_bytes = base64.b64decode(result["audio_data"])
+            assert verify_wav_bytes(audio_bytes), f"Item {i}: invalid WAV"
+            assert len(audio_bytes) > MIN_AUDIO_BYTES, f"Item {i}: audio too small ({len(audio_bytes)} bytes)"
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_single_item(self, omni_server) -> None:
+        """Batch with a single item works correctly."""
+        items = [{"input": "Single item batch test."}]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+
+        assert data["total"] == 1
+        assert data["succeeded"] == 1
+        assert data["failed"] == 0
+        assert len(data["results"]) == 1
+        assert data["results"][0]["status"] == "success"
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_per_item_voice_override(self, omni_server) -> None:
+        """Per-item voice overrides the batch-level default."""
+        items = [
+            {"input": "First item with default voice."},
+            {"input": "Second item with ryan voice.", "voice": "ryan"},
+        ]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+            voice="vivian",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+
+        assert data["total"] == 2
+        assert data["succeeded"] == 2
+        # Both items should produce valid audio
+        for result in data["results"]:
+            assert result["status"] == "success"
+            audio_bytes = base64.b64decode(result["audio_data"])
+            assert verify_wav_bytes(audio_bytes)
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_multiple_languages(self, omni_server) -> None:
+        """Batch items with different languages per item."""
+        items = [
+            {"input": "Hello, nice to meet you.", "language": "English"},
+            {"input": "你好，很高兴认识你。", "language": "Chinese"},
+        ]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+
+        assert data["succeeded"] == 2
+        for result in data["results"]:
+            assert result["status"] == "success"
+            audio_bytes = base64.b64decode(result["audio_data"])
+            assert len(audio_bytes) > MIN_AUDIO_BYTES
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_whisper_transcription(self, omni_server) -> None:
+        """Whisper transcription of batch output matches input text."""
+        input_text = "Good morning, welcome to the speech synthesis test."
+        items = [{"input": input_text}]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+        assert data["succeeded"] == 1
+
+        audio_bytes = base64.b64decode(data["results"][0]["audio_data"])
+        assert verify_wav_bytes(audio_bytes)
+
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+            f.write(audio_bytes)
+            wav_path = f.name
+
+        try:
+            transcript = convert_audio_file_to_text(wav_path)
+            print(f"Whisper transcript: {transcript}")
+            assert len(transcript.strip()) > 0, "Empty transcript — likely silence"
+            similarity = cosine_similarity_text(transcript.lower(), input_text.lower())
+            print(f"Cosine similarity: {similarity:.3f}")
+            assert similarity > 0.7, f"Transcript mismatch: similarity={similarity:.2f}, transcript='{transcript}'"
+        finally:
+            os.unlink(wav_path)
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_result_indices_ordered(self, omni_server) -> None:
+        """Result indices match the input item order."""
+        items = [
+            {"input": "First sentence."},
+            {"input": "Second sentence."},
+            {"input": "Third sentence."},
+        ]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+
+        indices = [r["index"] for r in data["results"]]
+        assert indices == [0, 1, 2]
+
+
+class TestSpeechBatchValidation:
+    """Validation / error-handling tests for the batch endpoint."""
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_empty_items_rejected(self, omni_server) -> None:
+        """Empty items list returns a 4xx error."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech/batch"
+        payload = {"items": [], "voice": "vivian"}
+
+        with httpx.Client(timeout=30.0) as client:
+            response = client.post(url, json=payload)
+
+        assert response.status_code in (400, 422)
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", default_server_params, indirect=True)
+    def test_batch_exceeds_max_items(self, omni_server) -> None:
+        """Batch exceeding 32 items returns 400 error."""
+        items = [{"input": f"Item {i}"} for i in range(33)]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 400
+
+
+def _make_batch2_stage_config() -> str:
+    """Create a modified stage config with max_num_seqs=2 for all stages."""
+    base_config_path = get_deploy_config_path("qwen3_tts.yaml")
+    with open(base_config_path, encoding="utf-8") as f:
+        cfg = yaml.safe_load(f) or {}
+    stage_updates = {stage["stage_id"]: {"max_num_seqs": 2} for stage in cfg["stages"]}
+    return modify_stage_config(base_config_path, updates={"stages": stage_updates})
+
+
+BATCH2_STAGE_CONFIG_PATH = _make_batch2_stage_config()
+
+
+batch2_server_params = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL,
+            stage_config_path=BATCH2_STAGE_CONFIG_PATH,
+            server_args=["--trust-remote-code", "--enforce-eager", "--disable-log-stats"],
+        ),
+        id="qwen3-tts-batch-max2",
+    )
+]
+
+
+def make_single_request(
+    host: str,
+    port: int,
+    text: str,
+    voice: str = "vivian",
+    language: str = "English",
+    timeout: float = 300.0,
+) -> httpx.Response:
+    """Make a request to the single /v1/audio/speech endpoint."""
+    url = f"http://{host}:{port}/v1/audio/speech"
+    payload = {"input": text, "voice": voice, "language": language}
+    with httpx.Client(timeout=timeout) as client:
+        return client.post(url, json=payload)
+
+
+class TestSpeechBatchSize2:
+    """E2E tests with max_batch_size=2 to verify true batched inference."""
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", batch2_server_params, indirect=True)
+    def test_batch2_produces_valid_audio(self, omni_server) -> None:
+        """Batch of 2 items with batched engine produces valid audio."""
+        items = [
+            {"input": "Hello, this is the first sentence."},
+            {"input": "And this is the second sentence."},
+        ]
+        response = make_batch_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            items=items,
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        data = response.json()
+
+        assert data["total"] == 2
+        assert data["succeeded"] == 2
+        assert data["failed"] == 0
+
+        # Save audio files for inspection
+        output_dir = Path(tempfile.mkdtemp(prefix="tts_batch2_output_"))
+        for result in data["results"]:
+            assert result["status"] == "success"
+            audio_bytes = base64.b64decode(result["audio_data"])
+            assert verify_wav_bytes(audio_bytes), f"Item {result['index']}: invalid WAV"
+            assert len(audio_bytes) > MIN_AUDIO_BYTES
+
+            wav_path = output_dir / f"batch_item_{result['index']}.wav"
+            wav_path.write_bytes(audio_bytes)
+            print(f"  Item {result['index']}: {len(audio_bytes)} bytes -> {wav_path}")
+
+        print(f"\nBatch audio saved to: {output_dir}")
+
+    @pytest.mark.advanced_model
+    @pytest.mark.tts
+    @hardware_test(res={"cuda": "H100"}, num_cards=1)
+    @pytest.mark.parametrize("omni_server", batch2_server_params, indirect=True)
+    def test_batch2_vs_sequential_timing(self, omni_server) -> None:
+        """Compare batch-of-2 vs 2 sequential single requests.
+
+        With max_batch_size=2 the engine can process both items
+        concurrently, so the batch call should be faster than two
+        sequential single calls (or at least comparable).
+        """
+        text_a = "The quick brown fox jumps over the lazy dog."
+        text_b = "A journey of a thousand miles begins with a single step."
+
+        host = omni_server.host
+        port = omni_server.port
+
+        # --- Sequential: 2 single /v1/audio/speech calls ---
+        t0 = time.perf_counter()
+        resp_a = make_single_request(host, port, text_a)
+        resp_b = make_single_request(host, port, text_b)
+        sequential_time = time.perf_counter() - t0
+
+        assert resp_a.status_code == 200, f"Single A failed: {resp_a.text}"
+        assert resp_b.status_code == 200, f"Single B failed: {resp_b.text}"
+
+        # --- Batch: 1 /v1/audio/speech/batch call with 2 items ---
+        items = [{"input": text_a}, {"input": text_b}]
+        t0 = time.perf_counter()
+        resp_batch = make_batch_request(host, port, items)
+        batch_time = time.perf_counter() - t0
+
+        assert resp_batch.status_code == 200, f"Batch failed: {resp_batch.text}"
+        batch_data = resp_batch.json()
+        assert batch_data["succeeded"] == 2
+
+        # Verify both produced valid audio
+        for result in batch_data["results"]:
+            audio_bytes = base64.b64decode(result["audio_data"])
+            assert verify_wav_bytes(audio_bytes)
+            assert len(audio_bytes) > MIN_AUDIO_BYTES
+
+        print(f"\n  Sequential (2 singles): {sequential_time:.2f}s")
+        print(f"  Batch (2 items):        {batch_time:.2f}s")
+        print(f"  Speedup:                {sequential_time / batch_time:.2f}x")

tests/entrypoints/test_omni_sleep_mode.py

@@ -1,3 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Omni sleep mode: entrypoint-level VRAM/ACK tests (L4) plus H100 multi-TP e2e."""
+
 import asyncio
 import logging
 import os
@@ -30,7 +35,7 @@ def clean_gpu_envs():
 
 
 def get_vram_info(device_id: int) -> dict:
-    """Obtain a snapshot of the specified GPU's memory (GiB)."""
+    """Per-**process** CUDA allocator stats (GiB). Does not see other processes' usage."""
     try:
         if current_omni_platform.is_rocm():
             num_gpus = torch.accelerator.device_count()
@@ -52,6 +57,23 @@ def get_vram_info(device_id: int) -> dict:
         return {"reserved": 0.0, "allocated": 0.0}
 
 
+def get_device_global_memory_used_gib(device_id: int) -> float:
+    """GPU-wide memory in use (GiB), includes all processes (driver view).
+
+    Uses ``torch.cuda.mem_get_info`` (CUDA / ROCm when available). Use this when stages run in
+    **separate worker processes**; ``memory_reserved`` in the test process does not reflect
+    child workers' allocations.
+    """
+    try:
+        with torch.cuda.device(device_id):
+            torch.accelerator.synchronize()
+            free_b, total_b = torch.cuda.mem_get_info()
+        return (total_b - free_b) / 1024**3
+    except Exception as e:
+        logger.warning("get_device_global_memory_used_gib(%s): %s", device_id, e)
+        return 0.0
+
+
 def get_ack_info(ack, key, default=None):
     """
     Since ACKs in a distributed environment can be either objects or dictionaries,
@@ -64,6 +86,20 @@ def get_ack_info(ack, key, default=None):
     return default
 
 
+MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
+MODEL_DIFF = "riverclouds/qwen_image_random"
+
+
+def get_dynamic_devices(stage_idx: int, num_stages: int, tp_size: int) -> str:
+    total_gpus = torch.accelerator.device_count()
+    gpus_per_stage = tp_size
+    start_idx = stage_idx * gpus_per_stage
+    if start_idx + gpus_per_stage > total_gpus:
+        start_idx = start_idx % total_gpus
+    device_ids = [str(start_idx + i) for i in range(gpus_per_stage)]
+    return ",".join(device_ids)
+
+
 @pytest.fixture(scope="function")
 async def llm_engine():
     if current_omni_platform.is_rocm():
@@ -97,6 +133,8 @@ async def llm_engine():
     engine = AsyncOmni(model=model_name, stages=stages, connectors=connectors, init_timeout=600, enable_sleep_mode=True)
     yield engine
     engine.shutdown()
+    # Subprocess / driver can lag releasing VRAM; brief pause before the next test spins up new workers.
+    await asyncio.sleep(1.5)
 
 
 @pytest.fixture(scope="function")
@@ -127,27 +165,43 @@ async def diffusion_engine():
     engine = AsyncOmni(model=model_name, stages=stages, init_timeout=600, enable_sleep_mode=True)
     yield engine
     engine.shutdown()
+    await asyncio.sleep(1.5)
 
 
 class TestOmniSleepMode:
     @pytest.mark.asyncio
-    @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
+    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)
     async def test_llm_sleep_ack(self, llm_engine: AsyncOmni):
         """LLM Thinker (GPU0) Signal and Physical Recycling Audit"""
         try:
+            device_id = 0
+            used_before = get_device_global_memory_used_gib(device_id)
             acks = await llm_engine.sleep(stage_ids=[0], level=2)
+            await asyncio.sleep(1.5)
+            used_after = get_device_global_memory_used_gib(device_id)
+            drop_gib = used_before - used_after
             # Verification signal successful
             assert all(get_ack_info(ack, "status") == "SUCCESS" for ack in acks)
-            # Verify physical recycling volume
+            # Worker-reported delta (can be 0 if get_current_memory_usage does not move) or
+            # GPU-global drop from mem_get_info (sees child worker processes).
             total_freed_bytes = sum(get_ack_info(ack, "freed_bytes", 0) for ack in acks)
             freed_gib = total_freed_bytes / 1024**3
-            logger.info(f"Thinker VRAM physically reclaimed: {freed_gib:.2f} GiB")
-            assert freed_gib > 5.0
+            logger.info(
+                "Thinker: ACK freed=%.2f GiB, global GPU used drop=%.2f GiB (before=%.2f, after=%.2f)",
+                freed_gib,
+                drop_gib,
+                used_before,
+                used_after,
+            )
+            assert freed_gib > 5.0 or drop_gib > 3.0, (
+                "Expected either ACK freed_bytes or global VRAM drop after sleep. "
+                f"ACK={freed_gib:.2f} GiB, global_drop={drop_gib:.2f} GiB"
+            )
         finally:
             llm_engine.shutdown()
 
     @pytest.mark.asyncio
-    @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
     async def test_diffusion_sleep_handshake(self, diffusion_engine: AsyncOmni):
         """Diffusion Worker stage signal loop"""
         try:
@@ -168,24 +222,31 @@ class TestOmniSleepMode:
             logger.info("Manual shutdown executed. Test should exit now.")
 
     @pytest.mark.asyncio
-    @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
     async def test_cross_device_cleanup(self, diffusion_engine: AsyncOmni):
         """Physical recycling audit: leveraging deterministic data returned by Workers"""
         try:
+            # TP2 uses GPUs 0 and 1; measure whole-GPU usage (includes worker subprocesses).
+            used_before = get_device_global_memory_used_gib(0) + get_device_global_memory_used_gib(1)
             acks = await diffusion_engine.sleep(stage_ids=[0], level=1)
-            # Sum up the release amounts reported by all Workers.
+            await asyncio.sleep(1.5)
+            used_after = get_device_global_memory_used_gib(0) + get_device_global_memory_used_gib(1)
+            drop_gib = used_before - used_after
             total_freed_bytes = sum(get_ack_info(ack, "freed_bytes", 0) for ack in acks)
             freed_gb = total_freed_bytes / 1024**3
             logger.info("Physical reclamation summary from workers:")
             logger.info(f"- Total Workers: {len(acks)}")
-            logger.info(f"- Total Freed: {freed_gb:.2f} GiB")
-            assert freed_gb > 14.0
+            logger.info(f"- Total Freed (ACK): {freed_gb:.2f} GiB, global used drop: {drop_gib:.2f} GiB")
+            assert freed_gb > 14.0 or drop_gib > 8.0, (
+                "Expected either ACK freed_bytes or global VRAM drop on GPUs 0+1. "
+                f"ACK={freed_gb:.2f} GiB, global_drop={drop_gib:.2f} GiB"
+            )
             logger.info("SUCCESS: 100% weights offloaded.")
         finally:
             diffusion_engine.shutdown()
 
     @pytest.mark.asyncio
-    @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
     async def test_diffusion_integrity_bit_level(self, diffusion_engine: AsyncOmni):
         """Bit-level consistency after Diffusion wake-up (prevent image corruption)"""
         try:
@@ -229,8 +290,16 @@ class TestOmniSleepMode:
             diffusion_engine.shutdown()
             logger.info("Cleanup complete, test exiting.")
 
+    @pytest.mark.skip(
+        reason=(
+            "Flaky/CI: dual AsyncOmni can fail with "
+            "RuntimeError: Orchestrator init failed, StageDiffusionProc died during handshake. "
+            "Re-enable when stable (no OOM on coordinated talker+diffusion). "
+            "Note: this test is fixture-free so skip does not init llm/diffusion engines."
+        )
+    )
     @pytest.mark.asyncio
-    @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
     async def test_coordinated_cross_device(self, llm_engine: AsyncOmni, diffusion_engine: AsyncOmni):
         """Heterogeneous Coordinated Cleanup Test (Talker and Diffusion on GPU 1)"""
         device_id = 1
@@ -270,7 +339,7 @@ class TestOmniSleepMode:
             logger.info("All engines scavenged. Ready for next test.")
 
     @pytest.mark.asyncio
-    @hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
     async def test_diffusion_vram_lifecycle_audit(self, diffusion_engine: AsyncOmni):
         """Diffusion memory loop: Active -> Deep Sleep -> Active -> inference sanity check"""
         device_id = 1
@@ -334,3 +403,134 @@ class TestOmniSleepMode:
             logger.info("Cleaning up engine and scavenging processes...")
             diffusion_engine.shutdown()
             await asyncio.sleep(1)
+
+
+# ---------------------------------------------------------------------------
+# H100 (or MI325) e2e: BAGEL / pure diffusion, multi-TP
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.omni
+@pytest.mark.advanced_model
+@pytest.mark.parametrize("tp_size", [1])
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)
+@pytest.mark.asyncio
+async def test_diffusion_model_sleep_tp(tp_size: int):
+    """Two-stage BAGEL default config: warmup, sleep all, wake, verify generate."""
+    if current_omni_platform.is_rocm():
+        clean_gpu_envs()
+    num_gpus = torch.accelerator.device_count()
+    if num_gpus < tp_size:
+        pytest.skip(f"Skipping TP={tp_size}")
+
+    engine_args = {
+        "model": MODEL,
+        "enable_sleep_mode": True,
+        "tensor_parallel_size": tp_size,
+        "enforce_eager": True,
+        "trust_remote_code": True,
+        "dtype": "bfloat16",
+        "gpu_memory_utilization": 0.5,
+    }
+
+    engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
+    try:
+        diff_sp = OmniDiffusionSamplingParams(num_inference_steps=2, height=256, width=256)
+        llm_sp = SamplingParams()
+
+        async for _ in engine.generate("test", sampling_params_list=[llm_sp, diff_sp]):
+            pass
+
+        acks = await engine.sleep(level=2)
+        statuses = [get_ack_info(ack, "status") for ack in acks]
+        assert all(s == "SUCCESS" for s in statuses), f"Sleep failed. Statuses: {statuses}"
+
+        await engine.wake_up()
+        async for _ in engine.generate("verify", sampling_params_list=[llm_sp, diff_sp]):
+            pass
+
+        logger.info("Diffusion TP=%s lifecycle OK", tp_size)
+    finally:
+        engine.shutdown()
+
+
+@pytest.mark.omni
+@pytest.mark.advanced_model
+@pytest.mark.parametrize("tp_size", [1, 2])
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.asyncio
+async def test_multistage_sleep_h100(tp_size: int):
+    """Explicit 2-stage (llm + diffusion) + connectors; sleep/wake both stages."""
+    if current_omni_platform.is_rocm():
+        clean_gpu_envs()
+    num_gpus = torch.accelerator.device_count()
+    if num_gpus < tp_size * 2:
+        pytest.skip("Not enough GPUs")
+
+    stages = []
+    for i in range(2):
+        devs = get_dynamic_devices(i, 2, tp_size)
+        stages.append(
+            {
+                "stage_id": i,
+                "stage_type": "llm" if i == 0 else "diffusion",
+                "runtime": {"process": True, "devices": devs},
+                "engine_args": {
+                    "model": MODEL,
+                    "model_stage": "thinker" if i == 0 else "base",
+                    "tensor_parallel_size": tp_size,
+                    "gpu_memory_utilization": 0.4,
+                    "dtype": "bfloat16",
+                    "enable_sleep_mode": True,
+                    "trust_remote_code": True,
+                },
+            }
+        )
+
+    connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
+
+    engine = AsyncOmni(
+        model=MODEL, stages=stages, connectors=connectors, enable_sleep_mode=True, stage_init_timeout=1200
+    )
+    try:
+        sp = OmniDiffusionSamplingParams(num_inference_steps=2)
+        async for _ in engine.generate("warmup", sampling_params_list=[SamplingParams(), sp]):
+            pass
+
+        acks = await engine.sleep(stage_ids=[0, 1], level=2)
+        assert len(acks) == 2
+
+        await engine.wake_up(stage_ids=[0, 1])
+        async for _ in engine.generate("verify", sampling_params_list=[SamplingParams(), sp]):
+            pass
+    finally:
+        engine.shutdown()
+
+
+@pytest.mark.omni
+@pytest.mark.advanced_model
+@pytest.mark.parametrize("tp_size", [1, 2])
+@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.asyncio
+async def test_pure_diffusion_scenario(tp_size: int):
+    """Single-stage random diffusion: sleep, wake, generate."""
+    if current_omni_platform.is_rocm():
+        clean_gpu_envs()
+    engine_args = {
+        "model": MODEL_DIFF,
+        "enable_sleep_mode": True,
+        "tensor_parallel_size": tp_size,
+        "enforce_eager": True,
+        "dtype": "bfloat16",
+        "gpu_memory_utilization": 0.5,
+    }
+
+    engine = AsyncOmni(**engine_args, stage_init_timeout=1200)
+    try:
+        await engine.sleep(level=1)
+        await engine.wake_up()
+        async for _ in engine.generate("test", sampling_params=SamplingParams()):
+            pass
+        logger.info("Pure diffusion OK (TP=%s)", tp_size)
+    finally:
+        engine.shutdown()

tests/entrypoints/test_utils.py

@@ -328,7 +328,7 @@ class TestResolveModelConfigPath:
         )
         mocker.patch(
             "vllm_omni.entrypoints.utils.current_omni_platform.get_default_stage_config_path",
-            return_value="vllm_omni/model_executor/stage_configs",
+            return_value="vllm_omni/deploy",
         )
 
         original_exists = os.path.exists