Revert "[Test] Add Qwen-tts test cases and unify the style of existing test cases" (#2192)

Signed-off-by: linyueqian <linyueqian@outlook.com>

tests/e2e/offline_inference/test_qwen3_tts_base.py

@@ -1,84 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E offline tests for Qwen3-TTS Base model with text input and audio output.
-
-Async_chunk disable, cuda_graph disabled (no_async_chunk stage config).
-CUDA graph is disabled by setting engine_args.enforce_eager=true via modify_stage_config().
-Same structure as test_qwen3_omni (models, stage_configs, test_params, parametrize omni_runner).
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import pytest
-
-from tests.conftest import modify_stage_config
-from tests.utils import hardware_test
-
-MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
-REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-
-
-def get_cuda_graph_config():
-    path = modify_stage_config(
-        get_stage_config(),
-        updates={
-            "stage_args": {
-                0: {
-                    "engine_args.enforce_eager": "true",
-                },
-                1: {"engine_args.enforce_eager": "true"},
-            },
-        },
-    )
-    return path
-
-
-def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
-    """Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-# Same structure as test_qwen3_omni: models, stage_configs, test_params
-tts_server_params = [
-    pytest.param(
-        (MODEL, get_cuda_graph_config()),
-        id="no_cuda_graph",
-    )
-]
-
-
-def get_prompt():
-    """Text prompt for text-to-audio (same role as get_question in test_qwen3_omni)."""
-    return "Hello, this is a test for text to audio."
-
-
-@pytest.mark.skip(reason="Known issue(2030): qwen3_tts_no_async_chunk path temporarily disabled.")
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
-def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
-    """
-    Test text input processing and audio output via offline Omni runner.
-    Deploy Setting: qwen3_tts_no_async_chunk.yaml + enforce_eager=true
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False
-    Extra Setting: task_type=Base, voice=clone, ref_audio/ref_text provided
-    Datasets: few requests
-    """
-    request_config = {
-        "input": get_prompt(),
-        "task_type": "Base",
-        "voice": "clone",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    omni_runner_handler.send_audio_speech_request(request_config)

tests/e2e/offline_inference/test_qwen3_tts_customvoice.py

@@ -1,74 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E offline tests for Qwen3-TTS CustomVoice model with text input and audio output.
-
-Async_chunk disable, cuda_graph disabled (no_async_chunk stage config).
-CUDA graph is disabled by setting engine_args.enforce_eager=true via modify_stage_config().
-Same structure as test_qwen3_omni (models, stage_configs, test_params, parametrize omni_runner).
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import pytest
-
-from tests.conftest import modify_stage_config
-from tests.utils import hardware_test
-
-MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
-
-
-def get_cuda_graph_config():
-    path = modify_stage_config(
-        get_stage_config(),
-        updates={
-            "stage_args": {
-                0: {
-                    "engine_args.enforce_eager": "true",
-                },
-                1: {"engine_args.enforce_eager": "true"},
-            },
-        },
-    )
-    return path
-
-
-def get_stage_config(name: str = "qwen3_tts_no_async_chunk.yaml"):
-    """Get the no_async_chunk stage config path (async_chunk disable, cuda_graph disabled)."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-# Same structure as test_qwen3_omni: models, stage_configs, test_params
-tts_server_params = [
-    pytest.param(
-        (MODEL, get_cuda_graph_config()),
-        id="no_cuda_graph",
-    )
-]
-
-
-def get_prompt():
-    """Text prompt for text-to-audio (same role as get_question in test_qwen3_omni)."""
-    return "Hello, this is a test for text to audio."
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
-def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
-    """
-    Test text input processing and audio output via offline Omni runner.
-    Deploy Setting: qwen3_tts_no_async_chunk.yaml + enforce_eager=true
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False
-    Datasets: few requests
-    """
-    request_config = {"input": get_prompt(), "voice": "vivian"}
-    omni_runner_handler.send_audio_speech_request(request_config)

tests/e2e/online_serving/test_qwen3_tts.py

@@ -0,0 +1,285 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E Online tests for Qwen3-TTS model with text input and audio output.
+
+These tests verify the /v1/audio/speech endpoint works correctly with
+actual model inference, not mocks.
+"""
+
+import os
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
+
+from pathlib import Path
+
+import httpx
+import pytest
+
+from tests.conftest import OmniServer
+from tests.utils import hardware_test
+
+MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice"
+
+
+def get_stage_config(name: str = "qwen3_tts.yaml"):
+    """Get the stage config path for Qwen3-TTS."""
+    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
+
+
+@pytest.fixture(scope="class")
+def omni_server():
+    """Start vLLM-Omni server with CustomVoice model."""
+    stage_config_path = get_stage_config()
+
+    print(f"Starting OmniServer with model: {MODEL}")
+
+    with OmniServer(
+        MODEL,
+        [
+            "--stage-configs-path",
+            stage_config_path,
+            "--stage-init-timeout",
+            "120",
+            "--trust-remote-code",
+            "--enforce-eager",
+            "--disable-log-stats",
+        ],
+    ) as server:
+        print("OmniServer started successfully")
+        yield server
+        print("OmniServer stopping...")
+
+    print("OmniServer stopped")
+
+
+def make_speech_request(
+    host: str,
+    port: int,
+    text: str,
+    voice: str = "vivian",
+    language: str = "English",
+    task_type: str | None = None,
+    instructions: str | None = None,
+    timeout: float = 120.0,
+) -> httpx.Response:
+    """Make a request to the /v1/audio/speech endpoint."""
+    url = f"http://{host}:{port}/v1/audio/speech"
+    payload = {
+        "input": text,
+        "voice": voice,
+        "language": language,
+    }
+    if task_type:
+        payload["task_type"] = task_type
+    if instructions:
+        payload["instructions"] = instructions
+
+    with httpx.Client(timeout=timeout) as client:
+        return client.post(url, json=payload)
+
+
+def verify_wav_audio(content: bytes) -> bool:
+    """Verify that content is valid WAV audio data."""
+    # WAV files start with "RIFF" header
+    if len(content) < 44:  # Minimum WAV header size
+        return False
+    return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
+
+
+# Minimum expected audio size for a short sentence (~1 second of 24kHz 16-bit mono WAV)
+MIN_AUDIO_BYTES = 10000
+
+
+class TestQwen3TTSCustomVoice:
+    """E2E tests for Qwen3-TTS CustomVoice model."""
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_speech_english_basic(self, omni_server) -> None:
+        """Test basic English TTS generation."""
+        response = make_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            text="Hello, how are you?",
+            voice="vivian",
+            language="English",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV audio"
+        assert len(response.content) > MIN_AUDIO_BYTES, (
+            f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
+        )
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_speech_chinese_basic(self, omni_server) -> None:
+        """Test basic Chinese TTS generation."""
+        response = make_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            text="你好，我是通义千问",
+            voice="vivian",
+            language="Chinese",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV audio"
+        assert len(response.content) > MIN_AUDIO_BYTES, (
+            f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
+        )
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_speech_different_voices(self, omni_server) -> None:
+        """Test TTS with different voice options."""
+        voices = ["vivian", "ryan"]
+        for voice in voices:
+            response = make_speech_request(
+                host=omni_server.host,
+                port=omni_server.port,
+                text="Testing voice selection.",
+                voice=voice,
+                language="English",
+            )
+
+            assert response.status_code == 200, f"Request failed for voice {voice}: {response.text}"
+            assert verify_wav_audio(response.content), f"Invalid WAV for voice {voice}"
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_speech_binary_response_not_utf8_error(self, omni_server) -> None:
+        """
+        Regression test: Verify binary audio is returned, not UTF-8 error.
+
+        This test ensures the multimodal_output property correctly retrieves
+        audio from completion outputs, preventing the "TTS model did not
+        produce audio output" error.
+        """
+        response = make_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            text="This should return binary audio, not a JSON error.",
+            voice="vivian",
+            language="English",
+        )
+
+        # Should NOT be a JSON error response
+        assert response.status_code == 200, f"Request failed: {response.text}"
+
+        # Verify it's binary audio, not JSON
+        try:
+            # If this succeeds and starts with {"error", it's a bug
+            text = response.content.decode("utf-8")
+            assert not text.startswith('{"error"'), f"Got error response instead of audio: {text}"
+        except UnicodeDecodeError:
+            # This is expected - binary audio can't be decoded as UTF-8
+            pass
+
+        assert verify_wav_audio(response.content), "Response is not valid WAV audio"
+
+
+class TestQwen3TTSAPIEndpoints:
+    """Test API endpoint functionality."""
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_list_voices_endpoint(self, omni_server) -> None:
+        """Test the /v1/audio/voices endpoint returns available voices."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/voices"
+
+        with httpx.Client(timeout=30.0) as client:
+            response = client.get(url)
+
+        assert response.status_code == 200
+        data = response.json()
+        assert "voices" in data
+        assert isinstance(data["voices"], list)
+        assert len(data["voices"]) > 0
+        # Check some expected voices are present
+        voices_lower = [v.lower() for v in data["voices"]]
+        assert "vivian" in voices_lower or "ryan" in voices_lower
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_models_endpoint(self, omni_server) -> None:
+        """Test the /v1/models endpoint returns loaded model."""
+        url = f"http://{omni_server.host}:{omni_server.port}/v1/models"
+
+        with httpx.Client(timeout=30.0) as client:
+            response = client.get(url)
+
+        assert response.status_code == 200
+        data = response.json()
+        assert "data" in data
+        assert len(data["data"]) > 0
+
+
+@pytest.fixture(scope="class")
+def omni_server_no_async_chunk():
+    """Start vLLM-Omni server with non-async-chunk config."""
+    stage_config_path = get_stage_config("qwen3_tts_no_async_chunk.yaml")
+
+    with OmniServer(
+        MODEL,
+        [
+            "--stage-configs-path",
+            stage_config_path,
+            "--stage-init-timeout",
+            "120",
+            "--trust-remote-code",
+            "--enforce-eager",
+            "--disable-log-stats",
+        ],
+    ) as server:
+        yield server
+
+
+class TestQwen3TTSNoAsyncChunk:
+    """E2E tests for Qwen3-TTS in non-async-chunk (full decode) mode."""
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_speech_english(self, omni_server_no_async_chunk) -> None:
+        """Test English TTS with non-async-chunk pipeline."""
+        response = make_speech_request(
+            host=omni_server_no_async_chunk.host,
+            port=omni_server_no_async_chunk.port,
+            text="Hello, how are you?",
+            voice="vivian",
+            language="English",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV audio"
+        assert len(response.content) > MIN_AUDIO_BYTES
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_speech_chinese(self, omni_server_no_async_chunk) -> None:
+        """Test Chinese TTS with non-async-chunk pipeline."""
+        response = make_speech_request(
+            host=omni_server_no_async_chunk.host,
+            port=omni_server_no_async_chunk.port,
+            text="你好，我是通义千问",
+            voice="vivian",
+            language="Chinese",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV audio"
+        assert len(response.content) > MIN_AUDIO_BYTES

tests/e2e/online_serving/test_qwen3_tts_base.py

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """
-E2E Online tests for Qwen3-TTS model with text input and audio output.
+E2E tests for Qwen3-TTS Base voice-clone model.
 
-These tests verify the /v1/audio/speech endpoint works correctly with
-actual model inference, not mocks.
+Regression test for #1663: speech tokenizer loaded in bfloat16 produced
+all-silence PCM due to NaN overflow in SnakeBeta activation.
 """
 
 import os
@@ -12,99 +12,212 @@ import os
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 
+import struct
+import tempfile
 from pathlib import Path
 
+import httpx
+import numpy as np
 import pytest
 
-from tests.conftest import OmniServerParams
+from tests.conftest import (
+    OmniServer,
+    convert_audio_file_to_text,
+    cosine_similarity_text,
+)
 from tests.utils import hardware_test
 
-MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
+MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
 
+# Official Qwen3-TTS reference audio/text from examples/offline_inference/qwen3_tts/end2end.py
 REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
 REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
+SYN_TEXT = "Good one. Okay, fine, I'm just gonna leave this sock monkey here. Goodbye."
 
 
-def get_stage_config(name: str = "qwen3_tts.yaml"):
-    """Get the stage config path from vllm_omni model_executor stage_configs."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-def get_prompt(prompt_type="text"):
-    """Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
-    prompts = {
-        "text": "The weather is nice today, perfect for a walk in the park.",
-    }
-    return prompts.get(prompt_type, prompts["text"])
-
-
-def get_max_batch_size(size_type="few"):
-    """Batch size for concurrent requests (same as test_qwen3_omni)."""
-    batch_sizes = {"few": 5, "medium": 100, "large": 256}
-    return batch_sizes.get(size_type, 5)
-
-
-tts_server_params = [
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("qwen3_tts.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="async_chunk",
+def get_stage_config():
+    return str(
+        Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
     )
-]
 
 
-@pytest.mark.advanced_model
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_text_to_audio_001(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False
-    Datasets: few requests
-    """
+@pytest.fixture(scope="module")
+def omni_server():
+    """Start vLLM-Omni server with Base voice-clone model."""
+    stage_config_path = get_stage_config()
 
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": False,
-        "response_format": "wav",
+    with OmniServer(
+        MODEL,
+        [
+            "--stage-configs-path",
+            stage_config_path,
+            "--stage-init-timeout",
+            "120",
+            "--trust-remote-code",
+            "--enforce-eager",
+            "--disable-log-stats",
+        ],
+    ) as server:
+        yield server
+
+
+def make_base_speech_request(
+    host: str,
+    port: int,
+    text: str = SYN_TEXT,
+    ref_text: str = REF_TEXT,
+    ref_audio: str = REF_AUDIO_URL,
+    response_format: str = "wav",
+    timeout: float = 120.0,
+) -> httpx.Response:
+    url = f"http://{host}:{port}/v1/audio/speech"
+    payload = {
+        "model": MODEL,
+        "input": text,
         "task_type": "Base",
-        "voice": "clone",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
+        "ref_text": ref_text,
+        "ref_audio": ref_audio,
+        "response_format": response_format,
     }
-    openai_client.send_audio_speech_request(request_config, request_num=get_max_batch_size("few"))
+    with httpx.Client(timeout=timeout) as client:
+        return client.post(url, json=payload)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_text_to_audio_002(omni_server, openai_client) -> None:
+def verify_wav_audio(content: bytes) -> bool:
+    if len(content) < 44:
+        return False
+    return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
+
+
+def assert_not_silence(pcm_bytes: bytes):
+    """Assert PCM16 samples are not all identical (e.g. all-silence)."""
+    samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
+    unique = set(samples)
+    assert len(unique) > 1, (
+        f"All-silence detected: {len(samples)} samples, unique values: {unique}. "
+        "See https://github.com/vllm-project/vllm-omni/issues/1663"
+    )
+
+
+MIN_AUDIO_BYTES = 10000
+MIN_HNR_DB = 1.2  # Clean voice clone > 1.2 dB; distorted < 1.0 dB
+
+
+def compute_hnr_db(pcm_samples: np.ndarray, sr: int = 24000) -> float:
+    """Compute mean Harmonic-to-Noise Ratio (dB) for speech quality.
+
+    Clean cloned speech has HNR > 1.2 dB; distorted speech (e.g. lost
+    ref_code decoder context) drops below 1.0 dB.
     """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": True,
-        "response_format": "wav",
-        "task_type": "Base",
-        "voice": "clone",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    openai_client.send_audio_speech_request(request_config)
+    frame_len = int(0.03 * sr)  # 30ms frames
+    hop = frame_len // 2
+    hnr_values: list[float] = []
+
+    for start in range(0, len(pcm_samples) - frame_len, hop):
+        frame = pcm_samples[start : start + frame_len]
+        if np.max(np.abs(frame)) < 0.01:
+            continue
+        ac = np.correlate(frame, frame, mode="full")[len(frame) - 1 :]
+        ac = ac / (ac[0] + 1e-10)
+        min_lag = int(sr / 400)
+        max_lag = min(int(sr / 80), len(ac))
+        if min_lag >= max_lag:
+            continue
+        peak = float(np.max(ac[min_lag:max_lag]))
+        if 0 < peak < 1:
+            hnr_values.append(10 * np.log10(peak / (1 - peak + 1e-10)))
+
+    return float(np.mean(hnr_values)) if hnr_values else 0.0
+
+
+class TestQwen3TTSBaseVoiceClone:
+    """Regression tests for Base voice-clone (fix #1663)."""
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_base_voice_clone_not_silence(self, omni_server) -> None:
+        """PCM output must contain real audio, not all-silence."""
+        response = make_base_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            response_format="pcm",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
+        assert_not_silence(response.content)
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_base_voice_clone_whisper_transcription(self, omni_server) -> None:
+        """Whisper must transcribe the output as intelligible speech."""
+        response = make_base_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            response_format="wav",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert verify_wav_audio(response.content), "Response is not valid WAV"
+        assert len(response.content) > MIN_AUDIO_BYTES
+
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+            f.write(response.content)
+            wav_path = f.name
+
+        try:
+            transcript = convert_audio_file_to_text(wav_path)
+            print(f"Whisper transcript: {transcript}")
+            assert len(transcript.strip()) > 0, "Whisper returned empty transcript — audio is likely silence"
+            similarity = cosine_similarity_text(transcript.lower(), SYN_TEXT.lower())
+            print(f"Cosine similarity: {similarity:.3f}")
+            assert similarity > 0.9, (
+                f"Transcript doesn't match input: similarity={similarity:.2f}, transcript='{transcript}'"
+            )
+        finally:
+            os.unlink(wav_path)
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_base_voice_clone_wav_format(self, omni_server) -> None:
+        """WAV response must have valid headers and sufficient size."""
+        response = make_base_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            response_format="wav",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert response.headers.get("content-type") == "audio/wav"
+        assert verify_wav_audio(response.content), "Response is not valid WAV"
+        assert len(response.content) > MIN_AUDIO_BYTES
+
+    @pytest.mark.core_model
+    @pytest.mark.omni
+    @hardware_test(res={"cuda": "L4"}, num_cards=4)
+    def test_base_voice_clone_no_distortion(self, omni_server) -> None:
+        """Audio must not be distorted (regression for ref_code context loss).
+
+        When the decoder loses ref_code speaker context on later streaming
+        chunks, HNR drops below 1.0 dB.  Clean voice clone should be > 1.2 dB.
+        See https://github.com/vllm-project/vllm-omni/issues/1944
+        """
+        response = make_base_speech_request(
+            host=omni_server.host,
+            port=omni_server.port,
+            response_format="pcm",
+        )
+
+        assert response.status_code == 200, f"Request failed: {response.text}"
+        assert len(response.content) > MIN_AUDIO_BYTES
+
+        pcm_samples = np.frombuffer(response.content, dtype=np.int16).astype(np.float32) / 32768.0
+        hnr = compute_hnr_db(pcm_samples)
+        print(f"Voice clone HNR: {hnr:.2f} dB (threshold: {MIN_HNR_DB} dB)")
+        assert hnr >= MIN_HNR_DB, (
+            f"Audio distortion detected: HNR={hnr:.2f} dB < {MIN_HNR_DB} dB. "
+            "Voice clone decoder may be losing ref_code speaker context on later chunks."
+        )

tests/e2e/online_serving/test_qwen3_tts_base_expansion.py

@@ -1,119 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E Online tests for Qwen3-TTS model with text input and audio output.
-
-These tests verify the /v1/audio/speech endpoint works correctly with
-actual model inference, not mocks.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import pytest
-
-from tests.conftest import OmniServerParams
-from tests.utils import hardware_test
-
-MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
-
-REF_AUDIO_URL = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-
-
-def get_stage_config(name: str = "qwen3_tts.yaml"):
-    """Get the stage config path from vllm_omni model_executor stage_configs."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-def get_prompt(prompt_type="text"):
-    """Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
-    prompts = {
-        "text": "The weather is nice today, perfect for a walk in the park.",
-    }
-    return prompts.get(prompt_type, prompts["text"])
-
-
-def get_max_batch_size(size_type="few"):
-    """Batch size for concurrent requests (same as test_qwen3_omni)."""
-    batch_sizes = {"few": 5, "medium": 100, "large": 256}
-    return batch_sizes.get(size_type, 5)
-
-
-tts_server_params = [
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("qwen3_tts.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="async_chunk",
-    ),
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="no_async_chunk",
-    ),
-]
-
-
-@pytest.mark.advanced_model
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_clone_streaming_001(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=True
-    Datasets: few requests
-    """
-
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": True,
-        "response_format": "wav",
-        "task_type": "Base",
-        "voice": "clone",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    openai_client.send_audio_speech_request(request_config, request_num=get_max_batch_size("few"))
-
-
-@pytest.mark.advanced_model
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_response_format_001(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: non-stream PCM
-    Datasets: few requests
-    """
-
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "response_format": "pcm",
-        "task_type": "Base",
-        "voice": "clone",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    openai_client.send_audio_speech_request(request_config)

tests/e2e/online_serving/test_qwen3_tts_customvoice.py

@@ -1,105 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E Online tests for Qwen3-TTS model with text input and audio output.
-
-These tests verify the /v1/audio/speech endpoint works correctly with
-actual model inference, not mocks.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import pytest
-
-from tests.conftest import OmniServerParams
-from tests.utils import hardware_test
-
-MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
-
-
-def get_stage_config(name: str = "qwen3_tts.yaml"):
-    """Get the stage config path from vllm_omni model_executor stage_configs."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-def get_prompt(prompt_type="text"):
-    """Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
-    prompts = {
-        "text": "Beijing, China's capital, blends ancient wonders like the Great Wall with modern marvels. This vibrant metropolis offers rich culture, delicious Peking duck, and endless exploration opportunities.",
-    }
-    return prompts.get(prompt_type, prompts["text"])
-
-
-def get_max_batch_size(size_type="few"):
-    """Batch size for concurrent requests (same as test_qwen3_omni)."""
-    batch_sizes = {"few": 5, "medium": 100, "large": 256}
-    return batch_sizes.get(size_type, 5)
-
-
-tts_server_params = [
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("qwen3_tts.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="async_chunk",
-    )
-]
-
-
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_text_to_audio_001(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False
-    Datasets: few requests
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": False,
-        "response_format": "wav",
-        "task_type": "CustomVoice",
-        "voice": "vivian",
-    }
-
-    openai_client.send_audio_speech_request(request_config, request_num=get_max_batch_size())
-
-
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_text_to_audio_002(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": True,
-        "response_format": "wav",
-        "task_type": "CustomVoice",
-        "voice": "vivian",
-    }
-
-    openai_client.send_audio_speech_request(request_config)

tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py

@@ -1,137 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E Online tests for Qwen3-TTS model with text input and audio output.
-
-These tests verify the /v1/audio/speech endpoint works correctly with
-actual model inference, not mocks.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import pytest
-
-from tests.conftest import OmniServerParams
-from tests.utils import hardware_test
-
-MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
-
-
-def get_stage_config(name: str = "qwen3_tts.yaml"):
-    """Get the stage config path from vllm_omni model_executor stage_configs."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-def get_prompt(prompt_type="english"):
-    """Text prompt for text-to-audio tests (same as test_qwen3_omni - beijing test case)."""
-    prompts = {
-        "english": "Beijing, China's capital, blends ancient wonders like the Great Wall with modern marvels. This vibrant metropolis offers rich culture, delicious Peking duck, and endless exploration opportunities.",
-        "chinese": "北京，中国的首都，是一座融合了紫禁城、长城等历史遗迹与现代建筑的国际化大都市，充满了独特的文化与活力",
-    }
-    return prompts.get(prompt_type, prompts["english"])
-
-
-def get_max_batch_size(size_type="few"):
-    """Batch size for concurrent requests (same as test_qwen3_omni)."""
-    batch_sizes = {"few": 5, "medium": 100, "large": 256}
-    return batch_sizes.get(size_type, 5)
-
-
-tts_server_params = [
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("qwen3_tts.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="async_chunk",
-    ),
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("qwen3_tts_no_async_chunk.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="no_async_chunk",
-    ),
-]
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_language_001(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False, language=chinese
-    Datasets: few requests
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt("chinese"),
-        "stream": False,
-        "response_format": "wav",
-        "task_type": "CustomVoice",
-        "voice": "vivian",
-    }
-
-    openai_client.send_audio_speech_request(request_config)
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_001(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False, voice=eric
-    Datasets: few requests
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": True,
-        "response_format": "wav",
-        "task_type": "CustomVoice",
-        "voice": "eric",
-    }
-
-    openai_client.send_audio_speech_request(request_config, request_num=get_max_batch_size())
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_002(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False, language=chinese
-    Datasets: few requests
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": False,
-        "response_format": "wav",
-        "task_type": "CustomVoice",
-        "voice": "Serena",
-    }
-
-    openai_client.send_audio_speech_request(request_config)