diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 01897db96..75cc818a6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -228,6 +228,26 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Qwen3-TTS E2E Test" + timeout_in_minutes: 10 + depends_on: image-build + commands: + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/online_serving/test_qwen3_tts.py + agents: + queue: "gpu_4_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + # - label: "Omni Model Test with H100" # timeout_in_minutes: 30 # depends_on: image-build diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index c5d847345..c927e139a 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -8,7 +8,7 @@ COPY . . # Install system dependencies RUN apt-get update && \ - apt-get install -y ffmpeg && \ + apt-get install -y ffmpeg sox libsox-fmt-all && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docs/.nav.yml b/docs/.nav.yml index 16821c595..e0eb6d674 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -8,6 +8,7 @@ nav: - OpenAI-Compatible API: - Image Generation: serving/image_generation_api.md - Image Edit: serving/image_edit_api.md + - Text to Speech: serving/speech_api.md - Examples: - examples/README.md - Offline Inference: diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md new file mode 100644 index 000000000..86b7c73f8 --- /dev/null +++ b/docs/serving/speech_api.md @@ -0,0 +1,267 @@ +# Speech API + +vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation using Qwen3-TTS models. + +Each server instance runs a single model (specified at startup via `vllm serve --omni`). + +## Quick Start + +### Start the Server + +```bash +# CustomVoice model (predefined speakers) +vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --omni \ + --port 8091 \ + --trust-remote-code \ + --enforce-eager +``` + +### Generate Speech + +**Using curl:** + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, how are you?", + "voice": "vivian", + "language": "English" + }' --output output.wav +``` + +**Using Python:** + +```python +import httpx + +response = httpx.post( + "http://localhost:8091/v1/audio/speech", + json={ + "input": "Hello, how are you?", + "voice": "vivian", + "language": "English", + }, + timeout=300.0, +) + +with open("output.wav", "wb") as f: + f.write(response.content) +``` + +**Using OpenAI SDK:** + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8091/v1", api_key="none") + +response = client.audio.speech.create( + model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + voice="vivian", + input="Hello, how are you?", +) + +response.stream_to_file("output.wav") +``` + +## API Reference + +### Endpoint + +``` +POST /v1/audio/speech +Content-Type: application/json +``` + +### Request Parameters + +#### OpenAI Standard Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | string | **required** | The text to synthesize into speech | +| `model` | string | server's model | Model to use (optional, should match server if specified) | +| `voice` | string | "vivian" | Speaker name (e.g., vivian, ryan, aiden) | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `speed` | float | 1.0 | Playback speed (0.25-4.0) | + +#### vLLM-Omni Extension Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `task_type` | string | "CustomVoice" | TTS task type: CustomVoice, VoiceDesign, or Base | +| `language` | string | "Auto" | Language (see supported languages below) | +| `instructions` | string | "" | Voice style/emotion instructions | +| `max_new_tokens` | integer | 2048 | Maximum tokens to generate | + +**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian + +#### Voice Clone Parameters (Base task) + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `ref_audio` | string | null | Reference audio (URL or base64 data URL) | +| `ref_text` | string | null | Transcript of reference audio | +| `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) | + +### Response Format + +Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`). + +### Voices Endpoint + +``` +GET /v1/audio/voices +``` + +Lists available voices for the loaded model. + +```json +{ + "voices": ["aiden", "dylan", "eric", "ono_anna", "ryan", "serena", "sohee", "uncle_fu", "vivian"] +} +``` + +## Examples + +### CustomVoice with Style Instruction + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "I am so excited!", + "voice": "vivian", + "instructions": "Speak with great enthusiasm" + }' --output excited.wav +``` + +### VoiceDesign (Natural Language Voice Description) + +```bash +# Start server with VoiceDesign model first +vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --omni \ + --port 8091 \ + --trust-remote-code \ + --enforce-eager +``` + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello world", + "task_type": "VoiceDesign", + "instructions": "A warm, friendly female voice with a gentle tone" + }' --output designed.wav +``` + +### Base (Voice Cloning) + +```bash +# Start server with Base model first +vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --omni \ + --port 8091 \ + --trust-remote-code \ + --enforce-eager +``` + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, this is a cloned voice", + "task_type": "Base", + "ref_audio": "https://example.com/reference.wav", + "ref_text": "Original transcript of the reference audio" + }' --output cloned.wav +``` + +## Supported Models + +| Model | Task Type | Description | +|-------|-----------|-------------| +| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control | +| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description | +| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio | +| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant | +| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning | + +## Error Responses + +### 400 Bad Request + +Invalid parameters: + +```json +{ + "error": { + "message": "Input text cannot be empty", + "type": "BadRequestError", + "param": null, + "code": 400 + } +} +``` + +### 404 Not Found + +Model not found: + +```json +{ + "error": { + "message": "The model `xxx` does not exist.", + "type": "NotFoundError", + "param": "model", + "code": 404 + } +} +``` + +## Troubleshooting + +### "TTS model did not produce audio output" + +Ensure you're using the correct model variant for your task type: +- CustomVoice task → CustomVoice model +- VoiceDesign task → VoiceDesign model +- Base task → Base model + +### Server Not Running + +```bash +# Check if server is responding +curl http://localhost:8091/v1/audio/voices +``` + +### Out of Memory + +If you encounter OOM errors: +1. Use smaller model variant: `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` +2. Reduce `--gpu-memory-utilization` + +### Unsupported Speaker + +Use `/v1/audio/voices` to list available voices for the loaded model. + +## Development + +Enable debug logging: + +```bash +vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --omni \ + --port 8091 \ + --trust-remote-code \ + --enforce-eager \ + --uvicorn-log-level debug +``` diff --git a/pyproject.toml b/pyproject.toml index 0748b7eab..6af193953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,6 +192,6 @@ extend-ignore-identifiers-re = [ ".*nothink.*", ".*NOTHINK.*", ".*nin.*", - "Ono_Anna", + ".*[Oo]no_[Aa]nna.*", ".*cann.*", ] diff --git a/tests/e2e/online_serving/test_qwen3_tts.py b/tests/e2e/online_serving/test_qwen3_tts.py new file mode 100644 index 000000000..9a1ff8b97 --- /dev/null +++ b/tests/e2e/online_serving/test_qwen3_tts.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E Online tests for Qwen3-TTS model with text input and audio output. + +These tests verify the /v1/audio/speech endpoint works correctly with +actual model inference, not mocks. +""" + +import os + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + +from pathlib import Path + +import httpx +import pytest + +from tests.conftest import OmniServer +from tests.utils import hardware_test + +MODEL = "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice" + + +def get_stage_config(): + """Get the stage config path for Qwen3-TTS.""" + return str( + Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml" + ) + + +@pytest.fixture(scope="module") +def omni_server(): + """Start vLLM-Omni server with CustomVoice model.""" + stage_config_path = get_stage_config() + + print(f"Starting OmniServer with model: {MODEL}") + + with OmniServer( + MODEL, + [ + "--stage-configs-path", + stage_config_path, + "--stage-init-timeout", + "120", + "--trust-remote-code", + "--enforce-eager", + ], + ) as server: + print("OmniServer started successfully") + yield server + print("OmniServer stopping...") + + print("OmniServer stopped") + + +def make_speech_request( + host: str, + port: int, + text: str, + voice: str = "vivian", + language: str = "English", + task_type: str | None = None, + instructions: str | None = None, + timeout: float = 120.0, +) -> httpx.Response: + """Make a request to the /v1/audio/speech endpoint.""" + url = f"http://{host}:{port}/v1/audio/speech" + payload = { + "input": text, + "voice": voice, + "language": language, + } + if task_type: + payload["task_type"] = task_type + if instructions: + payload["instructions"] = instructions + + with httpx.Client(timeout=timeout) as client: + return client.post(url, json=payload) + + +def verify_wav_audio(content: bytes) -> bool: + """Verify that content is valid WAV audio data.""" + # WAV files start with "RIFF" header + if len(content) < 44: # Minimum WAV header size + return False + return content[:4] == b"RIFF" and content[8:12] == b"WAVE" + + +# Minimum expected audio size for a short sentence (~1 second of 24kHz 16-bit mono WAV) +MIN_AUDIO_BYTES = 10000 + + +class TestQwen3TTSCustomVoice: + """E2E tests for Qwen3-TTS CustomVoice model.""" + + @pytest.mark.core_model + @pytest.mark.omni + @hardware_test(res={"cuda": "L4"}, num_cards=4) + def test_speech_english_basic(self, omni_server) -> None: + """Test basic English TTS generation.""" + response = make_speech_request( + host=omni_server.host, + port=omni_server.port, + text="Hello, how are you?", + voice="vivian", + language="English", + ) + + assert response.status_code == 200, f"Request failed: {response.text}" + assert response.headers.get("content-type") == "audio/wav" + assert verify_wav_audio(response.content), "Response is not valid WAV audio" + assert len(response.content) > MIN_AUDIO_BYTES, ( + f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes" + ) + + @pytest.mark.core_model + @pytest.mark.omni + @hardware_test(res={"cuda": "L4"}, num_cards=4) + def test_speech_chinese_basic(self, omni_server) -> None: + """Test basic Chinese TTS generation.""" + response = make_speech_request( + host=omni_server.host, + port=omni_server.port, + text="你好,我是通义千问", + voice="vivian", + language="Chinese", + ) + + assert response.status_code == 200, f"Request failed: {response.text}" + assert response.headers.get("content-type") == "audio/wav" + assert verify_wav_audio(response.content), "Response is not valid WAV audio" + assert len(response.content) > MIN_AUDIO_BYTES, ( + f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes" + ) + + @pytest.mark.core_model + @pytest.mark.omni + @hardware_test(res={"cuda": "L4"}, num_cards=4) + def test_speech_different_voices(self, omni_server) -> None: + """Test TTS with different voice options.""" + voices = ["vivian", "ryan"] + for voice in voices: + response = make_speech_request( + host=omni_server.host, + port=omni_server.port, + text="Testing voice selection.", + voice=voice, + language="English", + ) + + assert response.status_code == 200, f"Request failed for voice {voice}: {response.text}" + assert verify_wav_audio(response.content), f"Invalid WAV for voice {voice}" + + @pytest.mark.core_model + @pytest.mark.omni + @hardware_test(res={"cuda": "L4"}, num_cards=4) + def test_speech_binary_response_not_utf8_error(self, omni_server) -> None: + """ + Regression test: Verify binary audio is returned, not UTF-8 error. + + This test ensures the multimodal_output property correctly retrieves + audio from completion outputs, preventing the "TTS model did not + produce audio output" error. + """ + response = make_speech_request( + host=omni_server.host, + port=omni_server.port, + text="This should return binary audio, not a JSON error.", + voice="vivian", + language="English", + ) + + # Should NOT be a JSON error response + assert response.status_code == 200, f"Request failed: {response.text}" + + # Verify it's binary audio, not JSON + try: + # If this succeeds and starts with {"error", it's a bug + text = response.content.decode("utf-8") + assert not text.startswith('{"error"'), f"Got error response instead of audio: {text}" + except UnicodeDecodeError: + # This is expected - binary audio can't be decoded as UTF-8 + pass + + assert verify_wav_audio(response.content), "Response is not valid WAV audio" + + +class TestQwen3TTSAPIEndpoints: + """Test API endpoint functionality.""" + + @pytest.mark.core_model + @pytest.mark.omni + @hardware_test(res={"cuda": "L4"}, num_cards=4) + def test_list_voices_endpoint(self, omni_server) -> None: + """Test the /v1/audio/voices endpoint returns available voices.""" + url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/voices" + + with httpx.Client(timeout=30.0) as client: + response = client.get(url) + + assert response.status_code == 200 + data = response.json() + assert "voices" in data + assert isinstance(data["voices"], list) + assert len(data["voices"]) > 0 + # Check some expected voices are present + voices_lower = [v.lower() for v in data["voices"]] + assert "vivian" in voices_lower or "ryan" in voices_lower + + @pytest.mark.core_model + @pytest.mark.omni + @hardware_test(res={"cuda": "L4"}, num_cards=4) + def test_models_endpoint(self, omni_server) -> None: + """Test the /v1/models endpoint returns loaded model.""" + url = f"http://{omni_server.host}:{omni_server.port}/v1/models" + + with httpx.Client(timeout=30.0) as client: + response = client.get(url) + + assert response.status_code == 200 + data = response.json() + assert "data" in data + assert len(data["data"]) > 0 diff --git a/vllm_omni/metrics/stats.py b/vllm_omni/metrics/stats.py index 037db2ea8..d5c45a0a9 100644 --- a/vllm_omni/metrics/stats.py +++ b/vllm_omni/metrics/stats.py @@ -221,24 +221,33 @@ class OrchestratorAggregator: stage_id: int, request_id: str, ) -> None: - if ( - output_to_yield.final_output_type == "audio" - and finished - and (multimodal_output := output_to_yield.multimodal_output.get("audio")) is not None - ): - nframes = int(multimodal_output[-1].shape[0]) - stage_events_for_req = self.stage_events.get(request_id, []) - if stage_events_for_req: - for stage_event in stage_events_for_req: - if stage_event.stage_id == stage_id: - stage_event.audio_generated_frames += nframes - break - else: - logger.warning( - "Failed to record audio generated frames for request %s at stage %s: no stage event found", - request_id, - stage_id, - ) + try: + if ( + output_to_yield.final_output_type == "audio" + and finished + and (multimodal_output := output_to_yield.multimodal_output.get("audio")) is not None + and len(multimodal_output) > 0 + ): + last = multimodal_output[-1] + nframes = int(last.shape[0]) if last.ndim > 0 else 1 + stage_events_for_req = self.stage_events.get(request_id, []) + if stage_events_for_req: + for stage_event in stage_events_for_req: + if stage_event.stage_id == stage_id: + stage_event.audio_generated_frames += nframes + break + else: + logger.warning( + "Failed to record audio generated frames for request %s at stage %s: no stage event found", + request_id, + stage_id, + ) + except Exception: + logger.debug( + "Failed to record audio frames for request %s", + request_id, + exc_info=True, + ) def _as_stage_request_stats( self,