diff --git a/examples/online_serving/qwen3_tts/README.md b/examples/online_serving/qwen3_tts/README.md new file mode 100644 index 000000000..2cddd0fbf --- /dev/null +++ b/examples/online_serving/qwen3_tts/README.md @@ -0,0 +1,170 @@ +# Qwen3-TTS Online Serving + +This directory contains examples for running Qwen3-TTS models with vLLM-Omni's online serving API. + +## Supported Models + +| Model | Task Type | Description | +|-------|-----------|-------------| +| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control | +| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description | +| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio | + +## Quick Start + +### 1. Start the Server + +```bash +# CustomVoice model (default) +./run_server.sh + +# Or specify task type +./run_server.sh CustomVoice +./run_server.sh VoiceDesign +./run_server.sh Base +``` + +### 2. Run the Client + +```bash +# CustomVoice: Use predefined speaker +python openai_speech_client.py \ + --text "你好，我是通义千问" \ + --voice Vivian \ + --language Chinese + +# CustomVoice with style instruction +python openai_speech_client.py \ + --text "今天天气真好" \ + --voice Ryan \ + --instructions "用开心的语气说" + +# VoiceDesign: Describe the voice style +python openai_speech_client.py \ + --task-type VoiceDesign \ + --text "哥哥，你回来啦" \ + --instructions "体现撒娇稚嫩的萝莉女声，音调偏高" + +# Base: Voice cloning +python openai_speech_client.py \ + --task-type Base \ + --text "Hello, this is a cloned voice" \ + --ref-audio /path/to/reference.wav \ + --ref-text "Original transcript of the reference audio" +``` + +### 3. Using curl + +```bash +# Simple TTS request +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, how are you?", + "voice": "Vivian", + "language": "English" + }' --output output.wav + +# With style instruction +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "I am so excited!", + "voice": "Vivian", + "instructions": "Speak with great enthusiasm" + }' --output excited.wav +``` + +## API Reference + +### Endpoint + +``` +POST /v1/audio/speech +``` + +This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters. + +### Request Body + +```json +{ + "input": "Text to synthesize", + "voice": "Vivian", + "response_format": "wav", + "task_type": "CustomVoice", + "language": "Auto", + "instructions": "Optional style instructions", + "ref_audio": "URL or base64 for voice cloning", + "ref_text": "Reference audio transcript", + "x_vector_only_mode": false, + "max_new_tokens": 2048 +} +``` + +> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded. + +### Response + +Returns audio data in the requested format (default: WAV). + +## Parameters + +### Standard OpenAI Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | string | required | Text to synthesize | +| `voice` | string | "Vivian" | Speaker/voice name | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `speed` | float | 1.0 | Playback speed (0.25-4.0) | +| `model` | string | optional | Model name (optional when serving single model) | + +### Qwen3-TTS Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `task_type` | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base | +| `language` | string | "Auto" | Language: Auto, Chinese, English, Japanese, Korean | +| `instructions` | string | "" | Voice style/emotion instructions | +| `max_new_tokens` | int | 2048 | Maximum tokens to generate | + +### Voice Clone Parameters (Base task) + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `ref_audio` | string | Yes* | Reference audio (file path, URL, or base64) | +| `ref_text` | string | No | Transcript of reference audio (for ICL mode) | +| `x_vector_only_mode` | bool | false | Use speaker embedding only (no ICL) | + +## Python Usage + +```python +import httpx + +# Simple request +response = httpx.post( + "http://localhost:8000/v1/audio/speech", + json={ + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "input": "Hello world", + "voice": "Vivian", + }, + timeout=300.0, +) + +with open("output.wav", "wb") as f: + f.write(response.content) +``` + +## Limitations + +- **No streaming**: Audio is generated completely before being returned. Streaming will be supported after the pipeline is disaggregated (see RFC #938). +- **Single request**: Batch processing is not yet optimized for online serving. + +## Troubleshooting + +1. **Connection refused**: Make sure the server is running on the correct port +2. **Out of memory**: Reduce `--gpu-memory-utilization` in run_server.sh +3. **Unsupported speaker**: Check supported speakers via model documentation +4. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning diff --git a/examples/online_serving/qwen3_tts/openai_speech_client.py b/examples/online_serving/qwen3_tts/openai_speech_client.py new file mode 100644 index 000000000..272e84265 --- /dev/null +++ b/examples/online_serving/qwen3_tts/openai_speech_client.py @@ -0,0 +1,235 @@ +"""OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech endpoint. + +This script demonstrates how to use the OpenAI-compatible speech API +to generate audio from text using Qwen3-TTS models. + +Examples: + # CustomVoice task (predefined speaker) + python openai_speech_client.py --text "Hello, how are you?" --voice Vivian + + # CustomVoice with emotion instruction + python openai_speech_client.py --text "I'm so happy!" --voice Vivian \ + --instructions "Speak with excitement" + + # VoiceDesign task (voice from description) + python openai_speech_client.py --text "Hello world" \ + --task-type VoiceDesign \ + --instructions "A warm, friendly female voice" + + # Base task (voice cloning) + python openai_speech_client.py --text "Hello world" \ + --task-type Base \ + --ref-audio "https://example.com/reference.wav" \ + --ref-text "This is the reference transcript" +""" + +import argparse +import base64 +import os + +import httpx + +# Default server configuration +DEFAULT_API_BASE = "http://localhost:8000" +DEFAULT_API_KEY = "EMPTY" + + +def encode_audio_to_base64(audio_path: str) -> str: + """Encode a local audio file to base64 data URL.""" + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + + # Detect MIME type from extension + audio_path_lower = audio_path.lower() + if audio_path_lower.endswith(".wav"): + mime_type = "audio/wav" + elif audio_path_lower.endswith((".mp3", ".mpeg")): + mime_type = "audio/mpeg" + elif audio_path_lower.endswith(".flac"): + mime_type = "audio/flac" + elif audio_path_lower.endswith(".ogg"): + mime_type = "audio/ogg" + else: + mime_type = "audio/wav" # Default + + with open(audio_path, "rb") as f: + audio_bytes = f.read() + audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") + return f"data:{mime_type};base64,{audio_b64}" + + +def run_tts_generation(args) -> None: + """Run TTS generation via OpenAI-compatible /v1/audio/speech API.""" + + # Build request payload + payload = { + "model": args.model, + "input": args.text, + "voice": args.voice, + "response_format": args.response_format, + } + + # Add optional parameters + if args.instructions: + payload["instructions"] = args.instructions + if args.task_type: + payload["task_type"] = args.task_type + if args.language: + payload["language"] = args.language + if args.max_new_tokens: + payload["max_new_tokens"] = args.max_new_tokens + + # Voice clone parameters (Base task) + if args.ref_audio: + if args.ref_audio.startswith(("http://", "https://")): + payload["ref_audio"] = args.ref_audio + else: + payload["ref_audio"] = encode_audio_to_base64(args.ref_audio) + if args.ref_text: + payload["ref_text"] = args.ref_text + if args.x_vector_only: + payload["x_vector_only_mode"] = True + + print(f"Task type: {args.task_type or 'CustomVoice'}") + print(f"Text: {args.text}") + print(f"Voice: {args.voice}") + print("Generating audio...") + + # Make the API call + api_url = f"{args.api_base}/v1/audio/speech" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {args.api_key}", + } + + with httpx.Client(timeout=300.0) as client: + response = client.post(api_url, json=payload, headers=headers) + + if response.status_code != 200: + print(f"Error: {response.status_code}") + print(response.text) + return + + # Save audio response + output_path = args.output or "tts_output.wav" + with open(output_path, "wb") as f: + f.write(response.content) + print(f"Audio saved to: {output_path}") + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Server configuration + parser.add_argument( + "--api-base", + type=str, + default=DEFAULT_API_BASE, + help=f"API base URL (default: {DEFAULT_API_BASE})", + ) + parser.add_argument( + "--api-key", + type=str, + default=DEFAULT_API_KEY, + help="API key (default: EMPTY)", + ) + parser.add_argument( + "--model", + "-m", + type=str, + default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + help="Model name/path", + ) + + # Task configuration + parser.add_argument( + "--task-type", + "-t", + type=str, + default=None, + choices=["CustomVoice", "VoiceDesign", "Base"], + help="TTS task type (default: CustomVoice)", + ) + + # Input text + parser.add_argument( + "--text", + type=str, + required=True, + help="Text to synthesize", + ) + + # Voice/speaker + parser.add_argument( + "--voice", + type=str, + default="Vivian", + help="Speaker/voice name (default: Vivian). Options: Vivian, Ryan, etc.", + ) + parser.add_argument( + "--language", + type=str, + default=None, + help="Language: Auto, Chinese, English, etc.", + ) + parser.add_argument( + "--instructions", + type=str, + default=None, + help="Voice style/emotion instructions", + ) + + # Base (voice clone) parameters + parser.add_argument( + "--ref-audio", + type=str, + default=None, + help="Reference audio file path or URL for voice cloning (Base task)", + ) + parser.add_argument( + "--ref-text", + type=str, + default=None, + help="Reference audio transcript for voice cloning (Base task)", + ) + parser.add_argument( + "--x-vector-only", + action="store_true", + help="Use x-vector only mode for voice cloning (no ICL)", + ) + + # Generation parameters + parser.add_argument( + "--max-new-tokens", + type=int, + default=None, + help="Maximum new tokens to generate", + ) + + # Output + parser.add_argument( + "--response-format", + type=str, + default="wav", + choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], + help="Audio output format (default: wav)", + ) + parser.add_argument( + "--output", + "-o", + type=str, + default=None, + help="Output audio file path (default: tts_output.wav)", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + run_tts_generation(args) diff --git a/examples/online_serving/qwen3_tts/run_server.sh b/examples/online_serving/qwen3_tts/run_server.sh new file mode 100755 index 000000000..203ed76e8 --- /dev/null +++ b/examples/online_serving/qwen3_tts/run_server.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Launch vLLM-Omni server for Qwen3-TTS models +# +# Usage: +# ./run_server.sh # Default: CustomVoice model +# ./run_server.sh CustomVoice # CustomVoice model +# ./run_server.sh VoiceDesign # VoiceDesign model +# ./run_server.sh Base # Base (voice clone) model + +set -e + +TASK_TYPE="${1:-CustomVoice}" + +case "$TASK_TYPE" in + CustomVoice) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" + ;; + VoiceDesign) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign" + ;; + Base) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-Base" + ;; + *) + echo "Unknown task type: $TASK_TYPE" + echo "Supported: CustomVoice, VoiceDesign, Base" + exit 1 + ;; +esac + +echo "Starting Qwen3-TTS server with model: $MODEL" + +vllm-omni serve "$MODEL" \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --host 0.0.0.0 \ + --port 8000 \ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --enforce-eager \ + --omni diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index e2d33f317..82b5a4fef 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -10,8 +10,10 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin -from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio -from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio, OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.serving_speech import ( + OmniOpenAIServingSpeech, +) from vllm_omni.outputs import OmniRequestOutput logger = logging.getLogger(__name__) @@ -265,3 +267,78 @@ class TestSpeechAPI: audio_obj = call_args[0] assert isinstance(audio_obj, CreateAudio) assert audio_obj.speed == 2.5 + + +class TestTTSMethods: + """Unit tests for TTS validation and parameter building.""" + + @pytest.fixture + def speech_server(self): + mock_engine_client = MagicMock() + mock_engine_client.errored = False + mock_engine_client.stage_list = None + mock_models = MagicMock() + mock_models.is_base_model.return_value = True + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=MagicMock(), + ) + + def test_is_tts_model(self, speech_server): + """Test TTS model detection.""" + # No stage_list -> False + assert speech_server._is_tts_model() is False + + # With qwen3_tts stage -> True + mock_stage = MagicMock() + mock_stage.model_stage = "qwen3_tts" + speech_server.engine_client.stage_list = [mock_stage] + assert speech_server._is_tts_model() is True + + def test_build_tts_prompt(self, speech_server): + """Test TTS prompt format.""" + prompt = speech_server._build_tts_prompt("Hello") + assert prompt == "<|im_start|>assistant\nHello<|im_end|>\n<|im_start|>assistant\n" + + def test_validate_tts_request_basic(self, speech_server): + """Test basic validation cases.""" + # Empty input + req = OpenAICreateSpeechRequest(input="") + assert speech_server._validate_tts_request(req) == "Input text cannot be empty" + + # Invalid language + req = OpenAICreateSpeechRequest(input="Hello", language="French") + assert "Invalid language" in speech_server._validate_tts_request(req) + + # Invalid speaker + req = OpenAICreateSpeechRequest(input="Hello", voice="Invalid") + assert "Invalid speaker" in speech_server._validate_tts_request(req) + + # Valid request + req = OpenAICreateSpeechRequest(input="Hello", voice="Vivian") + assert speech_server._validate_tts_request(req) is None + + def test_validate_tts_request_task_types(self, speech_server): + """Test task-specific validation.""" + # Base task requires ref_audio + req = OpenAICreateSpeechRequest(input="Hello", task_type="Base") + assert "ref_audio" in speech_server._validate_tts_request(req) + + # VoiceDesign requires instructions + req = OpenAICreateSpeechRequest(input="Hello", task_type="VoiceDesign") + assert "instructions" in speech_server._validate_tts_request(req) + + # ref_text only for Base + req = OpenAICreateSpeechRequest(input="Hello", ref_text="text") + assert "Base task" in speech_server._validate_tts_request(req) + + def test_build_tts_params(self, speech_server): + """Test TTS parameter building.""" + req = OpenAICreateSpeechRequest(input="Hello", voice="Ryan", language="English") + params = speech_server._build_tts_params(req) + + assert params["text"] == ["Hello"] + assert params["speaker"] == ["Ryan"] + assert params["language"] == ["English"] + assert params["task_type"] == ["CustomVoice"] diff --git a/vllm_omni/distributed/omni_connectors/utils/serialization.py b/vllm_omni/distributed/omni_connectors/utils/serialization.py index ee7ddf45f..74aa6d4d0 100644 --- a/vllm_omni/distributed/omni_connectors/utils/serialization.py +++ b/vllm_omni/distributed/omni_connectors/utils/serialization.py @@ -82,7 +82,11 @@ class OmniMsgpackEncoder: def _encode_tensor(self, tensor: torch.Tensor) -> dict[str, Any]: """Encode torch.Tensor to dict.""" - t = tensor.detach().contiguous().cpu().view(torch.uint8) + t = tensor.detach().contiguous().cpu() + # Handle 0-dimensional (scalar) tensors by reshaping to 1D first + if t.dim() == 0: + t = t.reshape(1) + t = t.view(torch.uint8) return { _TENSOR_MARKER: True, "dtype": str(tensor.dtype).removeprefix("torch."), diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 267d06291..d23460626 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -7,9 +7,15 @@ from pydantic import BaseModel, Field, field_validator class OpenAICreateSpeechRequest(BaseModel): input: str model: str | None = None - voice: Literal["alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"] - instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] + voice: str | None = Field( + default=None, + description="Voice to use. For OpenAI: alloy, echo, etc. For Qwen3-TTS: Vivian, Ryan, etc.", + ) + instructions: str | None = Field( + default=None, + description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)", + ) + response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" speed: float | None = Field( default=1.0, ge=0.25, @@ -17,6 +23,32 @@ class OpenAICreateSpeechRequest(BaseModel): ) stream_format: Literal["sse", "audio"] | None = "audio" + # Qwen3-TTS specific parameters + task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = Field( + default=None, + description="TTS task type: CustomVoice, VoiceDesign, or Base (voice clone)", + ) + language: str | None = Field( + default=None, + description="Language code (e.g., 'Chinese', 'English', 'Auto')", + ) + ref_audio: str | None = Field( + default=None, + description="Reference audio for voice cloning (Base task). URL, base64, or file path.", + ) + ref_text: str | None = Field( + default=None, + description="Transcript of reference audio for voice cloning (Base task)", + ) + x_vector_only_mode: bool | None = Field( + default=None, + description="Use speaker embedding only without in-context learning (Base task)", + ) + max_new_tokens: int | None = Field( + default=None, + description="Maximum tokens to generate", + ) + @field_validator("stream_format") @classmethod def validate_stream_format(cls, v: str) -> str: diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index c6b87810e..d5bc20cde 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1,4 +1,5 @@ import asyncio +from typing import Any from fastapi import Request from fastapi.responses import Response @@ -7,13 +8,149 @@ from vllm.logger import init_logger from vllm.utils import random_uuid from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin -from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio, OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.protocol.audio import ( + AudioResponse, + CreateAudio, + OpenAICreateSpeechRequest, +) from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) +# TTS Configuration (currently supports Qwen3-TTS) +_TTS_MODEL_STAGES: set[str] = {"qwen3_tts"} +_TTS_SPEAKERS: set[str] = { + "Vivian", + "Serena", + "Uncle_Fu", + "Dylan", + "Eric", + "Ryan", + "Aiden", + "One_Anna", + "Sohee", +} +_TTS_LANGUAGES: set[str] = {"Auto", "Chinese", "English", "Japanese", "Korean"} +_TTS_MAX_INSTRUCTIONS_LENGTH = 500 +_TTS_MAX_NEW_TOKENS_MIN = 1 +_TTS_MAX_NEW_TOKENS_MAX = 4096 + class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): + def _is_tts_model(self) -> bool: + """Check if the current model is a supported TTS model.""" + stage_list = getattr(self.engine_client, "stage_list", None) + if stage_list: + for stage in stage_list: + model_stage = getattr(stage, "model_stage", None) + if model_stage in _TTS_MODEL_STAGES: + return True + return False + + def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: + """Validate TTS request parameters. Returns error message or None.""" + task_type = request.task_type or "CustomVoice" + + # Validate input is not empty + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # Validate language + if request.language is not None and request.language not in _TTS_LANGUAGES: + return f"Invalid language '{request.language}'. Supported: {', '.join(sorted(_TTS_LANGUAGES))}" + + # Validate speaker for CustomVoice task + if task_type == "CustomVoice" and request.voice is not None: + if request.voice not in _TTS_SPEAKERS: + return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(_TTS_SPEAKERS))}" + + # Validate Base task requirements + if task_type == "Base": + if request.ref_audio is None: + return "Base task requires 'ref_audio' for voice cloning" + # Validate ref_audio format + if not (request.ref_audio.startswith(("http://", "https://")) or request.ref_audio.startswith("data:")): + return "ref_audio must be a URL (http/https) or base64 data URL (data:...)" + + # Validate cross-parameter dependencies + if task_type != "Base": + if request.ref_text is not None: + return "'ref_text' is only valid for Base task" + if request.x_vector_only_mode is not None: + return "'x_vector_only_mode' is only valid for Base task" + + # Validate VoiceDesign task requirements + if task_type == "VoiceDesign" and not request.instructions: + return "VoiceDesign task requires 'instructions' to describe the voice" + + # Validate instructions length + if request.instructions and len(request.instructions) > _TTS_MAX_INSTRUCTIONS_LENGTH: + return f"Instructions too long (max {_TTS_MAX_INSTRUCTIONS_LENGTH} characters)" + + # Validate max_new_tokens range + if request.max_new_tokens is not None: + if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: + return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" + if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: + return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" + + return None + + def _build_tts_prompt(self, text: str) -> str: + """Build TTS prompt from input text.""" + return f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n" + + def _build_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: + """Build TTS parameters from request. + + Processes each parameter if present, skips if not. + Values are wrapped in lists as required by the model. + """ + params: dict[str, Any] = {} + + # Text content (always required) + params["text"] = [request.input] + + # Task type + if request.task_type is not None: + params["task_type"] = [request.task_type] + else: + params["task_type"] = ["CustomVoice"] + + # Language + if request.language is not None: + params["language"] = [request.language] + else: + params["language"] = ["Auto"] + + # Speaker (voice) + if request.voice is not None: + params["speaker"] = [request.voice] + elif params["task_type"][0] == "CustomVoice": + params["speaker"] = ["Vivian"] # Default for CustomVoice + + # Instructions for style/emotion control + if request.instructions is not None: + params["instruct"] = [request.instructions] + else: + params["instruct"] = [""] + + # Voice clone parameters (used with Base task) + if request.ref_audio is not None: + params["ref_audio"] = [request.ref_audio] + if request.ref_text is not None: + params["ref_text"] = [request.ref_text] + if request.x_vector_only_mode is not None: + params["x_vector_only_mode"] = [request.x_vector_only_mode] + + # Generation parameters + if request.max_new_tokens is not None: + params["max_new_tokens"] = [request.max_new_tokens] + else: + params["max_new_tokens"] = [2048] + + return params + async def create_speech( self, request: OpenAICreateSpeechRequest, @@ -26,8 +163,16 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): for the API specification. This API mimics the OpenAI Create Speech API. - NOTE: This implementation does not currently support streaming audio - generation. The full audio is generated before the response is sent. + For Qwen3-TTS models, additional parameters are supported: + - task_type: "CustomVoice", "VoiceDesign", or "Base" + - language: Language code (e.g., "Chinese", "English", "Auto") + - voice: Speaker name (e.g., "Vivian", "Ryan") for CustomVoice + - instructions: Voice style/emotion instructions + - ref_audio: Reference audio for voice cloning (Base task) + - ref_text: Transcript of reference audio (Base task) + - x_vector_only_mode: Use speaker embedding only (Base task) + + NOTE: Streaming audio generation is not currently supported. """ error_check_ret = await self._check_model(request) @@ -41,11 +186,38 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request_id = f"speech-{random_uuid()}" try: - prompt = {"prompt": request.input} + if self._is_tts_model(): + # Validate TTS parameters + validation_error = self._validate_tts_request(request) + if validation_error: + return self.create_error_response(validation_error) + + # Build TTS parameters and prompt + tts_params = self._build_tts_params(request) + prompt_text = self._build_tts_prompt(request.input) + prompt = { + "prompt": prompt_text, + "additional_information": tts_params, + } + else: + # Fallback for unsupported models + tts_params = {} + prompt = {"prompt": request.input} + + logger.info( + "TTS speech request %s: text=%r, task_type=%s", + request_id, + request.input[:50] + "..." if len(request.input) > 50 else request.input, + tts_params.get("task_type", ["unknown"])[0], + ) + sampling_params_list = self.engine_client.default_sampling_params_list generator = self.engine_client.generate( - prompt=prompt, request_id=request_id, sampling_params_list=sampling_params_list + prompt=prompt, + request_id=request_id, + sampling_params_list=sampling_params_list, + output_modalities=["audio"], ) final_output: OmniRequestOutput | None = None @@ -55,15 +227,36 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if final_output is None: return self.create_error_response("No output generated from the model.") - if final_output.final_output_type != "audio": - return self.create_error_response(f"Unexpected final output type: {final_output.final_output_type}") + # Extract audio from output + # Audio can be in final_output.multimodal_output or final_output.request_output.multimodal_output + audio_output = None + if hasattr(final_output, "multimodal_output") and final_output.multimodal_output: + audio_output = final_output.multimodal_output + if (not audio_output or "audio" not in audio_output) and hasattr(final_output, "request_output"): + if final_output.request_output and hasattr(final_output.request_output, "multimodal_output"): + audio_output = final_output.request_output.multimodal_output + + if not audio_output or "audio" not in audio_output: + return self.create_error_response("TTS model did not produce audio output.") + + audio_tensor = audio_output["audio"] + sample_rate = audio_output.get("sr", 24000) + if hasattr(sample_rate, "item"): + sample_rate = sample_rate.item() + + # Convert tensor to numpy + if hasattr(audio_tensor, "float"): + audio_tensor = audio_tensor.float().detach().cpu().numpy() + + # Squeeze batch dimension if present, but preserve channel dimension for stereo + if audio_tensor.ndim > 1: + audio_tensor = audio_tensor.squeeze() - audio_tensor = final_output.request_output.multimodal_output["audio"].float().detach().cpu().numpy() audio_obj = CreateAudio( audio_tensor=audio_tensor, - sample_rate=24000, - response_format=request.response_format, - speed=request.speed, + sample_rate=int(sample_rate), + response_format=request.response_format or "wav", + speed=request.speed or 1.0, stream_format=request.stream_format, base64_encode=False, ) @@ -74,7 +267,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): except asyncio.CancelledError: return self.create_error_response("Client disconnected") except ValueError as e: - # TODO: Use a vllm-specific Validation Error return self.create_error_response(str(e)) except Exception as e: - return self.create_error_response(f"{e} {e.__cause__}") + logger.exception("Speech generation failed: %s", e) + return self.create_error_response(f"Speech generation failed: {e}") diff --git a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py index 11c715f5f..31abe254b 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py @@ -39,7 +39,6 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from transformers.processing_utils import Unpack from transformers.utils import ModelOutput, auto_docstring, logging from transformers.utils.deprecation import deprecate_kwarg -from transformers.utils.generic import check_model_inputs from .configuration_qwen3_tts_tokenizer_v2 import ( Qwen3TTSTokenizerV2Config, @@ -495,7 +494,8 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr # Initialize weights and apply final processing self.post_init() - @check_model_inputs + # Note: @check_model_inputs decorator removed for vLLM compatibility + # The decorator causes "unexpected keyword argument 'inputs_embeds'" error @auto_docstring def forward( self,