[Frontend] Speaker embedding support for speech and voices APIs (#1227)

Signed-off-by: marksverdhei <marksverdhei@hotmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Signed-off-by: linyueqian <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-03-25 19:02:48 +00:00
1 parent d2b9f9ff0f
commit 29936a3edd
10 files changed
+1093 -23

No files matched your search

@@ -0,0 +1,249 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E tests for speaker_embedding API parameter.
Tests Base models (0.6B-Base and 1.7B-Base) which support voice cloning
via pre-computed ECAPA-TDNN embeddings, bypassing ref_audio extraction.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
import struct
from pathlib import Path
import httpx
import pytest
from tests.conftest import OmniServer
from tests.utils import hardware_test
MODEL_BASE = "Qwen/Qwen3-TTS-12Hz-0.6B-Base"
MODEL_BASE_1_7B = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
# A synthetic 1024-dim speaker embedding (all 0.1 — not a real voice, but
# exercises the full code path through the talker's _build_prompt_embeds).
DUMMY_EMBEDDING_1024 = [0.1] * 1024
DUMMY_EMBEDDING_2048 = [0.1] * 2048
SYN_TEXT = "Hello."
MIN_AUDIO_BYTES = 2000
# Limit generation to keep tests fast (dummy embeddings produce nonsensical
# output that may never hit a natural stop token).
MAX_NEW_TOKENS = 256
def get_stage_config():
return str(
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "qwen3_tts.yaml"
)
def _server_args():
return [
"--stage-configs-path",
get_stage_config(),
"--stage-init-timeout",
"120",
"--trust-remote-code",
"--enforce-eager",
"--disable-log-stats",
]
def verify_wav_audio(content: bytes) -> bool:
if len(content) < 44:
return False
return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
def assert_not_silence(pcm_bytes: bytes):
"""Assert PCM16 samples are not all identical (e.g. all-silence)."""
samples = struct.unpack(f"<{len(pcm_bytes) // 2}h", pcm_bytes)
unique = set(samples)
assert len(unique) > 1, f"All-silence detected: {len(samples)} samples, unique values: {unique}"
# ── 0.6B-Base model tests ──
@pytest.fixture(scope="module")
def base_server():
"""Start vLLM-Omni server with 0.6B-Base model."""
with OmniServer(MODEL_BASE, _server_args()) as server:
yield server
class TestSpeakerEmbeddingBase:
"""Speaker embedding tests against the 0.6B-Base model (supports Base task)."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_produces_audio(self, base_server) -> None:
"""speaker_embedding with Base task produces valid WAV audio."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "wav",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV"
assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_pcm_not_silence(self, base_server) -> None:
"""speaker_embedding PCM output contains real audio, not all-silence."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "pcm",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert len(response.content) > MIN_AUDIO_BYTES
assert_not_silence(response.content)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_streaming(self, base_server) -> None:
"""speaker_embedding works with streaming PCM output."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "pcm",
"stream": True,
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert "audio/pcm" in response.headers.get("content-type", "")
assert len(response.content) > MIN_AUDIO_BYTES
assert_not_silence(response.content)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_mutually_exclusive_with_ref_audio(self, base_server) -> None:
"""Sending both speaker_embedding and ref_audio returns 400."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"ref_audio": "https://example.com/audio.wav",
"response_format": "wav",
}
with httpx.Client(timeout=30.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
assert "mutually exclusive" in response.text
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_speaker_embedding_empty_rejected(self, base_server) -> None:
"""Empty speaker_embedding list returns 400."""
url = f"http://{base_server.host}:{base_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": [],
"response_format": "wav",
}
with httpx.Client(timeout=30.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 400, f"Expected 400, got {response.status_code}: {response.text}"
assert "non-empty" in response.text
# ── 1.7B-Base model tests (2048-dim embeddings) ──
@pytest.fixture(scope="module")
def base_1_7b_server():
"""Start vLLM-Omni server with 1.7B-Base model."""
with OmniServer(MODEL_BASE_1_7B, _server_args()) as server:
yield server
class TestSpeakerEmbedding1_7B:
"""Speaker embedding tests against the 1.7B-Base model (2048-dim embeddings)."""
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_2048_dim_embedding_produces_audio(self, base_1_7b_server) -> None:
"""2048-dim speaker_embedding with 1.7B-Base model produces valid WAV audio."""
url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE_1_7B,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_2048,
"x_vector_only_mode": True,
"response_format": "wav",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
assert response.status_code == 200, f"Request failed: {response.text}"
assert response.headers.get("content-type") == "audio/wav"
assert verify_wav_audio(response.content), "Response is not valid WAV"
assert len(response.content) > MIN_AUDIO_BYTES, f"Audio too small: {len(response.content)} bytes"
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_1024_dim_on_1_7b_model_rejected_or_errors(self, base_1_7b_server) -> None:
"""1024-dim embedding on a 1.7B model (expects 2048) should fail gracefully."""
url = f"http://{base_1_7b_server.host}:{base_1_7b_server.port}/v1/audio/speech"
payload = {
"model": MODEL_BASE_1_7B,
"input": SYN_TEXT,
"task_type": "Base",
"speaker_embedding": DUMMY_EMBEDDING_1024,
"x_vector_only_mode": True,
"response_format": "wav",
"max_new_tokens": MAX_NEW_TOKENS,
}
with httpx.Client(timeout=120.0) as client:
response = client.post(url, json=payload)
# Wrong dimensions should produce an error, not silent garbage.
assert response.status_code != 200 or len(response.content) < MIN_AUDIO_BYTES, (
"Expected failure or degenerate output with wrong embedding dimensions"
)
@@ -235,6 +235,8 @@ def test_app(mocker: MockerFixture):
"created_at": info.get("created_at", 0),
"file_size": info.get("file_size", 0),
"mime_type": info.get("mime_type", ""),
"embedding_source": info.get("embedding_source", "audio"),
"embedding_dim": info.get("embedding_dim"),
}
)
return {"voices": speakers, "uploaded_voices": uploaded_voices}
@@ -243,9 +245,21 @@ def test_app(mocker: MockerFixture):
app.add_api_route("/v1/audio/speech/batch", speech_server.create_speech_batch, methods=["POST"])
# Add upload_voice endpoint
async def upload_voice(audio_sample: UploadFile = File(...), consent: str = Form(...), name: str = Form(...)):
async def upload_voice(
audio_sample: UploadFile | None = File(None),
speaker_embedding: str | None = Form(None),
consent: str = Form(...),
name: str = Form(...),
):
try:
result = await speech_server.upload_voice(audio_sample, consent, name)
if speaker_embedding is not None and audio_sample is not None:
raise ValueError("'audio_sample' and 'speaker_embedding' are mutually exclusive")
if speaker_embedding is not None:
result = await speech_server.upload_voice_embedding(speaker_embedding, consent, name)
elif audio_sample is not None:
result = await speech_server.upload_voice(audio_sample, consent, name)
else:
raise ValueError("Either 'audio_sample' or 'speaker_embedding' must be provided")
return {"success": True, "voice": result}
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
@@ -446,13 +460,13 @@ class TestSpeechAPI:
response = client.post("/v1/audio/voices", files=files, data=data)
assert response.status_code == 422 # Validation error
# Missing file
# Missing both audio_sample and speaker_embedding
data = {
"consent": "user_consent_123",
"name": "test_voice6",
}
response = client.post("/v1/audio/voices", data=data)
assert response.status_code == 422 # Validation error
assert response.status_code == 400
def test_delete_voice_success(self, client):
"""Test successful voice deletion."""
@@ -489,6 +503,89 @@ class TestSpeechAPI:
result = response.json()
assert "not found" in result["detail"]
# ── speaker_embedding upload via voices endpoint ──
def test_upload_voice_embedding_success(self, client):
"""Upload a voice via speaker_embedding JSON."""
import json
emb = [0.1] * 1024
data = {
"speaker_embedding": json.dumps(emb),
"consent": "consent_emb_1",
"name": "emb_voice",
}
response = client.post("/v1/audio/voices", data=data)
assert response.status_code == 200, f"Upload failed: {response.text}"
result = response.json()
assert result["success"] is True
voice = result["voice"]
assert voice["name"] == "emb_voice"
assert voice["embedding_source"] == "direct"
assert voice["embedding_dim"] == 1024
# Clean up
client.delete("/v1/audio/voices/emb_voice")
def test_upload_voice_embedding_appears_in_listing(self, client):
"""Embedding-uploaded voice appears in list with correct source."""
import json
emb = [0.2] * 2048
data = {
"speaker_embedding": json.dumps(emb),
"consent": "consent_list",
"name": "listed_emb_voice",
}
response = client.post("/v1/audio/voices", data=data)
assert response.status_code == 200
listing = client.get("/v1/audio/voices").json()
uploaded = {v["name"]: v for v in listing["uploaded_voices"]}
assert "listed_emb_voice" in uploaded
assert uploaded["listed_emb_voice"]["embedding_source"] == "direct"
assert uploaded["listed_emb_voice"]["embedding_dim"] == 2048
# Clean up
client.delete("/v1/audio/voices/listed_emb_voice")
def test_upload_voice_embedding_and_audio_mutually_exclusive(self, client):
"""Providing both audio_sample and speaker_embedding returns 400."""
import json
emb = [0.1] * 1024
files = {"audio_sample": ("test.wav", b"fake", "audio/wav")}
data = {
"speaker_embedding": json.dumps(emb),
"consent": "consent_mx",
"name": "mx_voice",
}
response = client.post("/v1/audio/voices", files=files, data=data)
assert response.status_code == 400
assert "mutually exclusive" in response.json()["detail"]
def test_upload_voice_embedding_invalid_json(self, client):
"""Invalid JSON in speaker_embedding returns 400."""
data = {
"speaker_embedding": "not valid json [[[",
"consent": "consent_bad",
"name": "bad_json_voice",
}
response = client.post("/v1/audio/voices", data=data)
assert response.status_code == 400
assert "JSON" in response.json()["detail"]
def test_upload_voice_embedding_nan_rejected(self, client):
"""NaN values in speaker_embedding return 400."""
import json
data = {
"speaker_embedding": json.dumps([0.1] * 1023 + [float("nan")]),
"consent": "consent_nan",
"name": "nan_voice",
}
response = client.post("/v1/audio/voices", data=data)
assert response.status_code == 400
assert "finite" in response.json()["detail"]
class TestTTSMethods:
"""Unit tests for TTS validation and parameter building."""
@@ -610,6 +707,96 @@ class TestTTSMethods:
result = speech_server._validate_tts_request(req)
assert "does not support CustomVoice" in result
# ── speaker_embedding validation ──
def test_speaker_embedding_valid_base_task(self, speech_server):
"""speaker_embedding with Base task, x_vector_only_mode, and no ref_audio is accepted."""
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
assert speech_server._validate_tts_request(req) is None
def test_speaker_embedding_auto_sets_x_vector_only_mode(self, speech_server):
"""speaker_embedding auto-implies x_vector_only_mode, so validation passes."""
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
result = speech_server._validate_tts_request(req)
assert result is None
assert req.x_vector_only_mode is True
def test_speaker_embedding_wrong_task_type(self, speech_server):
"""speaker_embedding is only valid for Base task."""
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(
input="Hello", task_type="VoiceDesign", speaker_embedding=emb, instructions="warm"
)
result = speech_server._validate_tts_request(req)
assert "only valid for Base task" in result
def test_speaker_embedding_mutually_exclusive_with_ref_audio(self, speech_server):
"""speaker_embedding and ref_audio cannot both be provided (pydantic validation)."""
emb = [0.1] * 1024
with pytest.raises(ValidationError, match="mutually exclusive"):
OpenAICreateSpeechRequest(
input="Hello", task_type="Base", speaker_embedding=emb, ref_audio="data:audio/wav;base64,abc"
)
def test_speaker_embedding_nan_rejected(self, speech_server):
"""NaN values in speaker_embedding are rejected at parse level."""
with pytest.raises(ValidationError, match="finite"):
OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=[0.1] * 1023 + [float("nan")])
def test_speaker_embedding_inf_rejected(self, speech_server):
"""Inf values in speaker_embedding are rejected at parse level."""
with pytest.raises(ValidationError, match="finite"):
OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=[float("inf")] + [0.1] * 1023)
def test_speaker_embedding_empty_list_rejected(self, speech_server):
"""Empty speaker_embedding list is rejected."""
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=[])
result = speech_server._validate_tts_request(req)
assert "non-empty" in result
def test_speaker_embedding_wrong_dims_accepted(self, speech_server):
"""Non-standard dimensions pass validation (warning only, not an error)."""
emb = [0.1] * 512 # not 1024 or 2048
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
result = speech_server._validate_tts_request(req)
assert result is None
def test_speaker_embedding_2048_dims_accepted(self, speech_server):
"""2048-dim embedding (1.7B model) is accepted without warning."""
emb = [0.1] * 2048
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
assert speech_server._validate_tts_request(req) is None
def test_base_task_requires_ref_audio_or_speaker_embedding(self, speech_server):
"""Base task without ref_audio or speaker_embedding is rejected."""
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base")
result = speech_server._validate_tts_request(req)
assert "ref_audio" in result and "speaker_embedding" in result
# ── speaker_embedding in _build_tts_params ──
def test_build_tts_params_with_speaker_embedding(self, speech_server):
"""speaker_embedding produces voice_clone_prompt and x_vector_only_mode."""
emb = [0.1] * 1024
req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
params = speech_server._build_tts_params(req)
assert "voice_clone_prompt" in params
vcp = params["voice_clone_prompt"][0]
assert "ref_spk_embedding" in vcp
# Stored as plain list (not tensor) so it survives msgspec IPC serialization
assert isinstance(vcp["ref_spk_embedding"], list)
assert len(vcp["ref_spk_embedding"]) == 1024
assert params["x_vector_only_mode"] == [True]
def test_build_tts_params_without_speaker_embedding(self, speech_server):
"""Without speaker_embedding, voice_clone_prompt is not set."""
req = OpenAICreateSpeechRequest(input="Hello", voice="Ryan", language="English")
params = speech_server._build_tts_params(req)
assert "voice_clone_prompt" not in params
def test_build_tts_params(self, speech_server):
"""Test TTS parameter building."""
req = OpenAICreateSpeechRequest(input="Hello", voice="Ryan", language="English")
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the SLERP interpolation math in speaker_embedding_interpolation.py."""
import sys
from pathlib import Path
import numpy as np
import pytest
# Import the slerp function from the example script.
_examples_dir = str(Path(__file__).parent.parent.parent / "examples" / "online_serving" / "qwen3_tts")
sys.path.insert(0, _examples_dir)
from speaker_embedding_interpolation import slerp # noqa: E402
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestSlerp:
def test_endpoints(self):
"""t=0 returns v0, t=1 returns v1."""
v0 = np.random.randn(1024).astype(np.float64)
v1 = np.random.randn(1024).astype(np.float64)
np.testing.assert_allclose(slerp(v0, v1, 0.0), v0, atol=1e-6)
np.testing.assert_allclose(slerp(v0, v1, 1.0), v1, atol=1e-6)
def test_midpoint_unit_norm(self):
"""Midpoint of two unit vectors should also be approximately unit norm."""
v0 = np.random.randn(1024)
v0 /= np.linalg.norm(v0)
v1 = np.random.randn(1024)
v1 /= np.linalg.norm(v1)
mid = slerp(v0, v1, 0.5)
assert abs(np.linalg.norm(mid) - 1.0) < 0.05
def test_parallel_vectors_fallback(self):
"""Parallel vectors (omega~0) fall back to lerp without error."""
v0 = np.ones(1024)
v1 = np.ones(1024) * 1.001 # nearly parallel
result = slerp(v0, v1, 0.5)
expected = 0.5 * v0 + 0.5 * v1
np.testing.assert_allclose(result, expected, atol=1e-4)
def test_antiparallel_vectors(self):
"""Antiparallel vectors (omega~pi) should not produce NaN."""
v0 = np.ones(1024)
v1 = -np.ones(1024)
result = slerp(v0, v1, 0.5)
assert not np.any(np.isnan(result))
def test_output_shape_matches_input(self):
"""Output shape must match input shape."""
v0 = np.random.randn(2048)
v1 = np.random.randn(2048)
assert slerp(v0, v1, 0.3).shape == (2048,)