[Frontend] Speaker embedding support for speech and voices APIs (#1227)

Signed-off-by: marksverdhei <marksverdhei@hotmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Signed-off-by: linyueqian <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: linyueqian <linyueqian@outlook.com>

tests/entrypoints/openai_api/test_serving_speech.py

@@ -235,6 +235,8 @@ def test_app(mocker: MockerFixture):
                         "created_at": info.get("created_at", 0),
                         "file_size": info.get("file_size", 0),
                         "mime_type": info.get("mime_type", ""),
+                        "embedding_source": info.get("embedding_source", "audio"),
+                        "embedding_dim": info.get("embedding_dim"),
                     }
                 )
         return {"voices": speakers, "uploaded_voices": uploaded_voices}
@@ -243,9 +245,21 @@ def test_app(mocker: MockerFixture):
     app.add_api_route("/v1/audio/speech/batch", speech_server.create_speech_batch, methods=["POST"])
 
     # Add upload_voice endpoint
-    async def upload_voice(audio_sample: UploadFile = File(...), consent: str = Form(...), name: str = Form(...)):
+    async def upload_voice(
+        audio_sample: UploadFile | None = File(None),
+        speaker_embedding: str | None = Form(None),
+        consent: str = Form(...),
+        name: str = Form(...),
+    ):
         try:
-            result = await speech_server.upload_voice(audio_sample, consent, name)
+            if speaker_embedding is not None and audio_sample is not None:
+                raise ValueError("'audio_sample' and 'speaker_embedding' are mutually exclusive")
+            if speaker_embedding is not None:
+                result = await speech_server.upload_voice_embedding(speaker_embedding, consent, name)
+            elif audio_sample is not None:
+                result = await speech_server.upload_voice(audio_sample, consent, name)
+            else:
+                raise ValueError("Either 'audio_sample' or 'speaker_embedding' must be provided")
             return {"success": True, "voice": result}
         except ValueError as e:
             raise HTTPException(status_code=400, detail=str(e))
@@ -446,13 +460,13 @@ class TestSpeechAPI:
         response = client.post("/v1/audio/voices", files=files, data=data)
         assert response.status_code == 422  # Validation error
 
-        # Missing file
+        # Missing both audio_sample and speaker_embedding
         data = {
             "consent": "user_consent_123",
             "name": "test_voice6",
         }
         response = client.post("/v1/audio/voices", data=data)
-        assert response.status_code == 422  # Validation error
+        assert response.status_code == 400
 
     def test_delete_voice_success(self, client):
         """Test successful voice deletion."""
@@ -489,6 +503,89 @@ class TestSpeechAPI:
         result = response.json()
         assert "not found" in result["detail"]
 
+    # ── speaker_embedding upload via voices endpoint ──
+
+    def test_upload_voice_embedding_success(self, client):
+        """Upload a voice via speaker_embedding JSON."""
+        import json
+
+        emb = [0.1] * 1024
+        data = {
+            "speaker_embedding": json.dumps(emb),
+            "consent": "consent_emb_1",
+            "name": "emb_voice",
+        }
+        response = client.post("/v1/audio/voices", data=data)
+        assert response.status_code == 200, f"Upload failed: {response.text}"
+        result = response.json()
+        assert result["success"] is True
+        voice = result["voice"]
+        assert voice["name"] == "emb_voice"
+        assert voice["embedding_source"] == "direct"
+        assert voice["embedding_dim"] == 1024
+        # Clean up
+        client.delete("/v1/audio/voices/emb_voice")
+
+    def test_upload_voice_embedding_appears_in_listing(self, client):
+        """Embedding-uploaded voice appears in list with correct source."""
+        import json
+
+        emb = [0.2] * 2048
+        data = {
+            "speaker_embedding": json.dumps(emb),
+            "consent": "consent_list",
+            "name": "listed_emb_voice",
+        }
+        response = client.post("/v1/audio/voices", data=data)
+        assert response.status_code == 200
+
+        listing = client.get("/v1/audio/voices").json()
+        uploaded = {v["name"]: v for v in listing["uploaded_voices"]}
+        assert "listed_emb_voice" in uploaded
+        assert uploaded["listed_emb_voice"]["embedding_source"] == "direct"
+        assert uploaded["listed_emb_voice"]["embedding_dim"] == 2048
+        # Clean up
+        client.delete("/v1/audio/voices/listed_emb_voice")
+
+    def test_upload_voice_embedding_and_audio_mutually_exclusive(self, client):
+        """Providing both audio_sample and speaker_embedding returns 400."""
+        import json
+
+        emb = [0.1] * 1024
+        files = {"audio_sample": ("test.wav", b"fake", "audio/wav")}
+        data = {
+            "speaker_embedding": json.dumps(emb),
+            "consent": "consent_mx",
+            "name": "mx_voice",
+        }
+        response = client.post("/v1/audio/voices", files=files, data=data)
+        assert response.status_code == 400
+        assert "mutually exclusive" in response.json()["detail"]
+
+    def test_upload_voice_embedding_invalid_json(self, client):
+        """Invalid JSON in speaker_embedding returns 400."""
+        data = {
+            "speaker_embedding": "not valid json [[[",
+            "consent": "consent_bad",
+            "name": "bad_json_voice",
+        }
+        response = client.post("/v1/audio/voices", data=data)
+        assert response.status_code == 400
+        assert "JSON" in response.json()["detail"]
+
+    def test_upload_voice_embedding_nan_rejected(self, client):
+        """NaN values in speaker_embedding return 400."""
+        import json
+
+        data = {
+            "speaker_embedding": json.dumps([0.1] * 1023 + [float("nan")]),
+            "consent": "consent_nan",
+            "name": "nan_voice",
+        }
+        response = client.post("/v1/audio/voices", data=data)
+        assert response.status_code == 400
+        assert "finite" in response.json()["detail"]
+
 
 class TestTTSMethods:
     """Unit tests for TTS validation and parameter building."""
@@ -610,6 +707,96 @@ class TestTTSMethods:
         result = speech_server._validate_tts_request(req)
         assert "does not support CustomVoice" in result
 
+    # ── speaker_embedding validation ──
+
+    def test_speaker_embedding_valid_base_task(self, speech_server):
+        """speaker_embedding with Base task, x_vector_only_mode, and no ref_audio is accepted."""
+        emb = [0.1] * 1024
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
+        assert speech_server._validate_tts_request(req) is None
+
+    def test_speaker_embedding_auto_sets_x_vector_only_mode(self, speech_server):
+        """speaker_embedding auto-implies x_vector_only_mode, so validation passes."""
+        emb = [0.1] * 1024
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
+        result = speech_server._validate_tts_request(req)
+        assert result is None
+        assert req.x_vector_only_mode is True
+
+    def test_speaker_embedding_wrong_task_type(self, speech_server):
+        """speaker_embedding is only valid for Base task."""
+        emb = [0.1] * 1024
+        req = OpenAICreateSpeechRequest(
+            input="Hello", task_type="VoiceDesign", speaker_embedding=emb, instructions="warm"
+        )
+        result = speech_server._validate_tts_request(req)
+        assert "only valid for Base task" in result
+
+    def test_speaker_embedding_mutually_exclusive_with_ref_audio(self, speech_server):
+        """speaker_embedding and ref_audio cannot both be provided (pydantic validation)."""
+        emb = [0.1] * 1024
+        with pytest.raises(ValidationError, match="mutually exclusive"):
+            OpenAICreateSpeechRequest(
+                input="Hello", task_type="Base", speaker_embedding=emb, ref_audio="data:audio/wav;base64,abc"
+            )
+
+    def test_speaker_embedding_nan_rejected(self, speech_server):
+        """NaN values in speaker_embedding are rejected at parse level."""
+        with pytest.raises(ValidationError, match="finite"):
+            OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=[0.1] * 1023 + [float("nan")])
+
+    def test_speaker_embedding_inf_rejected(self, speech_server):
+        """Inf values in speaker_embedding are rejected at parse level."""
+        with pytest.raises(ValidationError, match="finite"):
+            OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=[float("inf")] + [0.1] * 1023)
+
+    def test_speaker_embedding_empty_list_rejected(self, speech_server):
+        """Empty speaker_embedding list is rejected."""
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=[])
+        result = speech_server._validate_tts_request(req)
+        assert "non-empty" in result
+
+    def test_speaker_embedding_wrong_dims_accepted(self, speech_server):
+        """Non-standard dimensions pass validation (warning only, not an error)."""
+        emb = [0.1] * 512  # not 1024 or 2048
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
+        result = speech_server._validate_tts_request(req)
+        assert result is None
+
+    def test_speaker_embedding_2048_dims_accepted(self, speech_server):
+        """2048-dim embedding (1.7B model) is accepted without warning."""
+        emb = [0.1] * 2048
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb, x_vector_only_mode=True)
+        assert speech_server._validate_tts_request(req) is None
+
+    def test_base_task_requires_ref_audio_or_speaker_embedding(self, speech_server):
+        """Base task without ref_audio or speaker_embedding is rejected."""
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base")
+        result = speech_server._validate_tts_request(req)
+        assert "ref_audio" in result and "speaker_embedding" in result
+
+    # ── speaker_embedding in _build_tts_params ──
+
+    def test_build_tts_params_with_speaker_embedding(self, speech_server):
+        """speaker_embedding produces voice_clone_prompt and x_vector_only_mode."""
+        emb = [0.1] * 1024
+        req = OpenAICreateSpeechRequest(input="Hello", task_type="Base", speaker_embedding=emb)
+        params = speech_server._build_tts_params(req)
+
+        assert "voice_clone_prompt" in params
+        vcp = params["voice_clone_prompt"][0]
+        assert "ref_spk_embedding" in vcp
+        # Stored as plain list (not tensor) so it survives msgspec IPC serialization
+        assert isinstance(vcp["ref_spk_embedding"], list)
+        assert len(vcp["ref_spk_embedding"]) == 1024
+        assert params["x_vector_only_mode"] == [True]
+
+    def test_build_tts_params_without_speaker_embedding(self, speech_server):
+        """Without speaker_embedding, voice_clone_prompt is not set."""
+        req = OpenAICreateSpeechRequest(input="Hello", voice="Ryan", language="English")
+        params = speech_server._build_tts_params(req)
+        assert "voice_clone_prompt" not in params
+
     def test_build_tts_params(self, speech_server):
         """Test TTS parameter building."""
         req = OpenAICreateSpeechRequest(input="Hello", voice="Ryan", language="English")