[Frontend] Opt-in WebSocket TTS split_granularity and session seed (#7046)
Signed-off-by: rk9595 <rakesh.kariya@somaiya.edu> Signed-off-by: Rakesh Kariya <rakesh.kariya@somaiya.edu> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
8 files changed
+610
-52
No files matched your search
@@ -197,7 +197,7 @@ class TestStreamingSpeechWebSocket:
|
||||
|
||||
error = ws.receive_json()
|
||||
assert error["type"] == "error"
|
||||
assert "while input is buffered" in error["message"]
|
||||
assert "while an utterance is in progress" in error["message"]
|
||||
|
||||
# The buffered text survives the rejected reconfiguration.
|
||||
ws.send_json({"type": "input.done"})
|
||||
@@ -742,6 +742,8 @@ class TestStreamingSpeechWebSocket:
|
||||
config.speaker_embedding = None
|
||||
config.stream_audio = True
|
||||
config.word_timestamps = False
|
||||
config.seed = None
|
||||
config.non_streaming_mode = None
|
||||
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
asyncio.run(
|
||||
@@ -758,6 +760,163 @@ class TestStreamingSpeechWebSocket:
|
||||
assert websocket.send_json.await_count == 2
|
||||
|
||||
|
||||
class TestWebSocketSentenceSplitting:
|
||||
def test_sentence_granularity_emits_one_request_per_sentence(self, mocker: MockerFixture):
|
||||
app, speech_service = _build_test_app(mocker=mocker)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"split_granularity": "sentence",
|
||||
}
|
||||
)
|
||||
ws.send_json({"type": "input.text", "text": "Hello world. How are you? "})
|
||||
ws.send_json({"type": "input.done"})
|
||||
|
||||
first = ws.receive_json()
|
||||
assert first["sentence_index"] == 0
|
||||
assert first["sentence_text"] == "Hello world."
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
|
||||
second = ws.receive_json()
|
||||
assert second["sentence_index"] == 1
|
||||
assert second["sentence_text"] == "How are you?"
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
|
||||
assert ws.receive_json() == {
|
||||
"type": "session.done",
|
||||
"utterance_index": 0,
|
||||
"total_sentences": 2,
|
||||
}
|
||||
|
||||
assert speech_service._generate_audio_bytes.await_count == 2
|
||||
assert [call.args[0].input for call in speech_service._generate_audio_bytes.await_args_list] == [
|
||||
"Hello world.",
|
||||
"How are you?",
|
||||
]
|
||||
|
||||
def test_sentence_granularity_emits_before_input_done(self, mocker: MockerFixture):
|
||||
app, speech_service = _build_test_app(mocker=mocker)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"split_granularity": "sentence",
|
||||
}
|
||||
)
|
||||
ws.send_json({"type": "input.text", "text": "First sentence. "})
|
||||
|
||||
start = ws.receive_json()
|
||||
assert start["sentence_text"] == "First sentence."
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
assert speech_service._generate_audio_bytes.await_count == 1
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {
|
||||
"type": "session.done",
|
||||
"utterance_index": 0,
|
||||
"total_sentences": 1,
|
||||
}
|
||||
|
||||
def test_indic_danda_splits_without_latin_period(self, mocker: MockerFixture):
|
||||
app, speech_service = _build_test_app(mocker=mocker)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"language": "Auto",
|
||||
"split_granularity": "sentence",
|
||||
}
|
||||
)
|
||||
ws.send_json({"type": "input.text", "text": "नमस्ते। कैसे हो?"})
|
||||
ws.send_json({"type": "input.done"})
|
||||
|
||||
first = ws.receive_json()
|
||||
assert first["sentence_text"] == "नमस्ते।"
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
|
||||
second = ws.receive_json()
|
||||
assert second["sentence_text"] == "कैसे हो?"
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
assert ws.receive_json()["total_sentences"] == 2
|
||||
|
||||
def test_session_config_rejected_after_a_split_unit_was_emitted(self, mocker: MockerFixture):
|
||||
"""The splitter buffer is empty here, but the utterance is still open."""
|
||||
app, speech_service = _build_test_app(mocker=mocker)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.config",
|
||||
"voice": "Vivian",
|
||||
"split_granularity": "sentence",
|
||||
}
|
||||
)
|
||||
ws.send_json({"type": "input.text", "text": "First sentence. "})
|
||||
ws.receive_json()
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
|
||||
ws.send_json({"type": "session.config", "voice": "Serena"})
|
||||
error = ws.receive_json()
|
||||
assert error["type"] == "error"
|
||||
assert "while an utterance is in progress" in error["message"]
|
||||
|
||||
ws.send_json({"type": "input.text", "text": "Second sentence. "})
|
||||
assert ws.receive_json()["sentence_index"] == 1
|
||||
ws.receive_bytes()
|
||||
assert ws.receive_json()["type"] == "audio.done"
|
||||
|
||||
ws.send_json({"type": "input.done"})
|
||||
assert ws.receive_json() == {
|
||||
"type": "session.done",
|
||||
"utterance_index": 0,
|
||||
"total_sentences": 2,
|
||||
}
|
||||
|
||||
# Reconfiguration is allowed again once the utterance closed.
|
||||
ws.send_json({"type": "session.config", "voice": "Serena"})
|
||||
ws.send_json({"type": "input.text", "text": "Third."})
|
||||
ws.send_json({"type": "input.done"})
|
||||
ws.receive_json()
|
||||
ws.receive_bytes()
|
||||
ws.receive_json()
|
||||
ws.receive_json()
|
||||
|
||||
voices = [call.args[0].voice for call in speech_service._generate_audio_bytes.await_args_list]
|
||||
assert voices == ["Vivian", "Vivian", "Serena"]
|
||||
|
||||
def test_seed_is_forwarded_to_speech_request(self, mocker: MockerFixture):
|
||||
app, speech_service = _build_test_app(mocker=mocker)
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||
ws.send_json({"type": "session.config", "voice": "Vivian", "seed": 42})
|
||||
ws.send_json({"type": "input.text", "text": "Hello."})
|
||||
ws.send_json({"type": "input.done"})
|
||||
ws.receive_json()
|
||||
ws.receive_bytes()
|
||||
ws.receive_json()
|
||||
ws.receive_json()
|
||||
|
||||
assert speech_service._generate_audio_bytes.await_args_list[0].args[0].seed == 42
|
||||
|
||||
|
||||
class TestGeneratePcmChunksContract:
|
||||
"""Guard: _generate_pcm_chunks must exist on OmniOpenAIServingSpeech.
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.openai import speech_text_splitter
|
||||
from vllm_omni.entrypoints.openai.speech_text_splitter import SpeechTextSplitter, extract_complete_units
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class TestExtractCompleteUnits:
|
||||
def test_ascii_sentence_needs_whitespace_or_flush(self):
|
||||
units, remainder, _ = extract_complete_units("Hello world.", frozenset(".!?。"), flush=False)
|
||||
assert units == []
|
||||
assert remainder == "Hello world."
|
||||
|
||||
units, remainder, _ = extract_complete_units("Hello world.", frozenset(".!?。"), flush=True)
|
||||
assert units == ["Hello world."]
|
||||
assert remainder == ""
|
||||
|
||||
def test_ascii_sentence_with_trailing_space(self):
|
||||
units, remainder, _ = extract_complete_units("Hello world. More", frozenset(".!?"), flush=False)
|
||||
assert units == ["Hello world."]
|
||||
assert remainder == "More"
|
||||
|
||||
def test_decimal_is_not_a_sentence_boundary(self):
|
||||
units, remainder, _ = extract_complete_units("It costs 3.14 dollars. ", frozenset(".!?"), flush=False)
|
||||
assert units == ["It costs 3.14 dollars."]
|
||||
assert remainder == ""
|
||||
|
||||
def test_thousands_separator_is_not_a_clause_boundary(self):
|
||||
units, _, _ = extract_complete_units("It is 1,000 rupees, sir. ", frozenset(".!?,;"), flush=False)
|
||||
assert units == ["It is 1,000 rupees,", "sir."]
|
||||
|
||||
def test_punctuation_run_stays_one_unit(self):
|
||||
units, _, _ = extract_complete_units("Wait... Really?! Yes.", frozenset(".!?"), flush=True)
|
||||
assert units == ["Wait...", "Really?!", "Yes."]
|
||||
|
||||
def test_closing_quote_stays_with_its_sentence(self):
|
||||
units, _, _ = extract_complete_units('He said "Hello." Bye.', frozenset(".!?"), flush=True)
|
||||
assert units == ['He said "Hello."', "Bye."]
|
||||
|
||||
def test_abbreviations_and_initials_do_not_split(self):
|
||||
units, _, _ = extract_complete_units("Dr. Smith, e.g. J. R. Tolkien. Next.", frozenset(".!?"), flush=True)
|
||||
assert units == ["Dr. Smith, e.g. J. R. Tolkien.", "Next."]
|
||||
|
||||
def test_long_token_before_period_is_not_an_abbreviation(self):
|
||||
units, _, _ = extract_complete_units("We landed in Washington. Next.", frozenset(".!?"), flush=True)
|
||||
assert units == ["We landed in Washington.", "Next."]
|
||||
|
||||
def test_abbreviation_lookback_is_bounded(self, monkeypatch):
|
||||
# One WebSocket frame can reach ~128 KiB; an unbounded lookback walked
|
||||
# the whole prefix at every `.`, quadratic in the frame size.
|
||||
checks = 0
|
||||
is_token_char = speech_text_splitter._is_token_char
|
||||
|
||||
def counting_is_token_char(ch):
|
||||
nonlocal checks
|
||||
checks += 1
|
||||
return is_token_char(ch)
|
||||
|
||||
monkeypatch.setattr(speech_text_splitter, "_is_token_char", counting_is_token_char)
|
||||
text = "a." * 20000
|
||||
units, _, _ = extract_complete_units(text, frozenset(".!?"), flush=True)
|
||||
assert units == [text]
|
||||
assert checks <= len(text) * (speech_text_splitter._MAX_ABBREVIATION_LEN + 1)
|
||||
|
||||
def test_cjk_period_completes_once_the_next_char_is_known(self):
|
||||
# The trailing 。 waits: a closing delimiter could still follow it.
|
||||
units, remainder, _ = extract_complete_units("你好。世界。", frozenset(".!?。!?"), flush=False)
|
||||
assert units == ["你好。"]
|
||||
assert remainder == "世界。"
|
||||
|
||||
units, remainder, _ = extract_complete_units("你好。世界。", frozenset(".!?。!?"), flush=True)
|
||||
assert units == ["你好。", "世界。"]
|
||||
assert remainder == ""
|
||||
|
||||
def test_indic_danda_and_question_mark(self):
|
||||
units, remainder, _ = extract_complete_units("नमस्ते। कैसे हो?", frozenset(".!?।॥؟"), flush=True)
|
||||
assert units == ["नमस्ते।", "कैसे हो?"]
|
||||
assert remainder == ""
|
||||
|
||||
def test_scan_cursor_skips_already_examined_text(self):
|
||||
units, remainder, scan = extract_complete_units("no terminator yet", frozenset(".!?"), flush=False)
|
||||
assert units == []
|
||||
assert remainder == "no terminator yet"
|
||||
assert scan == len(remainder)
|
||||
|
||||
units, remainder, scan = extract_complete_units(
|
||||
"no terminator yet. Next", frozenset(".!?"), flush=False, scan_from=scan
|
||||
)
|
||||
assert units == ["no terminator yet."]
|
||||
assert remainder == "Next"
|
||||
|
||||
|
||||
class TestSpeechTextSplitter:
|
||||
def test_none_buffers_until_flush(self):
|
||||
splitter = SpeechTextSplitter("none")
|
||||
assert splitter.feed("Hello world. ") == []
|
||||
assert splitter.flush() == ["Hello world."]
|
||||
assert splitter.flush() == []
|
||||
|
||||
def test_sentence_mode_incremental_words(self):
|
||||
splitter = SpeechTextSplitter("sentence")
|
||||
assert splitter.feed("Hello ") == []
|
||||
assert splitter.feed("world. ") == ["Hello world."]
|
||||
assert splitter.feed("How are you?") == []
|
||||
assert splitter.flush() == ["How are you?"]
|
||||
|
||||
def test_sentence_mode_resumes_scan_across_feeds(self):
|
||||
splitter = SpeechTextSplitter("sentence")
|
||||
assert splitter.feed("It costs 3") == []
|
||||
assert splitter.feed(".") == []
|
||||
assert splitter.feed("14 dollars. ") == ["It costs 3.14 dollars."]
|
||||
|
||||
def test_clause_mode_splits_on_comma(self):
|
||||
splitter = SpeechTextSplitter("clause")
|
||||
assert splitter.feed("Hello, world. ") == ["Hello,", "world."]
|
||||
|
||||
def test_clause_mode_splits_on_arabic_semicolon(self):
|
||||
splitter = SpeechTextSplitter("clause")
|
||||
assert splitter.feed("مرحبا؛ كيف حالك") == ["مرحبا؛"]
|
||||
|
||||
def test_empty_flush(self):
|
||||
splitter = SpeechTextSplitter("sentence")
|
||||
assert splitter.flush() == []
|
||||
Reference in new issue
Block a user