[Feat] Streaming TTS word-level timestamps via forced aligner (#4034)
Signed-off-by: wjinxu <1299461899@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
17 files changed
+2162
-20
No files matched your search
@@ -9,15 +9,26 @@ Protocol:
|
||||
{"type": "input.text", "text": "..."} # Text chunks
|
||||
{"type": "input.done"} # End of input
|
||||
|
||||
Server -> Client:
|
||||
Server -> Client (default, word_timestamps=false):
|
||||
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "wav"}
|
||||
<binary frame: audio bytes>
|
||||
...
|
||||
{"type": "audio.done", "sentence_index": 0}
|
||||
{"type": "session.done", "total_sentences": N}
|
||||
{"type": "error", "message": "..."}
|
||||
|
||||
Server -> Client (when word_timestamps=true):
|
||||
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm"}
|
||||
{"type": "audio.chunk", "sentence_index": 0, "chunk_id": 0, "audio_b64": "<base64 PCM>", "timestamps": null}
|
||||
...
|
||||
{"type": "audio.chunk", "audio_b64": "", "timestamps": [{"word", "start_ms", "end_ms"}, ...]}
|
||||
{"type": "audio.done", "sentence_index": 0}
|
||||
# Audio is JSON base64 PCM (not binary). A trailing empty-audio chunk carries the
|
||||
# full sentence-relative alignment. timestamps: list = aligned, [] = silence, null = failed.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from contextlib import aclosing
|
||||
|
||||
@@ -35,12 +46,15 @@ from vllm_omni.entrypoints.openai.text_splitter import (
|
||||
SPLIT_SENTENCE,
|
||||
SentenceSplitter,
|
||||
)
|
||||
from vllm_omni.utils.forced_aligner import ForcedAlignerLoadError
|
||||
from vllm_omni.utils.forced_aligner import align as forced_align
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DEFAULT_IDLE_TIMEOUT = 30.0 # seconds
|
||||
_DEFAULT_CONFIG_TIMEOUT = 10.0 # seconds
|
||||
_PCM_SAMPLE_RATE = 24000
|
||||
_BYTES_PER_SAMPLE = 2 # 16-bit mono PCM
|
||||
_MAX_CONFIG_MESSAGE_SIZE = 4 * 1024 * 1024 # allow large ref_audio payloads
|
||||
_MAX_INPUT_TEXT_MESSAGE_SIZE = 128 * 1024
|
||||
|
||||
@@ -210,6 +224,24 @@ class OmniStreamingSpeechHandler:
|
||||
"""Generate audio for a single sentence and send it over WebSocket."""
|
||||
response_format = config.response_format or "wav"
|
||||
|
||||
# Reject unmet word-timestamps preconditions early with a clear reason.
|
||||
if config.word_timestamps:
|
||||
if self._speech_service.forced_aligner_config is None:
|
||||
await self._send_error(
|
||||
websocket,
|
||||
"word_timestamps=true but the server was launched without "
|
||||
"--forced-aligner; either restart the server with that flag "
|
||||
"or set word_timestamps=false in session.config.",
|
||||
)
|
||||
return
|
||||
if not (config.stream_audio and response_format == "pcm"):
|
||||
await self._send_error(
|
||||
websocket,
|
||||
"word_timestamps=true requires stream_audio=true and "
|
||||
"response_format='pcm' (the aligner consumes raw PCM).",
|
||||
)
|
||||
return
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input=sentence_text,
|
||||
model=config.model,
|
||||
@@ -227,6 +259,7 @@ class OmniStreamingSpeechHandler:
|
||||
x_vector_only_mode=config.x_vector_only_mode,
|
||||
speaker_embedding=config.speaker_embedding,
|
||||
stream=config.stream_audio,
|
||||
word_timestamps=config.word_timestamps,
|
||||
)
|
||||
|
||||
start_payload = {
|
||||
@@ -236,7 +269,11 @@ class OmniStreamingSpeechHandler:
|
||||
"format": response_format,
|
||||
}
|
||||
if config.stream_audio and response_format == "pcm":
|
||||
# Nominal stream rate; each audio.chunk carries the authoritative
|
||||
# per-chunk sample_rate.
|
||||
start_payload["sample_rate"] = _PCM_SAMPLE_RATE
|
||||
if config.word_timestamps:
|
||||
start_payload["word_timestamps"] = True
|
||||
await websocket.send_json(start_payload)
|
||||
|
||||
total_bytes = 0
|
||||
@@ -245,10 +282,20 @@ class OmniStreamingSpeechHandler:
|
||||
try:
|
||||
if config.stream_audio:
|
||||
request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)
|
||||
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
|
||||
async for chunk in stream:
|
||||
total_bytes += len(chunk)
|
||||
await websocket.send_bytes(chunk)
|
||||
if config.word_timestamps:
|
||||
total_bytes = await self._stream_audio_with_alignments(
|
||||
websocket=websocket,
|
||||
request_id=request_id,
|
||||
generator=generator,
|
||||
sentence_text=sentence_text,
|
||||
sentence_index=sentence_index,
|
||||
language=config.language,
|
||||
)
|
||||
else:
|
||||
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
|
||||
async for chunk in stream:
|
||||
total_bytes += len(chunk)
|
||||
await websocket.send_bytes(chunk)
|
||||
else:
|
||||
audio_bytes, _ = await self._speech_service._generate_audio_bytes(request)
|
||||
total_bytes = len(audio_bytes)
|
||||
@@ -277,6 +324,115 @@ class OmniStreamingSpeechHandler:
|
||||
except Exception:
|
||||
logger.debug("Failed to send audio.done for sentence %d", sentence_index, exc_info=True)
|
||||
|
||||
async def _stream_audio_with_alignments(
|
||||
self,
|
||||
*,
|
||||
websocket: WebSocket,
|
||||
request_id: str,
|
||||
generator,
|
||||
sentence_text: str,
|
||||
sentence_index: int,
|
||||
language: str | None = None,
|
||||
) -> int:
|
||||
"""Stream PCM as JSON ``audio.chunk`` frames, aligned per sentence.
|
||||
|
||||
Forward each PCM chunk live (``timestamps: null``) while buffering the
|
||||
sentence audio, then run the forced aligner once over the whole
|
||||
sentence and emit a final empty-audio ``audio.chunk`` with the word
|
||||
timestamps. On aligner failure timestamps is ``null``; for silence it
|
||||
is ``[]`` (audio always flows regardless).
|
||||
"""
|
||||
aligner_config = self._speech_service.forced_aligner_config
|
||||
assert aligner_config is not None # gated by the precondition check
|
||||
|
||||
sentence_audio = bytearray()
|
||||
total_bytes = 0
|
||||
sample_rate = _PCM_SAMPLE_RATE
|
||||
chunk_id = 0
|
||||
|
||||
async def send_chunk(
|
||||
chunk: bytes,
|
||||
chunk_sample_rate: int,
|
||||
timestamps_payload: list[dict] | None,
|
||||
chunk_start_ms: int,
|
||||
chunk_end_ms: int,
|
||||
) -> None:
|
||||
nonlocal chunk_id
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "audio.chunk",
|
||||
"sentence_index": sentence_index,
|
||||
"chunk_id": chunk_id,
|
||||
"chunk_start_ms": chunk_start_ms,
|
||||
"chunk_end_ms": chunk_end_ms,
|
||||
"sample_rate": chunk_sample_rate,
|
||||
"audio_b64": base64.b64encode(chunk).decode("ascii"),
|
||||
"timestamps": timestamps_payload,
|
||||
}
|
||||
)
|
||||
chunk_id += 1
|
||||
|
||||
async with aclosing(
|
||||
self._speech_service._generate_pcm_chunks(
|
||||
generator,
|
||||
request_id,
|
||||
include_sample_rate=True,
|
||||
)
|
||||
) as stream:
|
||||
async for chunk, chunk_sample_rate in stream:
|
||||
sample_rate = chunk_sample_rate
|
||||
chunk_start_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||
sentence_audio.extend(chunk)
|
||||
chunk_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||
total_bytes += len(chunk)
|
||||
# Audio first, timestamps after the whole sentence is aligned.
|
||||
await send_chunk(chunk, chunk_sample_rate, None, chunk_start_ms, chunk_end_ms)
|
||||
|
||||
# Single alignment pass over the full sentence, then emit timestamps.
|
||||
# A load/config failure is permanent, so surface the reason once; audio
|
||||
# has already streamed, so the trailing frame still carries null.
|
||||
try:
|
||||
timestamps_payload = await self._align_sentence(
|
||||
audio=bytes(sentence_audio),
|
||||
text=sentence_text,
|
||||
sample_rate=sample_rate,
|
||||
config=aligner_config,
|
||||
language=language,
|
||||
)
|
||||
except ForcedAlignerLoadError as exc:
|
||||
await self._send_error(websocket, f"forced aligner unavailable: {exc}")
|
||||
timestamps_payload = None
|
||||
sentence_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||
await send_chunk(b"", sample_rate, timestamps_payload, 0, sentence_end_ms)
|
||||
|
||||
return total_bytes
|
||||
|
||||
@staticmethod
|
||||
async def _align_sentence(
|
||||
*,
|
||||
audio: bytes,
|
||||
text: str,
|
||||
sample_rate: int,
|
||||
config,
|
||||
language: str | None = None,
|
||||
) -> list[dict] | None:
|
||||
"""Convert a sentence alignment into JSON word-timestamp dicts.
|
||||
|
||||
Returns ``None`` on aligner failure, ``[]`` when it ran but produced no
|
||||
tokens. Monotonic, non-overlapping bounds are guaranteed by the decoder.
|
||||
``language`` is forwarded to word segmentation.
|
||||
"""
|
||||
aligned = await forced_align(
|
||||
audio=audio,
|
||||
text=text,
|
||||
sample_rate=sample_rate,
|
||||
config=config,
|
||||
language=language,
|
||||
)
|
||||
if aligned is None:
|
||||
return None
|
||||
return [{"word": ts.word, "start_ms": ts.start_ms, "end_ms": ts.end_ms} for ts in aligned]
|
||||
|
||||
@staticmethod
|
||||
async def _send_error(websocket: WebSocket, message: str) -> None:
|
||||
"""Send an error message to the client."""
|
||||
|
||||
Reference in new issue
Block a user