[Feat] Streaming TTS word-level timestamps via forced aligner (#4034)

Signed-off-by: wjinxu <1299461899@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-06-22 04:51:45 +00:00
1 parent a60242092a
commit ad07f98b73
17 files changed
+2162 -20

No files matched your search

+23
View File
@@ -216,6 +216,29 @@ class OmniServeCommand(CLISubcommand):
help="Default task type for TTS models (CustomVoice, VoiceDesign, or Base). "
"If not specified, will be inferred from model path.",
)
# Forced aligner / word timestamps. --forced-aligner is the opt-in
# toggle; heavier knobs (gpu_memory_utilization, dtype, max_model_len)
# live in the deploy YAML passed via --forced-aligner-config.
omni_config_group.add_argument(
"--forced-aligner",
type=str,
default=None,
help=(
"Enable streaming TTS word timestamps via a forced aligner. "
"Pass the aligner model path/name, e.g. 'Qwen/Qwen3-ForcedAligner-0.6B'. "
"Disabled when omitted."
),
)
omni_config_group.add_argument(
"--forced-aligner-config",
type=str,
default=None,
help=(
"Optional YAML file for forced aligner settings (model, runner, "
"gpu_memory_utilization, dtype, max_model_len). The --forced-aligner "
"flag, when set, overrides the YAML model field."
),
)
# TODO(@lishunyang12): deprecate once all models migrate to --deploy-config
omni_config_group.add_argument(
"--stage-configs-path",
+6 -1
View File
@@ -144,6 +144,7 @@ from vllm_omni.entrypoints.openai.video_api_utils import decode_audio_url, decod
from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
from vllm_omni.utils.forced_aligner import build_forced_aligner_config
from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace
logger = init_logger(__name__)
@@ -1058,7 +1059,11 @@ async def omni_init_app_state(
)
state.openai_serving_speech = OmniOpenAIServingSpeech(
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
engine_client,
state.openai_serving_models,
request_logger=request_logger,
model_name=model_name,
forced_aligner_config=build_forced_aligner_config(args),
)
# Warm up speech pipeline (CUDA Graph capture, torch.compile) so the first
@@ -145,6 +145,15 @@ class OpenAICreateSpeechRequest(BaseModel):
default=None,
description=("Optional model-specific parameters passed directly to the model's extra_args."),
)
word_timestamps: bool = Field(
default=False,
description=(
"When true, the server runs a shared forced aligner alongside the streamed "
"audio and emits per-chunk word timestamps. Requires the server to be "
"launched with --forced-aligner pointing at an aligner model. No effect "
"when streaming is off."
),
)
@field_validator("stream_format")
@classmethod
@@ -463,6 +472,15 @@ class StreamingSpeechSessionConfig(BaseModel):
"'clause' also splits on CJK commas , and semicolons ;."
),
)
word_timestamps: bool = Field(
default=False,
description=(
"When true, audio chunks are wrapped in JSON 'audio.chunk' frames carrying "
"base64-encoded PCM plus aligned word timestamps. Requires the server to be "
"launched with --forced-aligner. When false, audio is sent as raw binary "
"frames (existing behavior)."
),
)
@model_validator(mode="after")
def validate_streaming_constraints(self) -> "StreamingSpeechSessionConfig":
+21 -3
View File
@@ -411,6 +411,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def __init__(self, *args, **kwargs):
self.model_name = kwargs.pop("model_name", None)
self.forced_aligner_config: Any | None = kwargs.pop("forced_aligner_config", None)
super().__init__(*args, **kwargs)
self._init_speaker_storage()
@@ -2586,6 +2587,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
response_format: str = "pcm",
raw_request: Request | None = None,
request_start_s: float | None = None,
include_sample_rate: bool = False,
):
"""Generate audio chunks for streaming response.
@@ -2662,7 +2664,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
)
if first_audio_chunk_s is None:
first_audio_chunk_s = time.perf_counter()
yield self.create_audio(audio_obj).audio_data
audio_bytes = self.create_audio(audio_obj).audio_data
if include_sample_rate:
yield audio_bytes, sample_rate_val
else:
yield audio_bytes
self._mark_ref_audio_artifact_ready_for_request(request_id)
artifact_ready = True
total_ms = (time.perf_counter() - stream_start_s) * 1000.0
@@ -3508,13 +3514,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
self._track_ref_audio_artifact_warmup(request_id, qwen3_ref_audio_warmup_artifact_key)
return request_id, generator, tts_params
async def _generate_pcm_chunks(self, generator, request_id: str):
async def _generate_pcm_chunks(self, generator, request_id: str, *, include_sample_rate: bool = False):
"""Yield raw PCM byte chunks from the engine generator.
Delegates to ``_generate_audio_chunks`` with ``response_format="pcm"``.
Used by the WebSocket streaming handler and ``_iter_pcm_audio_bytes``.
"""
async for chunk in self._generate_audio_chunks(generator, request_id, response_format="pcm"):
async for chunk in self._generate_audio_chunks(
generator,
request_id,
response_format="pcm",
include_sample_rate=include_sample_rate,
):
yield chunk
async def _iter_pcm_audio_bytes(self, request: OpenAICreateSpeechRequest):
@@ -3810,6 +3821,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
try:
if request.stream:
if request.word_timestamps:
return self.create_error_response(
"word_timestamps=true is currently supported by the WebSocket "
"/v1/audio/speech/stream path. Use session.config with "
"stream_audio=true and response_format='pcm'."
)
# Determine response format and media type for streaming
response_format = (request.response_format or "wav").lower()
@@ -9,15 +9,26 @@ Protocol:
{"type": "input.text", "text": "..."} # Text chunks
{"type": "input.done"} # End of input
Server -> Client:
Server -> Client (default, word_timestamps=false):
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "wav"}
<binary frame: audio bytes>
...
{"type": "audio.done", "sentence_index": 0}
{"type": "session.done", "total_sentences": N}
{"type": "error", "message": "..."}
Server -> Client (when word_timestamps=true):
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm"}
{"type": "audio.chunk", "sentence_index": 0, "chunk_id": 0, "audio_b64": "<base64 PCM>", "timestamps": null}
...
{"type": "audio.chunk", "audio_b64": "", "timestamps": [{"word", "start_ms", "end_ms"}, ...]}
{"type": "audio.done", "sentence_index": 0}
# Audio is JSON base64 PCM (not binary). A trailing empty-audio chunk carries the
# full sentence-relative alignment. timestamps: list = aligned, [] = silence, null = failed.
"""
import asyncio
import base64
import json
from contextlib import aclosing
@@ -35,12 +46,15 @@ from vllm_omni.entrypoints.openai.text_splitter import (
SPLIT_SENTENCE,
SentenceSplitter,
)
from vllm_omni.utils.forced_aligner import ForcedAlignerLoadError
from vllm_omni.utils.forced_aligner import align as forced_align
logger = init_logger(__name__)
_DEFAULT_IDLE_TIMEOUT = 30.0 # seconds
_DEFAULT_CONFIG_TIMEOUT = 10.0 # seconds
_PCM_SAMPLE_RATE = 24000
_BYTES_PER_SAMPLE = 2 # 16-bit mono PCM
_MAX_CONFIG_MESSAGE_SIZE = 4 * 1024 * 1024 # allow large ref_audio payloads
_MAX_INPUT_TEXT_MESSAGE_SIZE = 128 * 1024
@@ -210,6 +224,24 @@ class OmniStreamingSpeechHandler:
"""Generate audio for a single sentence and send it over WebSocket."""
response_format = config.response_format or "wav"
# Reject unmet word-timestamps preconditions early with a clear reason.
if config.word_timestamps:
if self._speech_service.forced_aligner_config is None:
await self._send_error(
websocket,
"word_timestamps=true but the server was launched without "
"--forced-aligner; either restart the server with that flag "
"or set word_timestamps=false in session.config.",
)
return
if not (config.stream_audio and response_format == "pcm"):
await self._send_error(
websocket,
"word_timestamps=true requires stream_audio=true and "
"response_format='pcm' (the aligner consumes raw PCM).",
)
return
request = OpenAICreateSpeechRequest(
input=sentence_text,
model=config.model,
@@ -227,6 +259,7 @@ class OmniStreamingSpeechHandler:
x_vector_only_mode=config.x_vector_only_mode,
speaker_embedding=config.speaker_embedding,
stream=config.stream_audio,
word_timestamps=config.word_timestamps,
)
start_payload = {
@@ -236,7 +269,11 @@ class OmniStreamingSpeechHandler:
"format": response_format,
}
if config.stream_audio and response_format == "pcm":
# Nominal stream rate; each audio.chunk carries the authoritative
# per-chunk sample_rate.
start_payload["sample_rate"] = _PCM_SAMPLE_RATE
if config.word_timestamps:
start_payload["word_timestamps"] = True
await websocket.send_json(start_payload)
total_bytes = 0
@@ -245,10 +282,20 @@ class OmniStreamingSpeechHandler:
try:
if config.stream_audio:
request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
async for chunk in stream:
total_bytes += len(chunk)
await websocket.send_bytes(chunk)
if config.word_timestamps:
total_bytes = await self._stream_audio_with_alignments(
websocket=websocket,
request_id=request_id,
generator=generator,
sentence_text=sentence_text,
sentence_index=sentence_index,
language=config.language,
)
else:
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
async for chunk in stream:
total_bytes += len(chunk)
await websocket.send_bytes(chunk)
else:
audio_bytes, _ = await self._speech_service._generate_audio_bytes(request)
total_bytes = len(audio_bytes)
@@ -277,6 +324,115 @@ class OmniStreamingSpeechHandler:
except Exception:
logger.debug("Failed to send audio.done for sentence %d", sentence_index, exc_info=True)
async def _stream_audio_with_alignments(
self,
*,
websocket: WebSocket,
request_id: str,
generator,
sentence_text: str,
sentence_index: int,
language: str | None = None,
) -> int:
"""Stream PCM as JSON ``audio.chunk`` frames, aligned per sentence.
Forward each PCM chunk live (``timestamps: null``) while buffering the
sentence audio, then run the forced aligner once over the whole
sentence and emit a final empty-audio ``audio.chunk`` with the word
timestamps. On aligner failure timestamps is ``null``; for silence it
is ``[]`` (audio always flows regardless).
"""
aligner_config = self._speech_service.forced_aligner_config
assert aligner_config is not None # gated by the precondition check
sentence_audio = bytearray()
total_bytes = 0
sample_rate = _PCM_SAMPLE_RATE
chunk_id = 0
async def send_chunk(
chunk: bytes,
chunk_sample_rate: int,
timestamps_payload: list[dict] | None,
chunk_start_ms: int,
chunk_end_ms: int,
) -> None:
nonlocal chunk_id
await websocket.send_json(
{
"type": "audio.chunk",
"sentence_index": sentence_index,
"chunk_id": chunk_id,
"chunk_start_ms": chunk_start_ms,
"chunk_end_ms": chunk_end_ms,
"sample_rate": chunk_sample_rate,
"audio_b64": base64.b64encode(chunk).decode("ascii"),
"timestamps": timestamps_payload,
}
)
chunk_id += 1
async with aclosing(
self._speech_service._generate_pcm_chunks(
generator,
request_id,
include_sample_rate=True,
)
) as stream:
async for chunk, chunk_sample_rate in stream:
sample_rate = chunk_sample_rate
chunk_start_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
sentence_audio.extend(chunk)
chunk_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
total_bytes += len(chunk)
# Audio first, timestamps after the whole sentence is aligned.
await send_chunk(chunk, chunk_sample_rate, None, chunk_start_ms, chunk_end_ms)
# Single alignment pass over the full sentence, then emit timestamps.
# A load/config failure is permanent, so surface the reason once; audio
# has already streamed, so the trailing frame still carries null.
try:
timestamps_payload = await self._align_sentence(
audio=bytes(sentence_audio),
text=sentence_text,
sample_rate=sample_rate,
config=aligner_config,
language=language,
)
except ForcedAlignerLoadError as exc:
await self._send_error(websocket, f"forced aligner unavailable: {exc}")
timestamps_payload = None
sentence_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
await send_chunk(b"", sample_rate, timestamps_payload, 0, sentence_end_ms)
return total_bytes
@staticmethod
async def _align_sentence(
*,
audio: bytes,
text: str,
sample_rate: int,
config,
language: str | None = None,
) -> list[dict] | None:
"""Convert a sentence alignment into JSON word-timestamp dicts.
Returns ``None`` on aligner failure, ``[]`` when it ran but produced no
tokens. Monotonic, non-overlapping bounds are guaranteed by the decoder.
``language`` is forwarded to word segmentation.
"""
aligned = await forced_align(
audio=audio,
text=text,
sample_rate=sample_rate,
config=config,
language=language,
)
if aligned is None:
return None
return [{"word": ts.word, "start_ms": ts.start_ms, "end_ms": ts.end_ms} for ts in aligned]
@staticmethod
async def _send_error(websocket: WebSocket, message: str) -> None:
"""Send an error message to the client."""