[Feat] Streaming TTS word-level timestamps via forced aligner (#4034)
Signed-off-by: wjinxu <1299461899@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
17 files changed
+2162
-20
No files matched your search
@@ -216,6 +216,29 @@ class OmniServeCommand(CLISubcommand):
|
||||
help="Default task type for TTS models (CustomVoice, VoiceDesign, or Base). "
|
||||
"If not specified, will be inferred from model path.",
|
||||
)
|
||||
# Forced aligner / word timestamps. --forced-aligner is the opt-in
|
||||
# toggle; heavier knobs (gpu_memory_utilization, dtype, max_model_len)
|
||||
# live in the deploy YAML passed via --forced-aligner-config.
|
||||
omni_config_group.add_argument(
|
||||
"--forced-aligner",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Enable streaming TTS word timestamps via a forced aligner. "
|
||||
"Pass the aligner model path/name, e.g. 'Qwen/Qwen3-ForcedAligner-0.6B'. "
|
||||
"Disabled when omitted."
|
||||
),
|
||||
)
|
||||
omni_config_group.add_argument(
|
||||
"--forced-aligner-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Optional YAML file for forced aligner settings (model, runner, "
|
||||
"gpu_memory_utilization, dtype, max_model_len). The --forced-aligner "
|
||||
"flag, when set, overrides the YAML model field."
|
||||
),
|
||||
)
|
||||
# TODO(@lishunyang12): deprecate once all models migrate to --deploy-config
|
||||
omni_config_group.add_argument(
|
||||
"--stage-configs-path",
|
||||
|
||||
@@ -144,6 +144,7 @@ from vllm_omni.entrypoints.openai.video_api_utils import decode_audio_url, decod
|
||||
from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
||||
from vllm_omni.utils.forced_aligner import build_forced_aligner_config
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -1058,7 +1059,11 @@ async def omni_init_app_state(
|
||||
)
|
||||
|
||||
state.openai_serving_speech = OmniOpenAIServingSpeech(
|
||||
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
model_name=model_name,
|
||||
forced_aligner_config=build_forced_aligner_config(args),
|
||||
)
|
||||
|
||||
# Warm up speech pipeline (CUDA Graph capture, torch.compile) so the first
|
||||
|
||||
@@ -145,6 +145,15 @@ class OpenAICreateSpeechRequest(BaseModel):
|
||||
default=None,
|
||||
description=("Optional model-specific parameters passed directly to the model's extra_args."),
|
||||
)
|
||||
word_timestamps: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When true, the server runs a shared forced aligner alongside the streamed "
|
||||
"audio and emits per-chunk word timestamps. Requires the server to be "
|
||||
"launched with --forced-aligner pointing at an aligner model. No effect "
|
||||
"when streaming is off."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("stream_format")
|
||||
@classmethod
|
||||
@@ -463,6 +472,15 @@ class StreamingSpeechSessionConfig(BaseModel):
|
||||
"'clause' also splits on CJK commas , and semicolons ;."
|
||||
),
|
||||
)
|
||||
word_timestamps: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When true, audio chunks are wrapped in JSON 'audio.chunk' frames carrying "
|
||||
"base64-encoded PCM plus aligned word timestamps. Requires the server to be "
|
||||
"launched with --forced-aligner. When false, audio is sent as raw binary "
|
||||
"frames (existing behavior)."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_streaming_constraints(self) -> "StreamingSpeechSessionConfig":
|
||||
|
||||
@@ -411,6 +411,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.model_name = kwargs.pop("model_name", None)
|
||||
self.forced_aligner_config: Any | None = kwargs.pop("forced_aligner_config", None)
|
||||
super().__init__(*args, **kwargs)
|
||||
self._init_speaker_storage()
|
||||
|
||||
@@ -2586,6 +2587,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
response_format: str = "pcm",
|
||||
raw_request: Request | None = None,
|
||||
request_start_s: float | None = None,
|
||||
include_sample_rate: bool = False,
|
||||
):
|
||||
"""Generate audio chunks for streaming response.
|
||||
|
||||
@@ -2662,7 +2664,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
)
|
||||
if first_audio_chunk_s is None:
|
||||
first_audio_chunk_s = time.perf_counter()
|
||||
yield self.create_audio(audio_obj).audio_data
|
||||
audio_bytes = self.create_audio(audio_obj).audio_data
|
||||
if include_sample_rate:
|
||||
yield audio_bytes, sample_rate_val
|
||||
else:
|
||||
yield audio_bytes
|
||||
self._mark_ref_audio_artifact_ready_for_request(request_id)
|
||||
artifact_ready = True
|
||||
total_ms = (time.perf_counter() - stream_start_s) * 1000.0
|
||||
@@ -3508,13 +3514,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
self._track_ref_audio_artifact_warmup(request_id, qwen3_ref_audio_warmup_artifact_key)
|
||||
return request_id, generator, tts_params
|
||||
|
||||
async def _generate_pcm_chunks(self, generator, request_id: str):
|
||||
async def _generate_pcm_chunks(self, generator, request_id: str, *, include_sample_rate: bool = False):
|
||||
"""Yield raw PCM byte chunks from the engine generator.
|
||||
|
||||
Delegates to ``_generate_audio_chunks`` with ``response_format="pcm"``.
|
||||
Used by the WebSocket streaming handler and ``_iter_pcm_audio_bytes``.
|
||||
"""
|
||||
async for chunk in self._generate_audio_chunks(generator, request_id, response_format="pcm"):
|
||||
async for chunk in self._generate_audio_chunks(
|
||||
generator,
|
||||
request_id,
|
||||
response_format="pcm",
|
||||
include_sample_rate=include_sample_rate,
|
||||
):
|
||||
yield chunk
|
||||
|
||||
async def _iter_pcm_audio_bytes(self, request: OpenAICreateSpeechRequest):
|
||||
@@ -3810,6 +3821,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
try:
|
||||
if request.stream:
|
||||
if request.word_timestamps:
|
||||
return self.create_error_response(
|
||||
"word_timestamps=true is currently supported by the WebSocket "
|
||||
"/v1/audio/speech/stream path. Use session.config with "
|
||||
"stream_audio=true and response_format='pcm'."
|
||||
)
|
||||
|
||||
# Determine response format and media type for streaming
|
||||
response_format = (request.response_format or "wav").lower()
|
||||
|
||||
|
||||
@@ -9,15 +9,26 @@ Protocol:
|
||||
{"type": "input.text", "text": "..."} # Text chunks
|
||||
{"type": "input.done"} # End of input
|
||||
|
||||
Server -> Client:
|
||||
Server -> Client (default, word_timestamps=false):
|
||||
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "wav"}
|
||||
<binary frame: audio bytes>
|
||||
...
|
||||
{"type": "audio.done", "sentence_index": 0}
|
||||
{"type": "session.done", "total_sentences": N}
|
||||
{"type": "error", "message": "..."}
|
||||
|
||||
Server -> Client (when word_timestamps=true):
|
||||
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm"}
|
||||
{"type": "audio.chunk", "sentence_index": 0, "chunk_id": 0, "audio_b64": "<base64 PCM>", "timestamps": null}
|
||||
...
|
||||
{"type": "audio.chunk", "audio_b64": "", "timestamps": [{"word", "start_ms", "end_ms"}, ...]}
|
||||
{"type": "audio.done", "sentence_index": 0}
|
||||
# Audio is JSON base64 PCM (not binary). A trailing empty-audio chunk carries the
|
||||
# full sentence-relative alignment. timestamps: list = aligned, [] = silence, null = failed.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from contextlib import aclosing
|
||||
|
||||
@@ -35,12 +46,15 @@ from vllm_omni.entrypoints.openai.text_splitter import (
|
||||
SPLIT_SENTENCE,
|
||||
SentenceSplitter,
|
||||
)
|
||||
from vllm_omni.utils.forced_aligner import ForcedAlignerLoadError
|
||||
from vllm_omni.utils.forced_aligner import align as forced_align
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DEFAULT_IDLE_TIMEOUT = 30.0 # seconds
|
||||
_DEFAULT_CONFIG_TIMEOUT = 10.0 # seconds
|
||||
_PCM_SAMPLE_RATE = 24000
|
||||
_BYTES_PER_SAMPLE = 2 # 16-bit mono PCM
|
||||
_MAX_CONFIG_MESSAGE_SIZE = 4 * 1024 * 1024 # allow large ref_audio payloads
|
||||
_MAX_INPUT_TEXT_MESSAGE_SIZE = 128 * 1024
|
||||
|
||||
@@ -210,6 +224,24 @@ class OmniStreamingSpeechHandler:
|
||||
"""Generate audio for a single sentence and send it over WebSocket."""
|
||||
response_format = config.response_format or "wav"
|
||||
|
||||
# Reject unmet word-timestamps preconditions early with a clear reason.
|
||||
if config.word_timestamps:
|
||||
if self._speech_service.forced_aligner_config is None:
|
||||
await self._send_error(
|
||||
websocket,
|
||||
"word_timestamps=true but the server was launched without "
|
||||
"--forced-aligner; either restart the server with that flag "
|
||||
"or set word_timestamps=false in session.config.",
|
||||
)
|
||||
return
|
||||
if not (config.stream_audio and response_format == "pcm"):
|
||||
await self._send_error(
|
||||
websocket,
|
||||
"word_timestamps=true requires stream_audio=true and "
|
||||
"response_format='pcm' (the aligner consumes raw PCM).",
|
||||
)
|
||||
return
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input=sentence_text,
|
||||
model=config.model,
|
||||
@@ -227,6 +259,7 @@ class OmniStreamingSpeechHandler:
|
||||
x_vector_only_mode=config.x_vector_only_mode,
|
||||
speaker_embedding=config.speaker_embedding,
|
||||
stream=config.stream_audio,
|
||||
word_timestamps=config.word_timestamps,
|
||||
)
|
||||
|
||||
start_payload = {
|
||||
@@ -236,7 +269,11 @@ class OmniStreamingSpeechHandler:
|
||||
"format": response_format,
|
||||
}
|
||||
if config.stream_audio and response_format == "pcm":
|
||||
# Nominal stream rate; each audio.chunk carries the authoritative
|
||||
# per-chunk sample_rate.
|
||||
start_payload["sample_rate"] = _PCM_SAMPLE_RATE
|
||||
if config.word_timestamps:
|
||||
start_payload["word_timestamps"] = True
|
||||
await websocket.send_json(start_payload)
|
||||
|
||||
total_bytes = 0
|
||||
@@ -245,10 +282,20 @@ class OmniStreamingSpeechHandler:
|
||||
try:
|
||||
if config.stream_audio:
|
||||
request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)
|
||||
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
|
||||
async for chunk in stream:
|
||||
total_bytes += len(chunk)
|
||||
await websocket.send_bytes(chunk)
|
||||
if config.word_timestamps:
|
||||
total_bytes = await self._stream_audio_with_alignments(
|
||||
websocket=websocket,
|
||||
request_id=request_id,
|
||||
generator=generator,
|
||||
sentence_text=sentence_text,
|
||||
sentence_index=sentence_index,
|
||||
language=config.language,
|
||||
)
|
||||
else:
|
||||
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
|
||||
async for chunk in stream:
|
||||
total_bytes += len(chunk)
|
||||
await websocket.send_bytes(chunk)
|
||||
else:
|
||||
audio_bytes, _ = await self._speech_service._generate_audio_bytes(request)
|
||||
total_bytes = len(audio_bytes)
|
||||
@@ -277,6 +324,115 @@ class OmniStreamingSpeechHandler:
|
||||
except Exception:
|
||||
logger.debug("Failed to send audio.done for sentence %d", sentence_index, exc_info=True)
|
||||
|
||||
async def _stream_audio_with_alignments(
|
||||
self,
|
||||
*,
|
||||
websocket: WebSocket,
|
||||
request_id: str,
|
||||
generator,
|
||||
sentence_text: str,
|
||||
sentence_index: int,
|
||||
language: str | None = None,
|
||||
) -> int:
|
||||
"""Stream PCM as JSON ``audio.chunk`` frames, aligned per sentence.
|
||||
|
||||
Forward each PCM chunk live (``timestamps: null``) while buffering the
|
||||
sentence audio, then run the forced aligner once over the whole
|
||||
sentence and emit a final empty-audio ``audio.chunk`` with the word
|
||||
timestamps. On aligner failure timestamps is ``null``; for silence it
|
||||
is ``[]`` (audio always flows regardless).
|
||||
"""
|
||||
aligner_config = self._speech_service.forced_aligner_config
|
||||
assert aligner_config is not None # gated by the precondition check
|
||||
|
||||
sentence_audio = bytearray()
|
||||
total_bytes = 0
|
||||
sample_rate = _PCM_SAMPLE_RATE
|
||||
chunk_id = 0
|
||||
|
||||
async def send_chunk(
|
||||
chunk: bytes,
|
||||
chunk_sample_rate: int,
|
||||
timestamps_payload: list[dict] | None,
|
||||
chunk_start_ms: int,
|
||||
chunk_end_ms: int,
|
||||
) -> None:
|
||||
nonlocal chunk_id
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "audio.chunk",
|
||||
"sentence_index": sentence_index,
|
||||
"chunk_id": chunk_id,
|
||||
"chunk_start_ms": chunk_start_ms,
|
||||
"chunk_end_ms": chunk_end_ms,
|
||||
"sample_rate": chunk_sample_rate,
|
||||
"audio_b64": base64.b64encode(chunk).decode("ascii"),
|
||||
"timestamps": timestamps_payload,
|
||||
}
|
||||
)
|
||||
chunk_id += 1
|
||||
|
||||
async with aclosing(
|
||||
self._speech_service._generate_pcm_chunks(
|
||||
generator,
|
||||
request_id,
|
||||
include_sample_rate=True,
|
||||
)
|
||||
) as stream:
|
||||
async for chunk, chunk_sample_rate in stream:
|
||||
sample_rate = chunk_sample_rate
|
||||
chunk_start_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||
sentence_audio.extend(chunk)
|
||||
chunk_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||
total_bytes += len(chunk)
|
||||
# Audio first, timestamps after the whole sentence is aligned.
|
||||
await send_chunk(chunk, chunk_sample_rate, None, chunk_start_ms, chunk_end_ms)
|
||||
|
||||
# Single alignment pass over the full sentence, then emit timestamps.
|
||||
# A load/config failure is permanent, so surface the reason once; audio
|
||||
# has already streamed, so the trailing frame still carries null.
|
||||
try:
|
||||
timestamps_payload = await self._align_sentence(
|
||||
audio=bytes(sentence_audio),
|
||||
text=sentence_text,
|
||||
sample_rate=sample_rate,
|
||||
config=aligner_config,
|
||||
language=language,
|
||||
)
|
||||
except ForcedAlignerLoadError as exc:
|
||||
await self._send_error(websocket, f"forced aligner unavailable: {exc}")
|
||||
timestamps_payload = None
|
||||
sentence_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||
await send_chunk(b"", sample_rate, timestamps_payload, 0, sentence_end_ms)
|
||||
|
||||
return total_bytes
|
||||
|
||||
@staticmethod
|
||||
async def _align_sentence(
|
||||
*,
|
||||
audio: bytes,
|
||||
text: str,
|
||||
sample_rate: int,
|
||||
config,
|
||||
language: str | None = None,
|
||||
) -> list[dict] | None:
|
||||
"""Convert a sentence alignment into JSON word-timestamp dicts.
|
||||
|
||||
Returns ``None`` on aligner failure, ``[]`` when it ran but produced no
|
||||
tokens. Monotonic, non-overlapping bounds are guaranteed by the decoder.
|
||||
``language`` is forwarded to word segmentation.
|
||||
"""
|
||||
aligned = await forced_align(
|
||||
audio=audio,
|
||||
text=text,
|
||||
sample_rate=sample_rate,
|
||||
config=config,
|
||||
language=language,
|
||||
)
|
||||
if aligned is None:
|
||||
return None
|
||||
return [{"word": ts.word, "start_ms": ts.start_ms, "end_ms": ts.end_ms} for ts in aligned]
|
||||
|
||||
@staticmethod
|
||||
async def _send_error(websocket: WebSocket, message: str) -> None:
|
||||
"""Send an error message to the client."""
|
||||
|
||||
Reference in new issue
Block a user