[Feat] Streaming TTS word-level timestamps via forced aligner (#4034)
Signed-off-by: wjinxu <1299461899@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
17 files changed
+2162
-20
No files matched your search
@@ -369,6 +369,20 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
|
|||||||
```
|
```
|
||||||
Streaming requires `async_chunk: true` in the stage config.
|
Streaming requires `async_chunk: true` in the stage config.
|
||||||
|
|
||||||
|
### Word Timestamps
|
||||||
|
Generate a WAV offline and a JSON sidecar with word-level timestamps from
|
||||||
|
`Qwen/Qwen3-ForcedAligner-0.6B`:
|
||||||
|
```bash
|
||||||
|
python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
|
||||||
|
--model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||||
|
--forced-aligner Qwen/Qwen3-ForcedAligner-0.6B \
|
||||||
|
--text "Hello world." \
|
||||||
|
--output-dir /tmp/qwen3_tts_timestamps
|
||||||
|
```
|
||||||
|
The script writes `qwen3_tts_word_timestamps.wav` and
|
||||||
|
`qwen3_tts_word_timestamps.json`. On machines without a local CUDA toolkit,
|
||||||
|
set `VLLM_USE_FLASHINFER_SAMPLER=0` to avoid FlashInfer sampler JIT.
|
||||||
|
|
||||||
### Batched decoding
|
### Batched decoding
|
||||||
The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates):
|
The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates):
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Offline Qwen3-TTS example with word-level timestamps.
|
||||||
|
|
||||||
|
This demo runs Qwen3-TTS offline, aligns the synthesized audio in memory with
|
||||||
|
the same shared forced aligner used by the streaming server path, then writes
|
||||||
|
the WAV and JSON sidecar:
|
||||||
|
|
||||||
|
python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
|
||||||
|
--model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||||
|
--forced-aligner Qwen/Qwen3-ForcedAligner-0.6B
|
||||||
|
|
||||||
|
On machines without a local CUDA toolkit, set ``VLLM_USE_FLASHINFER_SAMPLER=0``
|
||||||
|
to avoid FlashInfer sampler JIT during warmup.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||||
|
# The in-process offline aligner consumes token_classify outputs. Keeping V1
|
||||||
|
# multiprocessing off avoids msgspec serialization issues for token-wise logits.
|
||||||
|
os.environ.setdefault("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import soundfile as sf
|
||||||
|
import torch
|
||||||
|
from end2end import _estimate_prompt_len
|
||||||
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
|
from vllm_omni import Omni
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
from vllm_omni.utils.forced_aligner import align, build_forced_aligner_config
|
||||||
|
|
||||||
|
|
||||||
|
def _default_stage_config() -> str:
|
||||||
|
repo_root = Path(__file__).resolve().parents[4]
|
||||||
|
return str(repo_root / "vllm_omni" / "deploy" / "qwen3_tts.yaml")
|
||||||
|
|
||||||
|
|
||||||
|
def _build_custom_voice_input(args: Any) -> dict[str, Any]:
|
||||||
|
additional_information = {
|
||||||
|
"task_type": ["CustomVoice"],
|
||||||
|
"text": [args.text],
|
||||||
|
"language": [args.language],
|
||||||
|
"speaker": [args.speaker],
|
||||||
|
"instruct": [args.instructions],
|
||||||
|
"max_new_tokens": [args.max_new_tokens],
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"prompt_token_ids": [0] * _estimate_prompt_len(additional_information, args.model),
|
||||||
|
"additional_information": additional_information,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _audio_tensor_and_sample_rate(mm: dict[str, Any]) -> tuple[torch.Tensor, int]:
|
||||||
|
audio_data = mm["audio"]
|
||||||
|
sr_raw = mm["sr"]
|
||||||
|
sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
|
||||||
|
sample_rate = int(sr_val.item()) if hasattr(sr_val, "item") else int(sr_val)
|
||||||
|
audio_tensor = torch.cat(audio_data, dim=-1) if isinstance(audio_data, list) else audio_data
|
||||||
|
return audio_tensor.float().detach().cpu().flatten(), sample_rate
|
||||||
|
|
||||||
|
|
||||||
|
def _float_audio_to_pcm16_bytes(audio: torch.Tensor) -> bytes:
|
||||||
|
samples = audio.numpy()
|
||||||
|
pcm = (np.clip(samples, -1.0, 1.0) * 32767.0).astype("<i2")
|
||||||
|
return pcm.tobytes()
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_alignment(args: Any, audio: torch.Tensor, sample_rate: int) -> list[dict[str, Any]] | None:
|
||||||
|
aligner_config = build_forced_aligner_config(args)
|
||||||
|
if aligner_config is None:
|
||||||
|
raise ValueError("--forced-aligner or --forced-aligner-config is required")
|
||||||
|
timestamps = await align(
|
||||||
|
audio=_float_audio_to_pcm16_bytes(audio),
|
||||||
|
text=args.text,
|
||||||
|
sample_rate=sample_rate,
|
||||||
|
config=aligner_config,
|
||||||
|
language=args.language,
|
||||||
|
)
|
||||||
|
if timestamps is None:
|
||||||
|
return None
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"word": item.word,
|
||||||
|
"start_ms": item.start_ms,
|
||||||
|
"end_ms": item.end_ms,
|
||||||
|
}
|
||||||
|
for item in timestamps
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def main(args: Any) -> None:
|
||||||
|
output_dir = Path(args.output_dir)
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
omni_kwargs = vars(args).copy()
|
||||||
|
for key in (
|
||||||
|
"forced_aligner",
|
||||||
|
"text",
|
||||||
|
"language",
|
||||||
|
"speaker",
|
||||||
|
"instructions",
|
||||||
|
"max_new_tokens",
|
||||||
|
"output_dir",
|
||||||
|
):
|
||||||
|
omni_kwargs.pop(key, None)
|
||||||
|
omni_kwargs["stage_configs_path"] = args.stage_configs_path or _default_stage_config()
|
||||||
|
omni_kwargs["log_stats"] = args.log_stats
|
||||||
|
omni = Omni(**omni_kwargs)
|
||||||
|
|
||||||
|
prompt = _build_custom_voice_input(args)
|
||||||
|
final_output = None
|
||||||
|
for stage_outputs in omni.generate([prompt]):
|
||||||
|
final_output = stage_outputs.request_output
|
||||||
|
if final_output is None:
|
||||||
|
raise RuntimeError("Qwen3-TTS did not produce an output.")
|
||||||
|
|
||||||
|
mm = final_output.outputs[0].multimodal_output
|
||||||
|
audio, sample_rate = _audio_tensor_and_sample_rate(mm)
|
||||||
|
|
||||||
|
wav_path = output_dir / "qwen3_tts_word_timestamps.wav"
|
||||||
|
# Align the in-memory tensor, not the saved WAV. The WAV is only a demo
|
||||||
|
# artifact so users can listen to the same audio referenced by the sidecar.
|
||||||
|
timestamps = asyncio.run(_run_alignment(args, audio, sample_rate))
|
||||||
|
sf.write(wav_path, audio.numpy(), samplerate=sample_rate, format="WAV")
|
||||||
|
|
||||||
|
sidecar = {
|
||||||
|
"text": args.text,
|
||||||
|
"sample_rate": sample_rate,
|
||||||
|
"audio_path": str(wav_path),
|
||||||
|
"timestamps": timestamps,
|
||||||
|
}
|
||||||
|
json_path = output_dir / "qwen3_tts_word_timestamps.json"
|
||||||
|
json_path.write_text(json.dumps(sidecar, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
print(f"Saved audio: {wav_path}")
|
||||||
|
print(f"Saved timestamps: {json_path}")
|
||||||
|
print(json.dumps(sidecar["timestamps"], ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> Any:
|
||||||
|
parser = FlexibleArgumentParser(description="Offline Qwen3-TTS word timestamp example")
|
||||||
|
parser.add_argument("--model", default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", help="Qwen3-TTS model path/name")
|
||||||
|
parser.add_argument(
|
||||||
|
"--forced-aligner",
|
||||||
|
default=None,
|
||||||
|
help="Qwen3 forced aligner model path/name",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--forced-aligner-config",
|
||||||
|
default=None,
|
||||||
|
help="Optional YAML file for forced aligner settings (incl. gpu_memory_utilization)",
|
||||||
|
)
|
||||||
|
parser.add_argument("--stage-configs-path", default=None, help="Qwen3-TTS deploy YAML")
|
||||||
|
parser.add_argument("--text", default="Hello world.", help="Text to synthesize and align")
|
||||||
|
parser.add_argument("--language", default="English", help="Qwen3-TTS language field")
|
||||||
|
parser.add_argument("--speaker", default="Vivian", help="CustomVoice speaker name")
|
||||||
|
parser.add_argument("--instructions", default="", help="Optional speaking style instruction")
|
||||||
|
parser.add_argument("--max-new-tokens", type=int, default=2048, help="TTS max_new_tokens")
|
||||||
|
parser.add_argument("--output-dir", default="output_audio", help="Directory for WAV and JSON sidecar")
|
||||||
|
parser.add_argument("--log-stats", action="store_true", default=False, help="Enable vLLM-Omni stats logging")
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main(parse_args())
|
||||||
@@ -504,6 +504,35 @@ python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I
|
|||||||
python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
|
python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To receive word-level timestamps, launch the server with a forced aligner:
|
||||||
|
```bash
|
||||||
|
vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||||
|
--omni \
|
||||||
|
--deploy-config vllm_omni/deploy/qwen3_tts.yaml \
|
||||||
|
--trust-remote-code \
|
||||||
|
--forced-aligner Qwen/Qwen3-ForcedAligner-0.6B
|
||||||
|
```
|
||||||
|
Then request PCM JSON sidecar chunks:
|
||||||
|
```bash
|
||||||
|
python qwen3_tts/streaming_speech_client.py \
|
||||||
|
--text "Hello world. How are you?" \
|
||||||
|
--stream-audio \
|
||||||
|
--response-format pcm \
|
||||||
|
--word-timestamps
|
||||||
|
```
|
||||||
|
The client writes one PCM file per sentence and a matching
|
||||||
|
`sentence_XXX_timestamps.json` sidecar.
|
||||||
|
|
||||||
|
To *see* the alignment instead of reading a JSON sidecar, run the
|
||||||
|
word-timestamp Gradio demo (server must be launched with `--forced-aligner`):
|
||||||
|
```bash
|
||||||
|
python qwen3_tts/word_timestamps_demo.py --api-base http://localhost:8091
|
||||||
|
```
|
||||||
|
Each sentence's audio plays in an `<audio>` element while its text is rendered
|
||||||
|
as inline word spans; the current word highlights as `audio.currentTime`
|
||||||
|
crosses each `start_ms`. The **Stop (barge-in)** button cuts playback and
|
||||||
|
reports the last-spoken word, useful for the voice-agent barge-in case.
|
||||||
|
|
||||||
### Gradio demos
|
### Gradio demos
|
||||||
```bash
|
```bash
|
||||||
./qwen3_tts/run_gradio_demo.sh # CustomVoice (default)
|
./qwen3_tts/run_gradio_demo.sh # CustomVoice (default)
|
||||||
|
|||||||
@@ -18,6 +18,21 @@ Usage:
|
|||||||
--text "Hello world. How are you? I am fine." \
|
--text "Hello world. How are you? I am fine." \
|
||||||
--simulate-stt --stt-delay 0.1
|
--simulate-stt --stt-delay 0.1
|
||||||
|
|
||||||
|
# Receive JSON sidecar chunks with word-level timestamps
|
||||||
|
python streaming_speech_client.py \
|
||||||
|
--text "Hello world. How are you?" \
|
||||||
|
--stream-audio \
|
||||||
|
--response-format pcm \
|
||||||
|
--word-timestamps
|
||||||
|
|
||||||
|
# Also save every received audio.chunk frame as individual PCM/WAV files
|
||||||
|
python streaming_speech_client.py \
|
||||||
|
--text "Hello world. How are you?" \
|
||||||
|
--stream-audio \
|
||||||
|
--response-format pcm \
|
||||||
|
--word-timestamps \
|
||||||
|
--save-chunks
|
||||||
|
|
||||||
# VoiceDesign task
|
# VoiceDesign task
|
||||||
python streaming_speech_client.py \
|
python streaming_speech_client.py \
|
||||||
--text "Today is a great day. The weather is nice." \
|
--text "Today is a great day. The weather is nice." \
|
||||||
@@ -37,8 +52,10 @@ Requirements:
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import base64
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import wave
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import websockets
|
import websockets
|
||||||
@@ -47,6 +64,37 @@ except ImportError:
|
|||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def save_audio_file(output_dir: str, sentence_index: int, response_format: str, chunks: list[bytes]) -> str:
|
||||||
|
audio_bytes = b"".join(chunks)
|
||||||
|
if response_format == "pcm":
|
||||||
|
raw_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.pcm")
|
||||||
|
with open(raw_path, "wb") as f:
|
||||||
|
f.write(audio_bytes)
|
||||||
|
|
||||||
|
wav_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.wav")
|
||||||
|
save_pcm_wav(wav_path, audio_bytes)
|
||||||
|
return wav_path
|
||||||
|
|
||||||
|
filename = os.path.join(output_dir, f"sentence_{sentence_index:03d}.{response_format}")
|
||||||
|
with open(filename, "wb") as f:
|
||||||
|
f.write(audio_bytes)
|
||||||
|
return filename
|
||||||
|
|
||||||
|
|
||||||
|
def save_pcm_wav(path: str, audio_bytes: bytes, sample_rate: int = 24000) -> None:
|
||||||
|
with wave.open(path, "wb") as wf:
|
||||||
|
wf.setnchannels(1)
|
||||||
|
wf.setsampwidth(2)
|
||||||
|
wf.setframerate(sample_rate)
|
||||||
|
wf.writeframes(audio_bytes)
|
||||||
|
|
||||||
|
|
||||||
|
def timestamp_words_text(timestamps: list[dict] | None) -> str:
|
||||||
|
if not timestamps:
|
||||||
|
return ""
|
||||||
|
return " ".join(str(item.get("word", "")) for item in timestamps if item.get("word"))
|
||||||
|
|
||||||
|
|
||||||
async def stream_tts(
|
async def stream_tts(
|
||||||
url: str,
|
url: str,
|
||||||
text: str,
|
text: str,
|
||||||
@@ -54,6 +102,7 @@ async def stream_tts(
|
|||||||
output_dir: str,
|
output_dir: str,
|
||||||
simulate_stt: bool = False,
|
simulate_stt: bool = False,
|
||||||
stt_delay: float = 0.1,
|
stt_delay: float = 0.1,
|
||||||
|
save_chunks: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Connect to the streaming TTS endpoint and process audio responses."""
|
"""Connect to the streaming TTS endpoint and process audio responses."""
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
os.makedirs(output_dir, exist_ok=True)
|
||||||
@@ -99,8 +148,14 @@ async def stream_tts(
|
|||||||
sender_task = asyncio.create_task(send_text())
|
sender_task = asyncio.create_task(send_text())
|
||||||
|
|
||||||
response_format = config.get("response_format", "wav")
|
response_format = config.get("response_format", "wav")
|
||||||
|
word_timestamps = bool(config.get("word_timestamps", False))
|
||||||
current_sentence_index = 0
|
current_sentence_index = 0
|
||||||
|
current_sentence_text = ""
|
||||||
current_chunks: list[bytes] = []
|
current_chunks: list[bytes] = []
|
||||||
|
current_timestamps: list[dict] = []
|
||||||
|
# Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
|
||||||
|
# the sentence is a failure only if no frame ever carried timestamps.
|
||||||
|
alignment_frame_seen = False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
@@ -116,22 +171,94 @@ async def stream_tts(
|
|||||||
|
|
||||||
if msg_type == "audio.start":
|
if msg_type == "audio.start":
|
||||||
current_sentence_index = msg["sentence_index"]
|
current_sentence_index = msg["sentence_index"]
|
||||||
|
current_sentence_text = msg.get("sentence_text", "")
|
||||||
current_chunks = []
|
current_chunks = []
|
||||||
print(f" [sentence {msg['sentence_index']}] Generating: {msg['sentence_text']!r}")
|
current_timestamps = []
|
||||||
|
alignment_frame_seen = False
|
||||||
|
print(f" [sentence {msg['sentence_index']}] Generating: {current_sentence_text!r}")
|
||||||
|
elif msg_type == "audio.chunk":
|
||||||
|
audio = base64.b64decode(msg["audio_b64"])
|
||||||
|
current_chunks.append(audio)
|
||||||
|
chunk_timestamps = msg.get("timestamps")
|
||||||
|
# Sentence-level alignment sends a trailing timestamp-only
|
||||||
|
# frame with empty audio; don't write an empty chunk file.
|
||||||
|
if save_chunks and audio:
|
||||||
|
chunk_base = os.path.join(
|
||||||
|
output_dir,
|
||||||
|
f"sentence_{msg['sentence_index']:03d}_chunk_{msg['chunk_id']:03d}",
|
||||||
|
)
|
||||||
|
chunk_pcm = f"{chunk_base}.pcm"
|
||||||
|
chunk_wav = f"{chunk_base}.wav"
|
||||||
|
chunk_json = f"{chunk_base}_timestamps.json"
|
||||||
|
with open(chunk_pcm, "wb") as f:
|
||||||
|
f.write(audio)
|
||||||
|
save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
|
||||||
|
with open(chunk_json, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(
|
||||||
|
{
|
||||||
|
"sentence_index": msg["sentence_index"],
|
||||||
|
"chunk_id": msg["chunk_id"],
|
||||||
|
"chunk_start_ms": msg.get("chunk_start_ms"),
|
||||||
|
"chunk_end_ms": msg.get("chunk_end_ms"),
|
||||||
|
"sentence_text": current_sentence_text,
|
||||||
|
"covered_text": timestamp_words_text(chunk_timestamps),
|
||||||
|
"timestamps": chunk_timestamps,
|
||||||
|
},
|
||||||
|
f,
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
if chunk_timestamps is None:
|
||||||
|
print(
|
||||||
|
f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
|
||||||
|
f"{len(audio)} bytes, timestamps unavailable"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
alignment_frame_seen = True
|
||||||
|
current_timestamps.extend(chunk_timestamps)
|
||||||
|
print(
|
||||||
|
f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
|
||||||
|
f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
|
||||||
|
)
|
||||||
|
if save_chunks and audio:
|
||||||
|
print(f" saved chunk -> {chunk_wav}")
|
||||||
elif msg_type == "audio.done":
|
elif msg_type == "audio.done":
|
||||||
filename = os.path.join(
|
filename = save_audio_file(
|
||||||
output_dir,
|
output_dir,
|
||||||
f"sentence_{msg['sentence_index']:03d}.{response_format}",
|
msg["sentence_index"],
|
||||||
|
response_format,
|
||||||
|
current_chunks,
|
||||||
)
|
)
|
||||||
with open(filename, "wb") as f:
|
if word_timestamps:
|
||||||
f.write(b"".join(current_chunks))
|
# `null` on failure, else the accumulated list (`[]` for silence).
|
||||||
|
sentence_timestamps = current_timestamps if alignment_frame_seen else None
|
||||||
|
ts_filename = os.path.join(
|
||||||
|
output_dir,
|
||||||
|
f"sentence_{msg['sentence_index']:03d}_timestamps.json",
|
||||||
|
)
|
||||||
|
with open(ts_filename, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(
|
||||||
|
{
|
||||||
|
"sentence_index": msg["sentence_index"],
|
||||||
|
"sentence_text": current_sentence_text,
|
||||||
|
"covered_text": timestamp_words_text(sentence_timestamps),
|
||||||
|
"timestamps": sentence_timestamps,
|
||||||
|
},
|
||||||
|
f,
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
print(
|
print(
|
||||||
f" [sentence {msg['sentence_index']}] Done"
|
f" [sentence {msg['sentence_index']}] Done"
|
||||||
f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
|
f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
|
||||||
f" error={msg.get('error', False)}"
|
f" error={msg.get('error', False)}"
|
||||||
f" -> {filename}"
|
f" -> {filename}"
|
||||||
)
|
)
|
||||||
|
if word_timestamps:
|
||||||
|
print(f" [sentence {msg['sentence_index']}] Timestamps -> {ts_filename}")
|
||||||
current_chunks = []
|
current_chunks = []
|
||||||
|
current_timestamps = []
|
||||||
|
alignment_frame_seen = False
|
||||||
elif msg_type == "session.done":
|
elif msg_type == "session.done":
|
||||||
print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated")
|
print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated")
|
||||||
break
|
break
|
||||||
@@ -189,6 +316,16 @@ def main():
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Receive one or more PCM chunks per sentence (requires --response-format pcm)",
|
help="Receive one or more PCM chunks per sentence (requires --response-format pcm)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--word-timestamps",
|
||||||
|
action="store_true",
|
||||||
|
help="Receive JSON sidecar chunks with word-level timestamps (requires --stream-audio --response-format pcm)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--save-chunks",
|
||||||
|
action="store_true",
|
||||||
|
help="Save each audio.chunk frame as sentence_XXX_chunk_YYY.{pcm,wav,json}",
|
||||||
|
)
|
||||||
parser.add_argument("--speed", type=float, default=1.0, help="Playback speed (0.25-4.0)")
|
parser.add_argument("--speed", type=float, default=1.0, help="Playback speed (0.25-4.0)")
|
||||||
parser.add_argument("--max-new-tokens", type=int, default=None, help="Max tokens")
|
parser.add_argument("--max-new-tokens", type=int, default=None, help="Max tokens")
|
||||||
|
|
||||||
@@ -238,6 +375,10 @@ def main():
|
|||||||
config[key] = val
|
config[key] = val
|
||||||
if args.stream_audio:
|
if args.stream_audio:
|
||||||
config["stream_audio"] = True
|
config["stream_audio"] = True
|
||||||
|
if args.word_timestamps:
|
||||||
|
config["word_timestamps"] = True
|
||||||
|
config["stream_audio"] = True
|
||||||
|
config["response_format"] = "pcm"
|
||||||
if args.x_vector_only_mode:
|
if args.x_vector_only_mode:
|
||||||
config["x_vector_only_mode"] = True
|
config["x_vector_only_mode"] = True
|
||||||
|
|
||||||
@@ -249,6 +390,7 @@ def main():
|
|||||||
output_dir=args.output_dir,
|
output_dir=args.output_dir,
|
||||||
simulate_stt=args.simulate_stt,
|
simulate_stt=args.simulate_stt,
|
||||||
stt_delay=args.stt_delay,
|
stt_delay=args.stt_delay,
|
||||||
|
save_chunks=args.save_chunks,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,457 @@
|
|||||||
|
"""Gradio demo for streaming TTS word-level timestamps.
|
||||||
|
|
||||||
|
Connects to the WebSocket endpoint ``/v1/audio/speech/stream`` with
|
||||||
|
``word_timestamps: true`` and visualizes the alignment: each sentence's
|
||||||
|
audio plays in a native ``<audio>`` element while its text is rendered as
|
||||||
|
inline word spans, the current word highlighting as ``audio.currentTime``
|
||||||
|
crosses each ``start_ms``.
|
||||||
|
|
||||||
|
A "Stop (barge-in)" button cuts playback and reports the last-spoken word,
|
||||||
|
demonstrating the voice-agent barge-in use case.
|
||||||
|
|
||||||
|
Timestamps are sentence-level: audio streams in real time, then the forced
|
||||||
|
aligner runs once per sentence and the complete word timestamps arrive in a
|
||||||
|
trailing frame. The demo therefore plays one sentence block at a time.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
# Launch the server with a forced aligner first, e.g.:
|
||||||
|
# vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni \
|
||||||
|
# --deploy-config vllm_omni/deploy/qwen3_tts.yaml --trust-remote-code \
|
||||||
|
# --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B
|
||||||
|
# Then:
|
||||||
|
python word_timestamps_demo.py --api-base http://localhost:8000
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
try:
|
||||||
|
import gradio as gr
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
|
||||||
|
try:
|
||||||
|
import websockets
|
||||||
|
except ImportError:
|
||||||
|
raise ImportError("websockets is required to run this demo. Install it with: pip install websockets") from None
|
||||||
|
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||||
|
from tts_common import (
|
||||||
|
PCM_SAMPLE_RATE,
|
||||||
|
SUPPORTED_LANGUAGES,
|
||||||
|
TASK_TYPES,
|
||||||
|
add_common_args,
|
||||||
|
fetch_voices,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _upstream_ws_url(api_base: str) -> str:
|
||||||
|
"""Map an http(s) API base to its WebSocket streaming endpoint."""
|
||||||
|
base = api_base.rstrip("/")
|
||||||
|
if base.startswith("https://"):
|
||||||
|
base = "wss://" + base[len("https://") :]
|
||||||
|
elif base.startswith("http://"):
|
||||||
|
base = "ws://" + base[len("http://") :]
|
||||||
|
else:
|
||||||
|
base = "ws://" + base
|
||||||
|
return base + "/v1/audio/speech/stream"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Front-end ────────────────────────────────────────────────────────
|
||||||
|
PLAYER_HTML = """
|
||||||
|
<div id="wt-root">
|
||||||
|
<div style="display:flex; align-items:center; gap:10px;">
|
||||||
|
<div id="wt-dot" style="width:10px;height:10px;border-radius:50%;background:#ccc;flex-shrink:0;"></div>
|
||||||
|
<span id="wt-status" style="font-weight:600;font-size:1.05em;">Ready</span>
|
||||||
|
<button id="wt-stop" onclick="window.wtStop()"
|
||||||
|
style="display:none; margin-left:auto; padding:5px 16px; border-radius:6px; border:1px solid #EF5552;
|
||||||
|
background:#fff; color:#EF5552; cursor:pointer; font-size:0.85em;">Stop (barge-in)</button>
|
||||||
|
</div>
|
||||||
|
<div id="wt-barge" style="display:none; margin-top:10px; padding:8px 12px; border-radius:6px;
|
||||||
|
background:#fff5f5; border:1px solid #f3c2c2; color:#b02a2a; font-size:0.9em;"></div>
|
||||||
|
<div id="wt-sentences" style="margin-top:12px; display:flex; flex-direction:column; gap:14px;"></div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _build_head_js(sample_rate: int) -> str:
|
||||||
|
"""Browser-side player: WebSocket -> per-sentence <audio> + word highlight."""
|
||||||
|
return f"""
|
||||||
|
<script>
|
||||||
|
const WT_SR = {sample_rate};
|
||||||
|
let wtWS = null, wtQueue = [], wtPlaying = false, wtCur = null, wtLastWord = "";
|
||||||
|
|
||||||
|
function wtSet(text, color) {{
|
||||||
|
const s = document.getElementById('wt-status');
|
||||||
|
const d = document.getElementById('wt-dot');
|
||||||
|
if (s) s.textContent = text;
|
||||||
|
if (d) d.style.background = color || '#ccc';
|
||||||
|
}}
|
||||||
|
|
||||||
|
function wtStopBtn(show) {{
|
||||||
|
const b = document.getElementById('wt-stop');
|
||||||
|
if (b) b.style.display = show ? 'inline-block' : 'none';
|
||||||
|
}}
|
||||||
|
|
||||||
|
// Build a WAV Blob (16-bit mono PCM) from concatenated Int16 samples.
|
||||||
|
function wtPcmToWav(int16, sr) {{
|
||||||
|
const n = int16.length;
|
||||||
|
const buf = new ArrayBuffer(44 + n * 2);
|
||||||
|
const v = new DataView(buf);
|
||||||
|
const w = (off, s) => {{ for (let i = 0; i < s.length; i++) v.setUint8(off + i, s.charCodeAt(i)); }};
|
||||||
|
w(0, 'RIFF'); v.setUint32(4, 36 + n * 2, true); w(8, 'WAVE');
|
||||||
|
w(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
|
||||||
|
v.setUint32(24, sr, true); v.setUint32(28, sr * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true);
|
||||||
|
w(36, 'data'); v.setUint32(40, n * 2, true);
|
||||||
|
let off = 44;
|
||||||
|
for (let i = 0; i < n; i++) {{ v.setInt16(off, int16[i], true); off += 2; }}
|
||||||
|
return new Blob([buf], {{ type: 'audio/wav' }});
|
||||||
|
}}
|
||||||
|
|
||||||
|
function wtConcat(chunks) {{
|
||||||
|
let total = 0;
|
||||||
|
for (const c of chunks) total += c.length;
|
||||||
|
const out = new Int16Array(total);
|
||||||
|
let off = 0;
|
||||||
|
for (const c of chunks) {{ out.set(c, off); off += c.length; }}
|
||||||
|
return out;
|
||||||
|
}}
|
||||||
|
|
||||||
|
// Render one finished sentence: <audio> + inline word spans.
|
||||||
|
function wtRenderSentence(sentence) {{
|
||||||
|
const wrap = document.getElementById('wt-sentences');
|
||||||
|
if (!wrap) return;
|
||||||
|
const block = document.createElement('div');
|
||||||
|
block.style.cssText = 'border:1px solid #e3e8ef;border-radius:8px;padding:10px 12px;';
|
||||||
|
|
||||||
|
const head = document.createElement('div');
|
||||||
|
head.style.cssText = 'font-size:0.75em;color:#888;margin-bottom:6px;';
|
||||||
|
head.textContent = 'Sentence ' + sentence.index +
|
||||||
|
(sentence.timestamps ? ' (' + sentence.timestamps.length + ' words)' : ' (no timestamps)');
|
||||||
|
block.appendChild(head);
|
||||||
|
|
||||||
|
const textEl = document.createElement('div');
|
||||||
|
textEl.style.cssText = 'font-size:1.15em;line-height:1.8;margin-bottom:8px;';
|
||||||
|
if (sentence.timestamps && sentence.timestamps.length) {{
|
||||||
|
sentence.spans = [];
|
||||||
|
for (const ts of sentence.timestamps) {{
|
||||||
|
const span = document.createElement('span');
|
||||||
|
span.textContent = ts.word;
|
||||||
|
span.dataset.start = ts.start_ms;
|
||||||
|
span.dataset.end = ts.end_ms;
|
||||||
|
span.style.cssText = 'padding:1px 2px;border-radius:3px;transition:background 0.1s;';
|
||||||
|
textEl.appendChild(span);
|
||||||
|
// Keep spacing for spaced languages; CJK chars sit flush.
|
||||||
|
textEl.appendChild(document.createTextNode(' '));
|
||||||
|
sentence.spans.push(span);
|
||||||
|
}}
|
||||||
|
}} else {{
|
||||||
|
textEl.textContent = sentence.text || '(timestamps unavailable)';
|
||||||
|
}}
|
||||||
|
block.appendChild(textEl);
|
||||||
|
|
||||||
|
const audio = document.createElement('audio');
|
||||||
|
audio.controls = true;
|
||||||
|
audio.style.cssText = 'width:100%;';
|
||||||
|
audio.src = URL.createObjectURL(wtPcmToWav(wtConcat(sentence.chunks), WT_SR));
|
||||||
|
block.appendChild(audio);
|
||||||
|
sentence.audio = audio;
|
||||||
|
|
||||||
|
// Highlight the current word. Driven by requestAnimationFrame (~60fps)
|
||||||
|
// rather than the <audio> 'timeupdate' event, which only fires ~4x/sec
|
||||||
|
// and is too coarse for short words. We light the latest word whose
|
||||||
|
// start_ms has passed, so there is always exactly one active word (no
|
||||||
|
// inter-word flicker) and wtLastWord stays well-defined for barge-in.
|
||||||
|
let raf = null;
|
||||||
|
function highlight() {{
|
||||||
|
if (!sentence.spans) return;
|
||||||
|
const t = audio.currentTime * 1000;
|
||||||
|
let idx = -1;
|
||||||
|
for (let i = 0; i < sentence.spans.length; i++) {{
|
||||||
|
if (t >= +sentence.spans[i].dataset.start) idx = i;
|
||||||
|
else break;
|
||||||
|
}}
|
||||||
|
sentence.spans.forEach((sp, i) => {{
|
||||||
|
const on = i === idx;
|
||||||
|
sp.style.background = on ? '#ffe680' : 'transparent';
|
||||||
|
sp.style.fontWeight = on ? '700' : '400';
|
||||||
|
}});
|
||||||
|
if (idx >= 0) wtLastWord = sentence.spans[idx].textContent;
|
||||||
|
}}
|
||||||
|
function loop() {{
|
||||||
|
highlight();
|
||||||
|
if (!audio.paused && !audio.ended) raf = requestAnimationFrame(loop);
|
||||||
|
}}
|
||||||
|
audio.addEventListener('play', () => {{
|
||||||
|
wtCur = audio;
|
||||||
|
wtSet('Playing sentence ' + sentence.index + '...', '#64dd17');
|
||||||
|
cancelAnimationFrame(raf);
|
||||||
|
raf = requestAnimationFrame(loop);
|
||||||
|
}});
|
||||||
|
audio.addEventListener('pause', () => cancelAnimationFrame(raf));
|
||||||
|
audio.addEventListener('seeked', highlight);
|
||||||
|
audio.addEventListener('ended', () => {{
|
||||||
|
cancelAnimationFrame(raf);
|
||||||
|
highlight();
|
||||||
|
wtPlaying = false;
|
||||||
|
wtPlayNext();
|
||||||
|
}});
|
||||||
|
|
||||||
|
wrap.appendChild(block);
|
||||||
|
wtQueue.push(audio);
|
||||||
|
wtPlayNext();
|
||||||
|
}}
|
||||||
|
|
||||||
|
function wtPlayNext() {{
|
||||||
|
if (wtPlaying) return;
|
||||||
|
const next = wtQueue.shift();
|
||||||
|
if (!next) {{
|
||||||
|
if (wtWS === null) {{ wtSet('Done', '#64dd17'); wtStopBtn(false); }}
|
||||||
|
return;
|
||||||
|
}}
|
||||||
|
wtPlaying = true;
|
||||||
|
next.play().catch(() => {{ wtPlaying = false; }});
|
||||||
|
}}
|
||||||
|
|
||||||
|
window.wtStop = function() {{
|
||||||
|
// Barge-in: cut playback, stop generation, report the last-spoken word.
|
||||||
|
if (wtCur) {{ try {{ wtCur.pause(); }} catch (e) {{}} }}
|
||||||
|
wtQueue = [];
|
||||||
|
wtPlaying = false;
|
||||||
|
if (wtWS) {{ try {{ wtWS.close(); }} catch (e) {{}} wtWS = null; }}
|
||||||
|
wtSet('Stopped (barge-in)', '#999');
|
||||||
|
wtStopBtn(false);
|
||||||
|
const b = document.getElementById('wt-barge');
|
||||||
|
if (b) {{
|
||||||
|
b.style.display = 'block';
|
||||||
|
b.textContent = wtLastWord
|
||||||
|
? 'Barge-in at word: "' + wtLastWord + '" (a voice agent would resume the conversation from here)'
|
||||||
|
: 'Barge-in before any word was spoken.';
|
||||||
|
}}
|
||||||
|
}};
|
||||||
|
|
||||||
|
window.wtGenerate = function(payload) {{
|
||||||
|
if (wtWS) {{ try {{ wtWS.close(); }} catch (e) {{}} }}
|
||||||
|
wtQueue = []; wtPlaying = false; wtCur = null; wtLastWord = "";
|
||||||
|
document.getElementById('wt-sentences').innerHTML = '';
|
||||||
|
const barge = document.getElementById('wt-barge');
|
||||||
|
if (barge) {{ barge.style.display = 'none'; barge.textContent = ''; }}
|
||||||
|
wtStopBtn(true);
|
||||||
|
wtSet('Connecting...', '#4A90D9');
|
||||||
|
|
||||||
|
const proto = location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||||
|
const ws = new WebSocket(proto + '//' + location.host + '/proxy/stream');
|
||||||
|
wtWS = ws;
|
||||||
|
const sentences = {{}};
|
||||||
|
|
||||||
|
ws.onopen = () => {{
|
||||||
|
wtSet('Streaming...', '#4A90D9');
|
||||||
|
ws.send(JSON.stringify(Object.assign({{ type: 'session.config' }}, payload.config)));
|
||||||
|
ws.send(JSON.stringify({{ type: 'input.text', text: payload.text }}));
|
||||||
|
ws.send(JSON.stringify({{ type: 'input.done' }}));
|
||||||
|
}};
|
||||||
|
|
||||||
|
ws.onmessage = (ev) => {{
|
||||||
|
let msg;
|
||||||
|
try {{ msg = JSON.parse(ev.data); }} catch (e) {{ return; }}
|
||||||
|
if (msg.type === 'audio.start') {{
|
||||||
|
sentences[msg.sentence_index] = {{
|
||||||
|
index: msg.sentence_index, text: msg.sentence_text || '',
|
||||||
|
chunks: [], timestamps: null
|
||||||
|
}};
|
||||||
|
}} else if (msg.type === 'audio.chunk') {{
|
||||||
|
const s = sentences[msg.sentence_index];
|
||||||
|
if (!s) return;
|
||||||
|
if (msg.audio_b64) {{
|
||||||
|
const bin = atob(msg.audio_b64);
|
||||||
|
const bytes = new Uint8Array(bin.length);
|
||||||
|
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||||
|
const usable = bytes.length - (bytes.length % 2);
|
||||||
|
if (usable > 0) s.chunks.push(new Int16Array(bytes.buffer, 0, usable / 2));
|
||||||
|
}}
|
||||||
|
if (msg.timestamps != null) s.timestamps = msg.timestamps;
|
||||||
|
}} else if (msg.type === 'audio.done') {{
|
||||||
|
const s = sentences[msg.sentence_index];
|
||||||
|
if (s) wtRenderSentence(s);
|
||||||
|
}} else if (msg.type === 'session.done') {{
|
||||||
|
wtWS = null;
|
||||||
|
try {{ ws.close(); }} catch (e) {{}}
|
||||||
|
wtPlayNext();
|
||||||
|
}} else if (msg.type === 'error') {{
|
||||||
|
wtSet('Error: ' + (msg.message || 'unknown'), '#EF5552');
|
||||||
|
}}
|
||||||
|
}};
|
||||||
|
|
||||||
|
ws.onerror = () => wtSet('WebSocket error', '#EF5552');
|
||||||
|
ws.onclose = () => {{ if (wtWS === ws) wtWS = null; }};
|
||||||
|
}};
|
||||||
|
</script>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def create_app(api_base: str) -> FastAPI:
|
||||||
|
"""FastAPI app with a same-origin WebSocket proxy + Gradio UI."""
|
||||||
|
fastapi_app = FastAPI()
|
||||||
|
upstream_url = _upstream_ws_url(api_base)
|
||||||
|
|
||||||
|
@fastapi_app.websocket("/proxy/stream")
|
||||||
|
async def proxy_stream(client_ws: WebSocket):
|
||||||
|
await client_ws.accept()
|
||||||
|
try:
|
||||||
|
async with websockets.connect(upstream_url, max_size=None) as upstream:
|
||||||
|
|
||||||
|
async def client_to_upstream():
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await upstream.send(await client_ws.receive_text())
|
||||||
|
except (WebSocketDisconnect, Exception):
|
||||||
|
await upstream.close()
|
||||||
|
|
||||||
|
async def upstream_to_client():
|
||||||
|
try:
|
||||||
|
async for msg in upstream:
|
||||||
|
if isinstance(msg, bytes):
|
||||||
|
await client_ws.send_bytes(msg)
|
||||||
|
else:
|
||||||
|
await client_ws.send_text(msg)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
await client_ws.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
await asyncio.gather(client_to_upstream(), upstream_to_client())
|
||||||
|
except Exception as exc:
|
||||||
|
logger.exception("WS proxy error")
|
||||||
|
try:
|
||||||
|
await client_ws.send_text(json.dumps({"type": "error", "message": f"proxy: {exc}"}))
|
||||||
|
await client_ws.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
voices = fetch_voices(api_base)
|
||||||
|
|
||||||
|
with gr.Blocks(title="Qwen3-TTS Word Timestamps") as demo:
|
||||||
|
gr.HTML(f"""
|
||||||
|
<div style="margin-bottom:8px;">
|
||||||
|
<h1 style="margin:0; font-size:1.5em;">Qwen3-TTS Streaming Word Timestamps</h1>
|
||||||
|
<span style="font-size:0.85em; color:#666;">
|
||||||
|
Served by <a href="https://github.com/vllm-project/vllm-omni" target="_blank">vLLM-Omni</a>
|
||||||
|
· <code>{api_base}</code> ·
|
||||||
|
launch the server with <code>--forced-aligner</code>.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
""")
|
||||||
|
|
||||||
|
with gr.Row():
|
||||||
|
with gr.Column(scale=3):
|
||||||
|
text_input = gr.Textbox(
|
||||||
|
label="Text to Synthesize",
|
||||||
|
value="Hello world. How are you today? I am doing great.",
|
||||||
|
lines=3,
|
||||||
|
)
|
||||||
|
with gr.Row():
|
||||||
|
task_type = gr.Radio(choices=TASK_TYPES, value="CustomVoice", label="Task Type", scale=2)
|
||||||
|
language = gr.Dropdown(choices=SUPPORTED_LANGUAGES, value="Auto", label="Language", scale=1)
|
||||||
|
voice = gr.Dropdown(
|
||||||
|
choices=voices,
|
||||||
|
value=voices[0] if voices else None,
|
||||||
|
label="Speaker",
|
||||||
|
allow_custom_value=True,
|
||||||
|
)
|
||||||
|
instructions = gr.Textbox(
|
||||||
|
label="Instructions",
|
||||||
|
placeholder="Optional style/emotion (required for VoiceDesign)",
|
||||||
|
lines=1,
|
||||||
|
)
|
||||||
|
with gr.Column(visible=False) as ref_group:
|
||||||
|
ref_audio_url = gr.Textbox(
|
||||||
|
label="Reference Audio URL (Base task)",
|
||||||
|
placeholder="https://example.com/reference.wav",
|
||||||
|
lines=1,
|
||||||
|
)
|
||||||
|
ref_text = gr.Textbox(label="Reference Transcript", lines=1)
|
||||||
|
generate_btn = gr.Button("Generate + Align", variant="primary", size="lg")
|
||||||
|
|
||||||
|
with gr.Column(scale=2):
|
||||||
|
gr.HTML(value=PLAYER_HTML)
|
||||||
|
|
||||||
|
hidden_payload = gr.Textbox(visible=False, elem_id="wt-payload")
|
||||||
|
|
||||||
|
def on_task_change(tt):
|
||||||
|
return gr.update(visible=(tt == "Base"))
|
||||||
|
|
||||||
|
task_type.change(fn=on_task_change, inputs=[task_type], outputs=[ref_group])
|
||||||
|
|
||||||
|
def on_generate(text, tt, voice_v, lang_v, instr_v, ref_url, ref_t):
|
||||||
|
if not text or not text.strip():
|
||||||
|
raise gr.Error("Please enter text to synthesize.")
|
||||||
|
config: dict = {
|
||||||
|
"task_type": tt,
|
||||||
|
"word_timestamps": True,
|
||||||
|
"stream_audio": True,
|
||||||
|
"response_format": "pcm",
|
||||||
|
}
|
||||||
|
if lang_v:
|
||||||
|
config["language"] = lang_v
|
||||||
|
if tt == "CustomVoice":
|
||||||
|
if voice_v:
|
||||||
|
config["voice"] = voice_v
|
||||||
|
if instr_v and instr_v.strip():
|
||||||
|
config["instructions"] = instr_v.strip()
|
||||||
|
elif tt == "VoiceDesign":
|
||||||
|
if not instr_v or not instr_v.strip():
|
||||||
|
raise gr.Error("VoiceDesign requires voice style instructions.")
|
||||||
|
config["instructions"] = instr_v.strip()
|
||||||
|
elif tt == "Base":
|
||||||
|
if not ref_url or not ref_url.strip():
|
||||||
|
raise gr.Error("Base (voice clone) needs a reference audio URL.")
|
||||||
|
config["ref_audio"] = ref_url.strip()
|
||||||
|
if ref_t and ref_t.strip():
|
||||||
|
config["ref_text"] = ref_t.strip()
|
||||||
|
return json.dumps({"config": config, "text": text.strip()})
|
||||||
|
|
||||||
|
generate_btn.click(
|
||||||
|
fn=on_generate,
|
||||||
|
inputs=[text_input, task_type, voice, language, instructions, ref_audio_url, ref_text],
|
||||||
|
outputs=[hidden_payload],
|
||||||
|
).then(
|
||||||
|
fn=lambda p: p,
|
||||||
|
inputs=[hidden_payload],
|
||||||
|
outputs=[hidden_payload],
|
||||||
|
js="(p) => { if (p && p.trim()) window.wtGenerate(JSON.parse(p)); return p; }",
|
||||||
|
)
|
||||||
|
|
||||||
|
demo.queue()
|
||||||
|
|
||||||
|
return gr.mount_gradio_app(
|
||||||
|
fastapi_app,
|
||||||
|
demo,
|
||||||
|
path="/",
|
||||||
|
head=_build_head_js(PCM_SAMPLE_RATE),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser(description="Gradio demo for streaming TTS word-level timestamps.")
|
||||||
|
add_common_args(parser)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
print(f"Connecting to vLLM server at: {args.api_base}")
|
||||||
|
app = create_app(args.api_base)
|
||||||
|
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
uvicorn.run(app, host=args.host, port=args.port)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -76,6 +76,10 @@ demo = [
|
|||||||
"cn2an>=0.5.22", # Chinese number normalization for TTS (e.g. "1次" -> "一次")
|
"cn2an>=0.5.22", # Chinese number normalization for TTS (e.g. "1次" -> "一次")
|
||||||
]
|
]
|
||||||
|
|
||||||
|
forced-aligner = [
|
||||||
|
"qwen-asr>=0.0.6", # faithful Japanese/Korean word segmentation for the TTS forced aligner
|
||||||
|
]
|
||||||
|
|
||||||
# Local-test-only extras
|
# Local-test-only extras
|
||||||
local = [
|
local = [
|
||||||
"FlagEmbedding",
|
"FlagEmbedding",
|
||||||
@@ -304,3 +308,6 @@ CANN = "CANN"
|
|||||||
thi = "thi"
|
thi = "thi"
|
||||||
ND = "ND"
|
ND = "ND"
|
||||||
byt5 = "byt5"
|
byt5 = "byt5"
|
||||||
|
# "seeked" is the standard HTML media element DOM event name (fires when a
|
||||||
|
# seek completes), used by the word-timestamps Gradio demo. Not a typo.
|
||||||
|
seeked = "seeked"
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import base64
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi import FastAPI, WebSocket
|
from fastapi import FastAPI, WebSocket
|
||||||
@@ -9,6 +11,7 @@ from starlette.websockets import WebSocketDisconnect
|
|||||||
from vllm_omni.entrypoints.openai import serving_speech_stream as streaming_speech_module
|
from vllm_omni.entrypoints.openai import serving_speech_stream as streaming_speech_module
|
||||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||||
from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler
|
from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler
|
||||||
|
from vllm_omni.utils.forced_aligner import WordTimestamp
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
@@ -25,10 +28,11 @@ def _build_test_app(
|
|||||||
speech_service = mocker.MagicMock(spec=OmniOpenAIServingSpeech)
|
speech_service = mocker.MagicMock(spec=OmniOpenAIServingSpeech)
|
||||||
speech_service._generate_audio_bytes = mocker.AsyncMock(return_value=(b"RIFF" + b"\x00" * 32, "audio/wav"))
|
speech_service._generate_audio_bytes = mocker.AsyncMock(return_value=(b"RIFF" + b"\x00" * 32, "audio/wav"))
|
||||||
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req-1", object(), {}))
|
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req-1", object(), {}))
|
||||||
|
speech_service.forced_aligner_config = None
|
||||||
|
|
||||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False):
|
||||||
for chunk in (b"\x01\x02", b"\x03\x04\x05"):
|
for chunk in (b"\x01\x02", b"\x03\x04\x05"):
|
||||||
yield chunk
|
yield (chunk, 24000) if include_sample_rate else chunk
|
||||||
|
|
||||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||||
speech_service.engine_client = mocker.MagicMock()
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
@@ -82,6 +86,7 @@ class TestStreamingSpeechWebSocket:
|
|||||||
speech_service._generate_audio_bytes = mocker.AsyncMock(return_value=(b"", "audio/wav"))
|
speech_service._generate_audio_bytes = mocker.AsyncMock(return_value=(b"", "audio/wav"))
|
||||||
speech_service.engine_client = mocker.MagicMock()
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||||
|
speech_service.forced_aligner_config = None
|
||||||
|
|
||||||
async def mock_prepare_speech_generation(request):
|
async def mock_prepare_speech_generation(request):
|
||||||
captured_requests.append(request)
|
captured_requests.append(request)
|
||||||
@@ -89,9 +94,9 @@ class TestStreamingSpeechWebSocket:
|
|||||||
|
|
||||||
speech_service._prepare_speech_generation = mock_prepare_speech_generation
|
speech_service._prepare_speech_generation = mock_prepare_speech_generation
|
||||||
|
|
||||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False):
|
||||||
for chunk in (b"\x01\x02", b"\x03\x04\x05", b"\x06"):
|
for chunk in (b"\x01\x02", b"\x03\x04\x05", b"\x06"):
|
||||||
yield chunk
|
yield (chunk, 24000) if include_sample_rate else chunk
|
||||||
|
|
||||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||||
app, _ = _build_test_app(speech_service)
|
app, _ = _build_test_app(speech_service)
|
||||||
@@ -130,6 +135,164 @@ class TestStreamingSpeechWebSocket:
|
|||||||
assert captured_requests[0].initial_codec_chunk_frames == 12
|
assert captured_requests[0].initial_codec_chunk_frames == 12
|
||||||
assert speech_service._generate_audio_bytes.await_count == 0
|
assert speech_service._generate_audio_bytes.await_count == 0
|
||||||
|
|
||||||
|
def test_word_timestamps_requires_configured_aligner(self, mocker: MockerFixture):
|
||||||
|
app, _ = _build_test_app(mocker=mocker)
|
||||||
|
|
||||||
|
with TestClient(app) as client:
|
||||||
|
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||||
|
ws.send_json(
|
||||||
|
{
|
||||||
|
"type": "session.config",
|
||||||
|
"voice": "Vivian",
|
||||||
|
"stream_audio": True,
|
||||||
|
"response_format": "pcm",
|
||||||
|
"word_timestamps": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||||
|
|
||||||
|
error = ws.receive_json()
|
||||||
|
assert error["type"] == "error"
|
||||||
|
assert "without --forced-aligner" in error["message"]
|
||||||
|
|
||||||
|
def test_word_timestamps_emit_sidecar_json_frame(self, mocker: MockerFixture):
|
||||||
|
captured_requests = []
|
||||||
|
speech_service = mocker.MagicMock(spec=OmniOpenAIServingSpeech)
|
||||||
|
speech_service._generate_audio_bytes = mocker.AsyncMock(return_value=(b"", "audio/wav"))
|
||||||
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
|
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||||
|
speech_service.forced_aligner_config = SimpleNamespace(model="aligner")
|
||||||
|
|
||||||
|
async def mock_prepare_speech_generation(request):
|
||||||
|
captured_requests.append(request)
|
||||||
|
return "req-stream", object(), {}
|
||||||
|
|
||||||
|
speech_service._prepare_speech_generation = mock_prepare_speech_generation
|
||||||
|
|
||||||
|
first_chunk = b"\x01" * 1000
|
||||||
|
second_chunk = b"\x02" * 1000
|
||||||
|
|
||||||
|
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False):
|
||||||
|
for chunk in (first_chunk, second_chunk):
|
||||||
|
yield (chunk, 1000) if include_sample_rate else chunk
|
||||||
|
|
||||||
|
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||||
|
# Sentence-level: aligner runs once over the whole sentence audio.
|
||||||
|
mock_align = mocker.AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
WordTimestamp("Hello", 0, 200),
|
||||||
|
WordTimestamp("world", 200, 900),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
mocker.patch.object(streaming_speech_module, "forced_align", mock_align)
|
||||||
|
app, _ = _build_test_app(speech_service)
|
||||||
|
|
||||||
|
with TestClient(app) as client:
|
||||||
|
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||||
|
ws.send_json(
|
||||||
|
{
|
||||||
|
"type": "session.config",
|
||||||
|
"voice": "Vivian",
|
||||||
|
"stream_audio": True,
|
||||||
|
"response_format": "pcm",
|
||||||
|
"word_timestamps": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||||
|
|
||||||
|
start = ws.receive_json()
|
||||||
|
assert start["type"] == "audio.start"
|
||||||
|
assert start["word_timestamps"] is True
|
||||||
|
|
||||||
|
# Audio streams first; timestamps are null until the sentence
|
||||||
|
# is fully aligned.
|
||||||
|
chunk = ws.receive_json()
|
||||||
|
assert chunk["type"] == "audio.chunk"
|
||||||
|
assert chunk["chunk_id"] == 0
|
||||||
|
assert chunk["chunk_start_ms"] == 0
|
||||||
|
assert chunk["chunk_end_ms"] == 500
|
||||||
|
assert chunk["sample_rate"] == 1000
|
||||||
|
assert base64.b64decode(chunk["audio_b64"]) == first_chunk
|
||||||
|
assert chunk["timestamps"] is None
|
||||||
|
|
||||||
|
chunk = ws.receive_json()
|
||||||
|
assert chunk["type"] == "audio.chunk"
|
||||||
|
assert chunk["chunk_id"] == 1
|
||||||
|
assert chunk["chunk_start_ms"] == 500
|
||||||
|
assert chunk["chunk_end_ms"] == 1000
|
||||||
|
assert chunk["sample_rate"] == 1000
|
||||||
|
assert base64.b64decode(chunk["audio_b64"]) == second_chunk
|
||||||
|
assert chunk["timestamps"] is None
|
||||||
|
|
||||||
|
# Final frame: empty audio carrying the whole-sentence timestamps.
|
||||||
|
chunk = ws.receive_json()
|
||||||
|
assert chunk["type"] == "audio.chunk"
|
||||||
|
assert chunk["chunk_id"] == 2
|
||||||
|
assert chunk["chunk_start_ms"] == 0
|
||||||
|
assert chunk["chunk_end_ms"] == 1000
|
||||||
|
assert chunk["sample_rate"] == 1000
|
||||||
|
assert base64.b64decode(chunk["audio_b64"]) == b""
|
||||||
|
assert chunk["timestamps"] == [
|
||||||
|
{"word": "Hello", "start_ms": 0, "end_ms": 200},
|
||||||
|
{"word": "world", "start_ms": 200, "end_ms": 900},
|
||||||
|
]
|
||||||
|
|
||||||
|
done = ws.receive_json()
|
||||||
|
assert done == {"type": "audio.done", "sentence_index": 0, "total_bytes": 2000, "error": False}
|
||||||
|
|
||||||
|
assert captured_requests[0].word_timestamps is True
|
||||||
|
assert mock_align.await_count == 1
|
||||||
|
|
||||||
|
def test_word_timestamps_emit_word_dicts(self, mocker: MockerFixture):
|
||||||
|
# The streaming layer forwards the aligner's (already monotonic,
|
||||||
|
# non-overlapping) words as JSON dicts in the trailing frame.
|
||||||
|
speech_service = mocker.MagicMock(spec=OmniOpenAIServingSpeech)
|
||||||
|
speech_service._generate_audio_bytes = mocker.AsyncMock(return_value=(b"", "audio/wav"))
|
||||||
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
|
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||||
|
speech_service.forced_aligner_config = SimpleNamespace(model="aligner")
|
||||||
|
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req", object(), {}))
|
||||||
|
|
||||||
|
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False):
|
||||||
|
chunk = b"\x01" * 1000
|
||||||
|
yield (chunk, 1000) if include_sample_rate else chunk
|
||||||
|
|
||||||
|
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||||
|
mock_align = mocker.AsyncMock(
|
||||||
|
return_value=[
|
||||||
|
WordTimestamp("Hello", 0, 1000),
|
||||||
|
WordTimestamp("world", 1000, 1200),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
mocker.patch.object(streaming_speech_module, "forced_align", mock_align)
|
||||||
|
app, _ = _build_test_app(speech_service)
|
||||||
|
|
||||||
|
with TestClient(app) as client:
|
||||||
|
with client.websocket_connect("/v1/audio/speech/stream") as ws:
|
||||||
|
ws.send_json(
|
||||||
|
{
|
||||||
|
"type": "session.config",
|
||||||
|
"voice": "Vivian",
|
||||||
|
"stream_audio": True,
|
||||||
|
"response_format": "pcm",
|
||||||
|
"word_timestamps": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
ws.send_json({"type": "input.text", "text": "Hello world. "})
|
||||||
|
|
||||||
|
assert ws.receive_json()["type"] == "audio.start"
|
||||||
|
# Real-time audio chunk (timestamps null), then the timestamp frame.
|
||||||
|
assert ws.receive_json()["timestamps"] is None
|
||||||
|
final = ws.receive_json()
|
||||||
|
assert final["type"] == "audio.chunk"
|
||||||
|
timestamps = final["timestamps"]
|
||||||
|
assert timestamps == [
|
||||||
|
{"word": "Hello", "start_ms": 0, "end_ms": 1000},
|
||||||
|
{"word": "world", "start_ms": 1000, "end_ms": 1200},
|
||||||
|
]
|
||||||
|
for ts in timestamps:
|
||||||
|
assert ts["end_ms"] >= ts["start_ms"]
|
||||||
|
|
||||||
def test_flush_on_input_done(self, mocker: MockerFixture):
|
def test_flush_on_input_done(self, mocker: MockerFixture):
|
||||||
app, _ = _build_test_app(mocker=mocker)
|
app, _ = _build_test_app(mocker=mocker)
|
||||||
|
|
||||||
@@ -249,6 +412,7 @@ class TestStreamingSpeechWebSocket:
|
|||||||
speech_service._generate_pcm_chunks = mocker.AsyncMock()
|
speech_service._generate_pcm_chunks = mocker.AsyncMock()
|
||||||
speech_service.engine_client = mocker.MagicMock()
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||||
|
speech_service.forced_aligner_config = None
|
||||||
app, _ = _build_test_app(speech_service)
|
app, _ = _build_test_app(speech_service)
|
||||||
|
|
||||||
with TestClient(app) as client:
|
with TestClient(app) as client:
|
||||||
@@ -269,8 +433,9 @@ class TestStreamingSpeechWebSocket:
|
|||||||
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req-stream-err", object(), {}))
|
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req-stream-err", object(), {}))
|
||||||
speech_service.engine_client = mocker.MagicMock()
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||||
|
speech_service.forced_aligner_config = None
|
||||||
|
|
||||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False):
|
||||||
yield b"\x01\x02"
|
yield b"\x01\x02"
|
||||||
raise RuntimeError("stream boom")
|
raise RuntimeError("stream boom")
|
||||||
|
|
||||||
@@ -361,8 +526,9 @@ class TestStreamingSpeechWebSocket:
|
|||||||
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req-abort", object(), {}))
|
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req-abort", object(), {}))
|
||||||
speech_service.engine_client = mocker.MagicMock()
|
speech_service.engine_client = mocker.MagicMock()
|
||||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||||
|
speech_service.forced_aligner_config = None
|
||||||
|
|
||||||
async def mock_generate_pcm_chunks(_generator, _request_id):
|
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False):
|
||||||
yield b"\x01\x02"
|
yield b"\x01\x02"
|
||||||
|
|
||||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||||
@@ -385,7 +551,9 @@ class TestStreamingSpeechWebSocket:
|
|||||||
config.ref_audio = None
|
config.ref_audio = None
|
||||||
config.ref_text = None
|
config.ref_text = None
|
||||||
config.x_vector_only_mode = None
|
config.x_vector_only_mode = None
|
||||||
|
config.speaker_embedding = None
|
||||||
config.stream_audio = True
|
config.stream_audio = True
|
||||||
|
config.word_timestamps = False
|
||||||
|
|
||||||
with pytest.raises(WebSocketDisconnect):
|
with pytest.raises(WebSocketDisconnect):
|
||||||
asyncio.run(handler._generate_and_send(websocket, config, "Hello world.", 0))
|
asyncio.run(handler._generate_and_send(websocket, config, "Hello world.", 0))
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm_omni.utils import forced_aligner
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_timestamps_maps_boundary_bins_to_words():
|
||||||
|
logits = np.zeros((4, 5), dtype=np.float32)
|
||||||
|
logits[0, 0] = 1.0
|
||||||
|
logits[1, 2] = 1.0
|
||||||
|
logits[2, 2] = 1.0
|
||||||
|
logits[3, 4] = 1.0
|
||||||
|
|
||||||
|
timestamps = forced_aligner._decode_timestamps(
|
||||||
|
logits=logits,
|
||||||
|
words=["hello", "world"],
|
||||||
|
timestamp_positions=[0, 1, 2, 3],
|
||||||
|
classify_num=5,
|
||||||
|
audio_duration_ms=1000,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert timestamps == [
|
||||||
|
forced_aligner.WordTimestamp("hello", 0, 400),
|
||||||
|
forced_aligner.WordTimestamp("world", 400, 800),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_timestamps_repairs_non_monotonic_bins():
|
||||||
|
# Word 1's end bin (1) dips below its start bin (2); fix_timestamp should
|
||||||
|
# snap it back so the frame never carries end_ms < start_ms.
|
||||||
|
logits = np.zeros((4, 5), dtype=np.float32)
|
||||||
|
logits[0, 0] = 1.0 # word0 start -> bin 0
|
||||||
|
logits[1, 2] = 1.0 # word0 end -> bin 2
|
||||||
|
logits[2, 1] = 1.0 # word1 start -> bin 1 (out of order)
|
||||||
|
logits[3, 3] = 1.0 # word1 end -> bin 3
|
||||||
|
|
||||||
|
timestamps = forced_aligner._decode_timestamps(
|
||||||
|
logits=logits,
|
||||||
|
words=["hello", "world"],
|
||||||
|
timestamp_positions=[0, 1, 2, 3],
|
||||||
|
classify_num=5,
|
||||||
|
timestamp_segment_time_ms=200,
|
||||||
|
audio_duration_ms=1000,
|
||||||
|
)
|
||||||
|
|
||||||
|
for ts in timestamps:
|
||||||
|
assert ts.end_ms >= ts.start_ms
|
||||||
|
# bins [0, 2, 1, 3] -> repaired [0, 2, 2, 3] -> ms x200
|
||||||
|
assert timestamps == [
|
||||||
|
forced_aligner.WordTimestamp("hello", 0, 400),
|
||||||
|
forced_aligner.WordTimestamp("world", 400, 600),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_timestamps_clamps_to_audio_duration_and_stays_ordered():
|
||||||
|
# Bins run past the 1000 ms audio; the decoder must cap each bound at the
|
||||||
|
# audio length and keep words non-overlapping (start <= end, monotonic).
|
||||||
|
logits = np.zeros((4, 5), dtype=np.float32)
|
||||||
|
logits[0, 0] = 1.0 # word0 start -> bin 0
|
||||||
|
logits[1, 3] = 1.0 # word0 end -> bin 3 (1200 ms, past audio)
|
||||||
|
logits[2, 3] = 1.0 # word1 start -> bin 3 (1200 ms, past audio)
|
||||||
|
logits[3, 4] = 1.0 # word1 end -> bin 4 (1600 ms, past audio)
|
||||||
|
|
||||||
|
timestamps = forced_aligner._decode_timestamps(
|
||||||
|
logits=logits,
|
||||||
|
words=["a", "b"],
|
||||||
|
timestamp_positions=[0, 1, 2, 3],
|
||||||
|
classify_num=5,
|
||||||
|
timestamp_segment_time_ms=400,
|
||||||
|
audio_duration_ms=1000,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert timestamps == [
|
||||||
|
forced_aligner.WordTimestamp("a", 0, 1000),
|
||||||
|
forced_aligner.WordTimestamp("b", 1000, 1000),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_timestamps_rejects_marker_count_mismatch():
|
||||||
|
logits = np.zeros((2, 5), dtype=np.float32)
|
||||||
|
|
||||||
|
timestamps = forced_aligner._decode_timestamps(
|
||||||
|
logits=logits,
|
||||||
|
words=["hello", "world"],
|
||||||
|
timestamp_positions=[0, 1],
|
||||||
|
classify_num=5,
|
||||||
|
audio_duration_ms=1000,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert timestamps == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_from_yaml(tmp_path):
|
||||||
|
cfg = tmp_path / "forced_aligner.yaml"
|
||||||
|
cfg.write_text(
|
||||||
|
"""
|
||||||
|
forced_aligner:
|
||||||
|
model: Qwen/Qwen3-ForcedAligner-0.6B
|
||||||
|
gpu_memory_utilization: 0.42
|
||||||
|
dtype: float16
|
||||||
|
max_model_len: 2048
|
||||||
|
trust_remote_code: false
|
||||||
|
""",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
args = type("Args", (), {"forced_aligner": None, "forced_aligner_config": str(cfg)})()
|
||||||
|
|
||||||
|
out = forced_aligner.build_forced_aligner_config(args)
|
||||||
|
|
||||||
|
assert out == forced_aligner.ForcedAlignerConfig(
|
||||||
|
model="Qwen/Qwen3-ForcedAligner-0.6B",
|
||||||
|
runner="pooling",
|
||||||
|
architecture="Qwen3ASRForcedAlignerForTokenClassification",
|
||||||
|
pooling_task="token_classify",
|
||||||
|
gpu_memory_utilization=0.42,
|
||||||
|
dtype="float16",
|
||||||
|
max_model_len=2048,
|
||||||
|
trust_remote_code=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_cli_model_overrides_yaml(tmp_path):
|
||||||
|
cfg = tmp_path / "forced_aligner.yaml"
|
||||||
|
cfg.write_text(
|
||||||
|
"forced_aligner:\n model: old\n gpu_memory_utilization: 0.2\n dtype: float16\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
args = type(
|
||||||
|
"Args",
|
||||||
|
(),
|
||||||
|
{
|
||||||
|
"forced_aligner": "new",
|
||||||
|
"forced_aligner_config": str(cfg),
|
||||||
|
},
|
||||||
|
)()
|
||||||
|
|
||||||
|
out = forced_aligner.build_forced_aligner_config(args)
|
||||||
|
|
||||||
|
assert out is not None
|
||||||
|
# --forced-aligner overrides the YAML model; gpu_memory_utilization/dtype
|
||||||
|
# come from the user YAML (there is no longer a CLI flag for gpu mem).
|
||||||
|
assert out.model == "new"
|
||||||
|
assert out.gpu_memory_utilization == 0.2
|
||||||
|
assert out.dtype == "float16"
|
||||||
|
assert out.runner == "pooling"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_from_cli_model_uses_default_yaml():
|
||||||
|
args = type(
|
||||||
|
"Args",
|
||||||
|
(),
|
||||||
|
{
|
||||||
|
"forced_aligner": "local-aligner",
|
||||||
|
"forced_aligner_config": None,
|
||||||
|
},
|
||||||
|
)()
|
||||||
|
|
||||||
|
out = forced_aligner.build_forced_aligner_config(args)
|
||||||
|
|
||||||
|
assert out is not None
|
||||||
|
assert out.model == "local-aligner"
|
||||||
|
assert out.runner == "pooling"
|
||||||
|
assert out.architecture == "Qwen3ASRForcedAlignerForTokenClassification"
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm_omni.utils import qwen3_force_align_processor as processor
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_prompt_has_boundary_timestamp_markers():
|
||||||
|
prompt = processor.build_prompt(["hello", "world"])
|
||||||
|
|
||||||
|
assert prompt.count("<timestamp>") == 4
|
||||||
|
assert "hello<timestamp><timestamp>world" in prompt
|
||||||
|
assert prompt.endswith("<|im_start|>assistant\n")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"text,expected",
|
||||||
|
[
|
||||||
|
# Punctuation inside a whitespace segment is stripped, not split on:
|
||||||
|
# the hand-rolled splitter used to break these into many fake words.
|
||||||
|
("U.S.A", ["USA"]),
|
||||||
|
("hello, world!", ["hello", "world"]),
|
||||||
|
("don't stop", ["don't", "stop"]),
|
||||||
|
# CJK characters peel off as individual tokens, Latin runs stay whole.
|
||||||
|
("你好world", ["你", "好", "world"]),
|
||||||
|
("我爱 China", ["我", "爱", "China"]),
|
||||||
|
(" spaced out ", ["spaced", "out"]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_tokenize_space_lang_matches_official_segmentation(text, expected):
|
||||||
|
assert processor._tokenize_space_lang(text) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_segment_words_falls_back_to_port_without_qwen_asr(monkeypatch):
|
||||||
|
# With qwen_asr unavailable, segmentation must use the built-in port and
|
||||||
|
# still produce the faithful result for the common (non-JP/KO) case.
|
||||||
|
monkeypatch.setattr(processor, "_get_official_processor", lambda: None)
|
||||||
|
|
||||||
|
assert processor.segment_words("U.S.A is here", "auto") == ["USA", "is", "here"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_fix_timestamp_repairs_dip_onto_monotonic_sequence():
|
||||||
|
# 200 dips below the preceding 400; it snaps back up to 400.
|
||||||
|
assert processor.fix_timestamp([0, 400, 200, 800]) == [0, 400, 400, 800]
|
||||||
|
|
||||||
|
|
||||||
|
def test_fix_timestamp_passes_through_monotonic():
|
||||||
|
assert processor.fix_timestamp([0, 100, 100, 250]) == [0, 100, 100, 250]
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_timestamp_token_id_defaults_to_marker_token():
|
||||||
|
# Regression: the default must resolve the same <timestamp> marker that
|
||||||
|
# build_prompt inserts, not None (which would degrade every request).
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
class FakeTokenizer:
|
||||||
|
def convert_tokens_to_ids(self, token):
|
||||||
|
seen["token"] = token
|
||||||
|
return 151705
|
||||||
|
|
||||||
|
tid = processor.resolve_timestamp_token_id(FakeTokenizer())
|
||||||
|
|
||||||
|
assert tid == 151705
|
||||||
|
assert seen["token"] == processor.TIMESTAMP_TOKEN
|
||||||
|
assert processor.TIMESTAMP_TOKEN in processor.build_prompt(["hello", "world"])
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# Reference deploy knobs for Qwen/Qwen3-ForcedAligner-0.6B, the shared forced
|
||||||
|
# aligner for streaming TTS word timestamps. Pass with --forced-aligner-config;
|
||||||
|
# --forced-aligner overrides the model path without editing this file.
|
||||||
|
#
|
||||||
|
# Run as LLM(runner="pooling") with pooling_task="token_classify". Prompt tokens,
|
||||||
|
# chat template and word segmentation are Qwen-specific and live in
|
||||||
|
# qwen3_force_align_processor (not configurable here).
|
||||||
|
|
||||||
|
forced_aligner:
|
||||||
|
model: Qwen/Qwen3-ForcedAligner-0.6B
|
||||||
|
runner: pooling
|
||||||
|
architecture: Qwen3ASRForcedAlignerForTokenClassification
|
||||||
|
pooling_task: token_classify
|
||||||
|
|
||||||
|
gpu_memory_utilization: 0.30
|
||||||
|
|
||||||
|
dtype: bfloat16
|
||||||
|
max_model_len: 4096
|
||||||
|
trust_remote_code: true
|
||||||
@@ -216,6 +216,29 @@ class OmniServeCommand(CLISubcommand):
|
|||||||
help="Default task type for TTS models (CustomVoice, VoiceDesign, or Base). "
|
help="Default task type for TTS models (CustomVoice, VoiceDesign, or Base). "
|
||||||
"If not specified, will be inferred from model path.",
|
"If not specified, will be inferred from model path.",
|
||||||
)
|
)
|
||||||
|
# Forced aligner / word timestamps. --forced-aligner is the opt-in
|
||||||
|
# toggle; heavier knobs (gpu_memory_utilization, dtype, max_model_len)
|
||||||
|
# live in the deploy YAML passed via --forced-aligner-config.
|
||||||
|
omni_config_group.add_argument(
|
||||||
|
"--forced-aligner",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Enable streaming TTS word timestamps via a forced aligner. "
|
||||||
|
"Pass the aligner model path/name, e.g. 'Qwen/Qwen3-ForcedAligner-0.6B'. "
|
||||||
|
"Disabled when omitted."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
omni_config_group.add_argument(
|
||||||
|
"--forced-aligner-config",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Optional YAML file for forced aligner settings (model, runner, "
|
||||||
|
"gpu_memory_utilization, dtype, max_model_len). The --forced-aligner "
|
||||||
|
"flag, when set, overrides the YAML model field."
|
||||||
|
),
|
||||||
|
)
|
||||||
# TODO(@lishunyang12): deprecate once all models migrate to --deploy-config
|
# TODO(@lishunyang12): deprecate once all models migrate to --deploy-config
|
||||||
omni_config_group.add_argument(
|
omni_config_group.add_argument(
|
||||||
"--stage-configs-path",
|
"--stage-configs-path",
|
||||||
|
|||||||
@@ -144,6 +144,7 @@ from vllm_omni.entrypoints.openai.video_api_utils import decode_audio_url, decod
|
|||||||
from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI
|
from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI
|
||||||
from vllm_omni.errors import OmniClientError
|
from vllm_omni.errors import OmniClientError
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
||||||
|
from vllm_omni.utils.forced_aligner import build_forced_aligner_config
|
||||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace
|
from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingNamespace
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
@@ -1058,7 +1059,11 @@ async def omni_init_app_state(
|
|||||||
)
|
)
|
||||||
|
|
||||||
state.openai_serving_speech = OmniOpenAIServingSpeech(
|
state.openai_serving_speech = OmniOpenAIServingSpeech(
|
||||||
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
|
engine_client,
|
||||||
|
state.openai_serving_models,
|
||||||
|
request_logger=request_logger,
|
||||||
|
model_name=model_name,
|
||||||
|
forced_aligner_config=build_forced_aligner_config(args),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Warm up speech pipeline (CUDA Graph capture, torch.compile) so the first
|
# Warm up speech pipeline (CUDA Graph capture, torch.compile) so the first
|
||||||
|
|||||||
@@ -145,6 +145,15 @@ class OpenAICreateSpeechRequest(BaseModel):
|
|||||||
default=None,
|
default=None,
|
||||||
description=("Optional model-specific parameters passed directly to the model's extra_args."),
|
description=("Optional model-specific parameters passed directly to the model's extra_args."),
|
||||||
)
|
)
|
||||||
|
word_timestamps: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"When true, the server runs a shared forced aligner alongside the streamed "
|
||||||
|
"audio and emits per-chunk word timestamps. Requires the server to be "
|
||||||
|
"launched with --forced-aligner pointing at an aligner model. No effect "
|
||||||
|
"when streaming is off."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
@field_validator("stream_format")
|
@field_validator("stream_format")
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -463,6 +472,15 @@ class StreamingSpeechSessionConfig(BaseModel):
|
|||||||
"'clause' also splits on CJK commas , and semicolons ;."
|
"'clause' also splits on CJK commas , and semicolons ;."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
word_timestamps: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description=(
|
||||||
|
"When true, audio chunks are wrapped in JSON 'audio.chunk' frames carrying "
|
||||||
|
"base64-encoded PCM plus aligned word timestamps. Requires the server to be "
|
||||||
|
"launched with --forced-aligner. When false, audio is sent as raw binary "
|
||||||
|
"frames (existing behavior)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def validate_streaming_constraints(self) -> "StreamingSpeechSessionConfig":
|
def validate_streaming_constraints(self) -> "StreamingSpeechSessionConfig":
|
||||||
|
|||||||
@@ -411,6 +411,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.model_name = kwargs.pop("model_name", None)
|
self.model_name = kwargs.pop("model_name", None)
|
||||||
|
self.forced_aligner_config: Any | None = kwargs.pop("forced_aligner_config", None)
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self._init_speaker_storage()
|
self._init_speaker_storage()
|
||||||
|
|
||||||
@@ -2586,6 +2587,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
response_format: str = "pcm",
|
response_format: str = "pcm",
|
||||||
raw_request: Request | None = None,
|
raw_request: Request | None = None,
|
||||||
request_start_s: float | None = None,
|
request_start_s: float | None = None,
|
||||||
|
include_sample_rate: bool = False,
|
||||||
):
|
):
|
||||||
"""Generate audio chunks for streaming response.
|
"""Generate audio chunks for streaming response.
|
||||||
|
|
||||||
@@ -2662,7 +2664,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
)
|
)
|
||||||
if first_audio_chunk_s is None:
|
if first_audio_chunk_s is None:
|
||||||
first_audio_chunk_s = time.perf_counter()
|
first_audio_chunk_s = time.perf_counter()
|
||||||
yield self.create_audio(audio_obj).audio_data
|
audio_bytes = self.create_audio(audio_obj).audio_data
|
||||||
|
if include_sample_rate:
|
||||||
|
yield audio_bytes, sample_rate_val
|
||||||
|
else:
|
||||||
|
yield audio_bytes
|
||||||
self._mark_ref_audio_artifact_ready_for_request(request_id)
|
self._mark_ref_audio_artifact_ready_for_request(request_id)
|
||||||
artifact_ready = True
|
artifact_ready = True
|
||||||
total_ms = (time.perf_counter() - stream_start_s) * 1000.0
|
total_ms = (time.perf_counter() - stream_start_s) * 1000.0
|
||||||
@@ -3508,13 +3514,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
self._track_ref_audio_artifact_warmup(request_id, qwen3_ref_audio_warmup_artifact_key)
|
self._track_ref_audio_artifact_warmup(request_id, qwen3_ref_audio_warmup_artifact_key)
|
||||||
return request_id, generator, tts_params
|
return request_id, generator, tts_params
|
||||||
|
|
||||||
async def _generate_pcm_chunks(self, generator, request_id: str):
|
async def _generate_pcm_chunks(self, generator, request_id: str, *, include_sample_rate: bool = False):
|
||||||
"""Yield raw PCM byte chunks from the engine generator.
|
"""Yield raw PCM byte chunks from the engine generator.
|
||||||
|
|
||||||
Delegates to ``_generate_audio_chunks`` with ``response_format="pcm"``.
|
Delegates to ``_generate_audio_chunks`` with ``response_format="pcm"``.
|
||||||
Used by the WebSocket streaming handler and ``_iter_pcm_audio_bytes``.
|
Used by the WebSocket streaming handler and ``_iter_pcm_audio_bytes``.
|
||||||
"""
|
"""
|
||||||
async for chunk in self._generate_audio_chunks(generator, request_id, response_format="pcm"):
|
async for chunk in self._generate_audio_chunks(
|
||||||
|
generator,
|
||||||
|
request_id,
|
||||||
|
response_format="pcm",
|
||||||
|
include_sample_rate=include_sample_rate,
|
||||||
|
):
|
||||||
yield chunk
|
yield chunk
|
||||||
|
|
||||||
async def _iter_pcm_audio_bytes(self, request: OpenAICreateSpeechRequest):
|
async def _iter_pcm_audio_bytes(self, request: OpenAICreateSpeechRequest):
|
||||||
@@ -3810,6 +3821,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
if request.stream:
|
if request.stream:
|
||||||
|
if request.word_timestamps:
|
||||||
|
return self.create_error_response(
|
||||||
|
"word_timestamps=true is currently supported by the WebSocket "
|
||||||
|
"/v1/audio/speech/stream path. Use session.config with "
|
||||||
|
"stream_audio=true and response_format='pcm'."
|
||||||
|
)
|
||||||
|
|
||||||
# Determine response format and media type for streaming
|
# Determine response format and media type for streaming
|
||||||
response_format = (request.response_format or "wav").lower()
|
response_format = (request.response_format or "wav").lower()
|
||||||
|
|
||||||
|
|||||||
@@ -9,15 +9,26 @@ Protocol:
|
|||||||
{"type": "input.text", "text": "..."} # Text chunks
|
{"type": "input.text", "text": "..."} # Text chunks
|
||||||
{"type": "input.done"} # End of input
|
{"type": "input.done"} # End of input
|
||||||
|
|
||||||
Server -> Client:
|
Server -> Client (default, word_timestamps=false):
|
||||||
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "wav"}
|
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "wav"}
|
||||||
<binary frame: audio bytes>
|
<binary frame: audio bytes>
|
||||||
|
...
|
||||||
{"type": "audio.done", "sentence_index": 0}
|
{"type": "audio.done", "sentence_index": 0}
|
||||||
{"type": "session.done", "total_sentences": N}
|
{"type": "session.done", "total_sentences": N}
|
||||||
{"type": "error", "message": "..."}
|
{"type": "error", "message": "..."}
|
||||||
|
|
||||||
|
Server -> Client (when word_timestamps=true):
|
||||||
|
{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm"}
|
||||||
|
{"type": "audio.chunk", "sentence_index": 0, "chunk_id": 0, "audio_b64": "<base64 PCM>", "timestamps": null}
|
||||||
|
...
|
||||||
|
{"type": "audio.chunk", "audio_b64": "", "timestamps": [{"word", "start_ms", "end_ms"}, ...]}
|
||||||
|
{"type": "audio.done", "sentence_index": 0}
|
||||||
|
# Audio is JSON base64 PCM (not binary). A trailing empty-audio chunk carries the
|
||||||
|
# full sentence-relative alignment. timestamps: list = aligned, [] = silence, null = failed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import base64
|
||||||
import json
|
import json
|
||||||
from contextlib import aclosing
|
from contextlib import aclosing
|
||||||
|
|
||||||
@@ -35,12 +46,15 @@ from vllm_omni.entrypoints.openai.text_splitter import (
|
|||||||
SPLIT_SENTENCE,
|
SPLIT_SENTENCE,
|
||||||
SentenceSplitter,
|
SentenceSplitter,
|
||||||
)
|
)
|
||||||
|
from vllm_omni.utils.forced_aligner import ForcedAlignerLoadError
|
||||||
|
from vllm_omni.utils.forced_aligner import align as forced_align
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
_DEFAULT_IDLE_TIMEOUT = 30.0 # seconds
|
_DEFAULT_IDLE_TIMEOUT = 30.0 # seconds
|
||||||
_DEFAULT_CONFIG_TIMEOUT = 10.0 # seconds
|
_DEFAULT_CONFIG_TIMEOUT = 10.0 # seconds
|
||||||
_PCM_SAMPLE_RATE = 24000
|
_PCM_SAMPLE_RATE = 24000
|
||||||
|
_BYTES_PER_SAMPLE = 2 # 16-bit mono PCM
|
||||||
_MAX_CONFIG_MESSAGE_SIZE = 4 * 1024 * 1024 # allow large ref_audio payloads
|
_MAX_CONFIG_MESSAGE_SIZE = 4 * 1024 * 1024 # allow large ref_audio payloads
|
||||||
_MAX_INPUT_TEXT_MESSAGE_SIZE = 128 * 1024
|
_MAX_INPUT_TEXT_MESSAGE_SIZE = 128 * 1024
|
||||||
|
|
||||||
@@ -210,6 +224,24 @@ class OmniStreamingSpeechHandler:
|
|||||||
"""Generate audio for a single sentence and send it over WebSocket."""
|
"""Generate audio for a single sentence and send it over WebSocket."""
|
||||||
response_format = config.response_format or "wav"
|
response_format = config.response_format or "wav"
|
||||||
|
|
||||||
|
# Reject unmet word-timestamps preconditions early with a clear reason.
|
||||||
|
if config.word_timestamps:
|
||||||
|
if self._speech_service.forced_aligner_config is None:
|
||||||
|
await self._send_error(
|
||||||
|
websocket,
|
||||||
|
"word_timestamps=true but the server was launched without "
|
||||||
|
"--forced-aligner; either restart the server with that flag "
|
||||||
|
"or set word_timestamps=false in session.config.",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if not (config.stream_audio and response_format == "pcm"):
|
||||||
|
await self._send_error(
|
||||||
|
websocket,
|
||||||
|
"word_timestamps=true requires stream_audio=true and "
|
||||||
|
"response_format='pcm' (the aligner consumes raw PCM).",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
request = OpenAICreateSpeechRequest(
|
request = OpenAICreateSpeechRequest(
|
||||||
input=sentence_text,
|
input=sentence_text,
|
||||||
model=config.model,
|
model=config.model,
|
||||||
@@ -227,6 +259,7 @@ class OmniStreamingSpeechHandler:
|
|||||||
x_vector_only_mode=config.x_vector_only_mode,
|
x_vector_only_mode=config.x_vector_only_mode,
|
||||||
speaker_embedding=config.speaker_embedding,
|
speaker_embedding=config.speaker_embedding,
|
||||||
stream=config.stream_audio,
|
stream=config.stream_audio,
|
||||||
|
word_timestamps=config.word_timestamps,
|
||||||
)
|
)
|
||||||
|
|
||||||
start_payload = {
|
start_payload = {
|
||||||
@@ -236,7 +269,11 @@ class OmniStreamingSpeechHandler:
|
|||||||
"format": response_format,
|
"format": response_format,
|
||||||
}
|
}
|
||||||
if config.stream_audio and response_format == "pcm":
|
if config.stream_audio and response_format == "pcm":
|
||||||
|
# Nominal stream rate; each audio.chunk carries the authoritative
|
||||||
|
# per-chunk sample_rate.
|
||||||
start_payload["sample_rate"] = _PCM_SAMPLE_RATE
|
start_payload["sample_rate"] = _PCM_SAMPLE_RATE
|
||||||
|
if config.word_timestamps:
|
||||||
|
start_payload["word_timestamps"] = True
|
||||||
await websocket.send_json(start_payload)
|
await websocket.send_json(start_payload)
|
||||||
|
|
||||||
total_bytes = 0
|
total_bytes = 0
|
||||||
@@ -245,10 +282,20 @@ class OmniStreamingSpeechHandler:
|
|||||||
try:
|
try:
|
||||||
if config.stream_audio:
|
if config.stream_audio:
|
||||||
request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)
|
request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)
|
||||||
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
|
if config.word_timestamps:
|
||||||
async for chunk in stream:
|
total_bytes = await self._stream_audio_with_alignments(
|
||||||
total_bytes += len(chunk)
|
websocket=websocket,
|
||||||
await websocket.send_bytes(chunk)
|
request_id=request_id,
|
||||||
|
generator=generator,
|
||||||
|
sentence_text=sentence_text,
|
||||||
|
sentence_index=sentence_index,
|
||||||
|
language=config.language,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
async with aclosing(self._speech_service._generate_pcm_chunks(generator, request_id)) as stream:
|
||||||
|
async for chunk in stream:
|
||||||
|
total_bytes += len(chunk)
|
||||||
|
await websocket.send_bytes(chunk)
|
||||||
else:
|
else:
|
||||||
audio_bytes, _ = await self._speech_service._generate_audio_bytes(request)
|
audio_bytes, _ = await self._speech_service._generate_audio_bytes(request)
|
||||||
total_bytes = len(audio_bytes)
|
total_bytes = len(audio_bytes)
|
||||||
@@ -277,6 +324,115 @@ class OmniStreamingSpeechHandler:
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.debug("Failed to send audio.done for sentence %d", sentence_index, exc_info=True)
|
logger.debug("Failed to send audio.done for sentence %d", sentence_index, exc_info=True)
|
||||||
|
|
||||||
|
async def _stream_audio_with_alignments(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
websocket: WebSocket,
|
||||||
|
request_id: str,
|
||||||
|
generator,
|
||||||
|
sentence_text: str,
|
||||||
|
sentence_index: int,
|
||||||
|
language: str | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Stream PCM as JSON ``audio.chunk`` frames, aligned per sentence.
|
||||||
|
|
||||||
|
Forward each PCM chunk live (``timestamps: null``) while buffering the
|
||||||
|
sentence audio, then run the forced aligner once over the whole
|
||||||
|
sentence and emit a final empty-audio ``audio.chunk`` with the word
|
||||||
|
timestamps. On aligner failure timestamps is ``null``; for silence it
|
||||||
|
is ``[]`` (audio always flows regardless).
|
||||||
|
"""
|
||||||
|
aligner_config = self._speech_service.forced_aligner_config
|
||||||
|
assert aligner_config is not None # gated by the precondition check
|
||||||
|
|
||||||
|
sentence_audio = bytearray()
|
||||||
|
total_bytes = 0
|
||||||
|
sample_rate = _PCM_SAMPLE_RATE
|
||||||
|
chunk_id = 0
|
||||||
|
|
||||||
|
async def send_chunk(
|
||||||
|
chunk: bytes,
|
||||||
|
chunk_sample_rate: int,
|
||||||
|
timestamps_payload: list[dict] | None,
|
||||||
|
chunk_start_ms: int,
|
||||||
|
chunk_end_ms: int,
|
||||||
|
) -> None:
|
||||||
|
nonlocal chunk_id
|
||||||
|
await websocket.send_json(
|
||||||
|
{
|
||||||
|
"type": "audio.chunk",
|
||||||
|
"sentence_index": sentence_index,
|
||||||
|
"chunk_id": chunk_id,
|
||||||
|
"chunk_start_ms": chunk_start_ms,
|
||||||
|
"chunk_end_ms": chunk_end_ms,
|
||||||
|
"sample_rate": chunk_sample_rate,
|
||||||
|
"audio_b64": base64.b64encode(chunk).decode("ascii"),
|
||||||
|
"timestamps": timestamps_payload,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
chunk_id += 1
|
||||||
|
|
||||||
|
async with aclosing(
|
||||||
|
self._speech_service._generate_pcm_chunks(
|
||||||
|
generator,
|
||||||
|
request_id,
|
||||||
|
include_sample_rate=True,
|
||||||
|
)
|
||||||
|
) as stream:
|
||||||
|
async for chunk, chunk_sample_rate in stream:
|
||||||
|
sample_rate = chunk_sample_rate
|
||||||
|
chunk_start_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||||
|
sentence_audio.extend(chunk)
|
||||||
|
chunk_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||||
|
total_bytes += len(chunk)
|
||||||
|
# Audio first, timestamps after the whole sentence is aligned.
|
||||||
|
await send_chunk(chunk, chunk_sample_rate, None, chunk_start_ms, chunk_end_ms)
|
||||||
|
|
||||||
|
# Single alignment pass over the full sentence, then emit timestamps.
|
||||||
|
# A load/config failure is permanent, so surface the reason once; audio
|
||||||
|
# has already streamed, so the trailing frame still carries null.
|
||||||
|
try:
|
||||||
|
timestamps_payload = await self._align_sentence(
|
||||||
|
audio=bytes(sentence_audio),
|
||||||
|
text=sentence_text,
|
||||||
|
sample_rate=sample_rate,
|
||||||
|
config=aligner_config,
|
||||||
|
language=language,
|
||||||
|
)
|
||||||
|
except ForcedAlignerLoadError as exc:
|
||||||
|
await self._send_error(websocket, f"forced aligner unavailable: {exc}")
|
||||||
|
timestamps_payload = None
|
||||||
|
sentence_end_ms = int(round((len(sentence_audio) / _BYTES_PER_SAMPLE / sample_rate) * 1000.0))
|
||||||
|
await send_chunk(b"", sample_rate, timestamps_payload, 0, sentence_end_ms)
|
||||||
|
|
||||||
|
return total_bytes
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _align_sentence(
|
||||||
|
*,
|
||||||
|
audio: bytes,
|
||||||
|
text: str,
|
||||||
|
sample_rate: int,
|
||||||
|
config,
|
||||||
|
language: str | None = None,
|
||||||
|
) -> list[dict] | None:
|
||||||
|
"""Convert a sentence alignment into JSON word-timestamp dicts.
|
||||||
|
|
||||||
|
Returns ``None`` on aligner failure, ``[]`` when it ran but produced no
|
||||||
|
tokens. Monotonic, non-overlapping bounds are guaranteed by the decoder.
|
||||||
|
``language`` is forwarded to word segmentation.
|
||||||
|
"""
|
||||||
|
aligned = await forced_align(
|
||||||
|
audio=audio,
|
||||||
|
text=text,
|
||||||
|
sample_rate=sample_rate,
|
||||||
|
config=config,
|
||||||
|
language=language,
|
||||||
|
)
|
||||||
|
if aligned is None:
|
||||||
|
return None
|
||||||
|
return [{"word": ts.word, "start_ms": ts.start_ms, "end_ms": ts.end_ms} for ts in aligned]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def _send_error(websocket: WebSocket, message: str) -> None:
|
async def _send_error(websocket: WebSocket, message: str) -> None:
|
||||||
"""Send an error message to the client."""
|
"""Send an error message to the client."""
|
||||||
|
|||||||
@@ -0,0 +1,399 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Shared forced aligner for streaming TTS word timestamps.
|
||||||
|
|
||||||
|
Hosts one lazy-loaded, in-process pooling ``vllm.LLM`` shared by the TTS
|
||||||
|
frontend. ``llm.encode`` is sync/blocking, so :func:`align` runs it in
|
||||||
|
``asyncio.to_thread``, serialized on a lock since the offline LLM is not
|
||||||
|
thread-safe. See :func:`align` for the return-value contract.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from vllm_omni.utils import qwen3_force_align_processor as _processor
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
_DEFAULT_CONFIG_PATH = Path(__file__).resolve().parents[1] / "deploy" / "qwen3_tts_forced_aligner.yaml"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class WordTimestamp:
|
||||||
|
"""Internal alignment record. Serialized to a plain JSON object
|
||||||
|
(``{"word", "start_ms", "end_ms"}``) at the WebSocket boundary.
|
||||||
|
"""
|
||||||
|
|
||||||
|
word: str
|
||||||
|
start_ms: int
|
||||||
|
end_ms: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ForcedAlignerConfig:
|
||||||
|
"""Plain-data config for constructing the aligner ``vllm.LLM`` (lazy-loaded)."""
|
||||||
|
|
||||||
|
model: str
|
||||||
|
runner: str | None = None
|
||||||
|
architecture: str | None = None
|
||||||
|
pooling_task: str | None = None
|
||||||
|
gpu_memory_utilization: float | None = None
|
||||||
|
dtype: str | None = None
|
||||||
|
max_model_len: int | None = None
|
||||||
|
trust_remote_code: bool | None = None
|
||||||
|
extra_llm_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_forced_aligner_yaml(path: str | Path) -> dict[str, Any]:
|
||||||
|
with Path(path).open(encoding="utf-8") as f:
|
||||||
|
raw = yaml.safe_load(f) or {}
|
||||||
|
return dict(raw.get("forced_aligner", raw))
|
||||||
|
|
||||||
|
|
||||||
|
def build_forced_aligner_config(args: Any) -> ForcedAlignerConfig | None:
|
||||||
|
"""Build a config from CLI args, or ``None`` when the feature is off.
|
||||||
|
|
||||||
|
Precedence (lowest to highest): the packaged default YAML
|
||||||
|
(:data:`_DEFAULT_CONFIG_PATH`, Qwen deploy defaults) -> a user YAML passed
|
||||||
|
via ``--forced-aligner-config`` -> the ``--forced-aligner`` model path. The
|
||||||
|
feature is off (returns ``None``) unless a model resolves from this chain.
|
||||||
|
Per-field overrides such as ``gpu_memory_utilization`` live in the YAML.
|
||||||
|
"""
|
||||||
|
config_path = getattr(args, "forced_aligner_config", None)
|
||||||
|
config_data: dict[str, Any] = {}
|
||||||
|
model = getattr(args, "forced_aligner", None)
|
||||||
|
if config_path or model:
|
||||||
|
config_data.update(_load_forced_aligner_yaml(_DEFAULT_CONFIG_PATH))
|
||||||
|
if config_path:
|
||||||
|
config_data.update(_load_forced_aligner_yaml(config_path))
|
||||||
|
if model:
|
||||||
|
config_data["model"] = str(model)
|
||||||
|
model = config_data.get("model")
|
||||||
|
if not model:
|
||||||
|
return None
|
||||||
|
allowed = set(ForcedAlignerConfig.__dataclass_fields__)
|
||||||
|
return ForcedAlignerConfig(**{k: v for k, v in config_data.items() if k in allowed})
|
||||||
|
|
||||||
|
|
||||||
|
class ForcedAlignerLoadError(RuntimeError):
|
||||||
|
"""The aligner model/config could not be loaded (permanent until restart).
|
||||||
|
|
||||||
|
Distinguished from a per-request alignment failure so the streaming layer
|
||||||
|
can report a clear reason once rather than degrading every request to
|
||||||
|
``timestamps: null``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# --- Singleton state ---
|
||||||
|
# A single LLM serves the whole API server. ``_lock`` guards the lazy
|
||||||
|
# constructor; ``_encode_lock`` serializes ``encode`` because the offline
|
||||||
|
# ``vllm.LLM`` is a sync batch interface that is not safe to call from multiple
|
||||||
|
# threads at once (concurrent WebSocket sessions otherwise race the engine).
|
||||||
|
_lock = threading.Lock()
|
||||||
|
_encode_lock = asyncio.Lock()
|
||||||
|
_llm: Any = None
|
||||||
|
_classify_num: int | None = None
|
||||||
|
_timestamp_token_id: int | None = None
|
||||||
|
_timestamp_segment_time_ms: float | None = None
|
||||||
|
_loaded_config: ForcedAlignerConfig | None = None
|
||||||
|
|
||||||
|
|
||||||
|
async def align(
|
||||||
|
*,
|
||||||
|
audio: bytes,
|
||||||
|
text: str,
|
||||||
|
sample_rate: int,
|
||||||
|
config: ForcedAlignerConfig,
|
||||||
|
language: str | None = None,
|
||||||
|
) -> list[WordTimestamp] | None:
|
||||||
|
"""Run one forced-alignment pass.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Signed-int16 little-endian mono PCM bytes.
|
||||||
|
text: Ground-truth text whose words to align.
|
||||||
|
sample_rate: Sample rate of ``audio`` in Hz.
|
||||||
|
config: Aligner config (same instance for every call across the
|
||||||
|
server's lifetime; reload requires a server restart).
|
||||||
|
language: Optional language hint for word segmentation. ``None`` /
|
||||||
|
``"auto"`` use the space + Chinese-mixed path; ``"japanese"`` /
|
||||||
|
``"korean"`` (or their codes) need ``qwen_asr`` installed for
|
||||||
|
faithful tokenisation, else they degrade to whitespace splitting.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of :class:`WordTimestamp` on success (possibly empty for
|
||||||
|
silence / no aligned tokens), ``None`` on a per-request alignment
|
||||||
|
failure (decode error).
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ForcedAlignerLoadError: the model/config could not be loaded. This is
|
||||||
|
permanent until restart, so callers surface it once instead of
|
||||||
|
degrading every request to ``None``.
|
||||||
|
"""
|
||||||
|
async with _encode_lock:
|
||||||
|
try:
|
||||||
|
await asyncio.to_thread(_ensure_loaded, config)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
logger.exception("Forced aligner failed to load")
|
||||||
|
raise ForcedAlignerLoadError(str(exc)) from exc
|
||||||
|
try:
|
||||||
|
return await asyncio.to_thread(_align_sync, audio, text, sample_rate, config, language)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
logger.exception("Forced aligner failed for text=%r", text)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _align_sync(
|
||||||
|
audio: bytes,
|
||||||
|
text: str,
|
||||||
|
sample_rate: int,
|
||||||
|
config: ForcedAlignerConfig,
|
||||||
|
language: str | None = None,
|
||||||
|
) -> list[WordTimestamp]:
|
||||||
|
# Precondition: align() has already run _ensure_loaded(config) under
|
||||||
|
# _encode_lock and owns the load-failure -> ForcedAlignerLoadError mapping.
|
||||||
|
# Don't reload here, so a load error can't be swallowed as a per-request None.
|
||||||
|
audio_arr = _pcm_bytes_to_float32(audio)
|
||||||
|
if audio_arr.size == 0:
|
||||||
|
return []
|
||||||
|
audio_duration_ms = (audio_arr.size / sample_rate) * 1000.0
|
||||||
|
|
||||||
|
# Segment once and reuse for both the prompt and the decode: the word
|
||||||
|
# units MUST match between the two or the markers drift out of sync.
|
||||||
|
words = _processor.segment_words(text, language)
|
||||||
|
prompt = _processor.build_prompt(words)
|
||||||
|
request = {
|
||||||
|
"prompt": prompt,
|
||||||
|
"multi_modal_data": {"audio": (audio_arr, sample_rate)},
|
||||||
|
}
|
||||||
|
|
||||||
|
from vllm.pooling_params import PoolingParams
|
||||||
|
|
||||||
|
outputs = _llm.encode( # type: ignore[union-attr]
|
||||||
|
[request],
|
||||||
|
pooling_params=PoolingParams(),
|
||||||
|
pooling_task=config.pooling_task or "token_classify",
|
||||||
|
use_tqdm=False,
|
||||||
|
)
|
||||||
|
if not outputs:
|
||||||
|
return []
|
||||||
|
|
||||||
|
result = outputs[0]
|
||||||
|
logits = result.outputs.data # [n_token, classify_num]
|
||||||
|
prompt_token_ids = list(result.prompt_token_ids)
|
||||||
|
timestamp_positions = [i for i, tid in enumerate(prompt_token_ids) if tid == _timestamp_token_id]
|
||||||
|
if not timestamp_positions:
|
||||||
|
logger.warning(
|
||||||
|
"No <|timestamp|> tokens found in prompt for text=%r; aligner returned %d rows.",
|
||||||
|
text,
|
||||||
|
logits.shape[0] if hasattr(logits, "shape") else len(logits),
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
return _decode_timestamps(
|
||||||
|
logits=logits,
|
||||||
|
words=words,
|
||||||
|
timestamp_positions=timestamp_positions,
|
||||||
|
classify_num=_classify_num,
|
||||||
|
timestamp_segment_time_ms=_timestamp_segment_time_ms,
|
||||||
|
audio_duration_ms=audio_duration_ms,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_loaded(config: ForcedAlignerConfig) -> None:
|
||||||
|
"""Lazy-load the singleton ``vllm.LLM`` under lock; idempotent."""
|
||||||
|
global _llm, _classify_num, _timestamp_token_id, _timestamp_segment_time_ms, _loaded_config
|
||||||
|
|
||||||
|
if _llm is not None:
|
||||||
|
if _loaded_config is not None and _loaded_config.model != config.model:
|
||||||
|
# Multiple configs from different requests — refuse rather
|
||||||
|
# than swap models silently. A server restart is required.
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Forced aligner already loaded with config={_loaded_config!r}; "
|
||||||
|
f"cannot serve a request that asks for {config!r}. "
|
||||||
|
"Restart the server to change the aligner config."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
with _lock:
|
||||||
|
if _llm is not None:
|
||||||
|
return # raced; another caller did the load
|
||||||
|
|
||||||
|
# Lazy import: vllm pulls torch + CUDA, which we want to avoid at
|
||||||
|
# module import time.
|
||||||
|
from vllm import LLM
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Loading forced aligner %s (gpu_memory_utilization=%s)",
|
||||||
|
config.model,
|
||||||
|
config.gpu_memory_utilization if config.gpu_memory_utilization is not None else "default",
|
||||||
|
)
|
||||||
|
llm_kwargs: dict[str, Any] = {"model": config.model}
|
||||||
|
if config.runner is not None:
|
||||||
|
llm_kwargs["runner"] = config.runner
|
||||||
|
if config.architecture is not None:
|
||||||
|
llm_kwargs["hf_overrides"] = {"architectures": [config.architecture]}
|
||||||
|
if config.gpu_memory_utilization is not None:
|
||||||
|
llm_kwargs["gpu_memory_utilization"] = config.gpu_memory_utilization
|
||||||
|
if config.trust_remote_code is not None:
|
||||||
|
llm_kwargs["trust_remote_code"] = config.trust_remote_code
|
||||||
|
if config.dtype is not None:
|
||||||
|
llm_kwargs["dtype"] = config.dtype
|
||||||
|
if config.max_model_len is not None:
|
||||||
|
llm_kwargs["max_model_len"] = config.max_model_len
|
||||||
|
llm_kwargs.update(config.extra_llm_kwargs)
|
||||||
|
|
||||||
|
# Force the aligner engine in-process: in multiprocessing mode the
|
||||||
|
# token_classify ``pooling_output`` fails to round-trip over the
|
||||||
|
# engine-core IPC on current vLLM. Hard-set (not setdefault) so a
|
||||||
|
# globally exported ``=1`` can't push it into a subprocess; save/restore
|
||||||
|
# to avoid leaking the override to anything built later in this process.
|
||||||
|
prev_mp = os.environ.get("VLLM_ENABLE_V1_MULTIPROCESSING")
|
||||||
|
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||||
|
try:
|
||||||
|
llm = LLM(**llm_kwargs)
|
||||||
|
finally:
|
||||||
|
if prev_mp is None:
|
||||||
|
os.environ.pop("VLLM_ENABLE_V1_MULTIPROCESSING", None)
|
||||||
|
else:
|
||||||
|
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = prev_mp
|
||||||
|
|
||||||
|
thinker_config = getattr(llm.llm_engine.model_config.hf_config, "thinker_config", None)
|
||||||
|
if thinker_config is None or not hasattr(thinker_config, "classify_num"):
|
||||||
|
raise RuntimeError(
|
||||||
|
"Loaded aligner has no thinker_config.classify_num; "
|
||||||
|
"expected a Qwen3ASRForcedAlignerForTokenClassification checkpoint."
|
||||||
|
)
|
||||||
|
timestamp_segment_time_ms = getattr(llm.llm_engine.model_config.hf_config, "timestamp_segment_time", None)
|
||||||
|
if timestamp_segment_time_ms is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Loaded aligner has no timestamp_segment_time; expected a Qwen3ASR forced aligner checkpoint."
|
||||||
|
)
|
||||||
|
|
||||||
|
tokenizer = llm.get_tokenizer()
|
||||||
|
timestamp_token_id = _processor.resolve_timestamp_token_id(tokenizer)
|
||||||
|
|
||||||
|
# Publish in this order so a concurrent reader either sees
|
||||||
|
# _llm == None (will block on the lock) or sees a fully
|
||||||
|
# initialized aligner.
|
||||||
|
_classify_num = int(thinker_config.classify_num)
|
||||||
|
_timestamp_token_id = timestamp_token_id
|
||||||
|
_timestamp_segment_time_ms = float(timestamp_segment_time_ms)
|
||||||
|
_loaded_config = config
|
||||||
|
_llm = llm
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Forced aligner ready: timestamp_token_id=%d, classify_num=%d, timestamp_segment_time_ms=%.1f",
|
||||||
|
timestamp_token_id,
|
||||||
|
_classify_num,
|
||||||
|
_timestamp_segment_time_ms,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- pure helpers (testable without a GPU / vllm) ---
|
||||||
|
#
|
||||||
|
# Word segmentation, prompt building, timestamp repair and marker-token
|
||||||
|
# lookup are Qwen-specific and live in
|
||||||
|
# :mod:`vllm_omni.utils.qwen3_force_align_processor`. This module owns only
|
||||||
|
# the generic vLLM orchestration.
|
||||||
|
|
||||||
|
|
||||||
|
def _pcm_bytes_to_float32(audio: bytes) -> np.ndarray:
|
||||||
|
"""Decode signed-int16 mono PCM bytes into a [-1, 1] float32 array."""
|
||||||
|
if not audio:
|
||||||
|
return np.zeros(0, dtype=np.float32)
|
||||||
|
if len(audio) % 2 != 0:
|
||||||
|
# Drop a trailing odd byte rather than raise; keeps streaming
|
||||||
|
# robust against off-by-one chunk boundaries.
|
||||||
|
audio = audio[:-1]
|
||||||
|
pcm = np.frombuffer(audio, dtype=np.int16)
|
||||||
|
return (pcm.astype(np.float32) / 32768.0).copy()
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_timestamps(
|
||||||
|
*,
|
||||||
|
logits: Any,
|
||||||
|
words: list[str],
|
||||||
|
timestamp_positions: list[int],
|
||||||
|
classify_num: int,
|
||||||
|
audio_duration_ms: float,
|
||||||
|
timestamp_segment_time_ms: float | None = None,
|
||||||
|
) -> list[WordTimestamp]:
|
||||||
|
"""Translate ``[n_token, classify_num]`` logits into word timestamps.
|
||||||
|
|
||||||
|
``words`` must be the exact segmentation used to build the prompt (see
|
||||||
|
:func:`qwen3_force_align_processor.segment_words`); each word owns two
|
||||||
|
consecutive markers.
|
||||||
|
"""
|
||||||
|
arr = logits.detach().cpu().numpy() if hasattr(logits, "detach") else np.asarray(logits)
|
||||||
|
if arr.ndim != 2:
|
||||||
|
raise ValueError(f"Expected 2D logits [n_token, classify_num]; got shape {arr.shape}")
|
||||||
|
if arr.shape[1] != classify_num:
|
||||||
|
raise ValueError(
|
||||||
|
f"Logits last dim {arr.shape[1]} != classify_num {classify_num}; "
|
||||||
|
"model config and prompt template may be out of sync."
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = len(words) * 2
|
||||||
|
if len(timestamp_positions) != expected:
|
||||||
|
logger.warning(
|
||||||
|
"Got %d timestamp positions but text has %d words (expected %d start/end markers); "
|
||||||
|
"returning empty alignment.",
|
||||||
|
len(timestamp_positions),
|
||||||
|
len(words),
|
||||||
|
expected,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
marker_logits = arr[timestamp_positions, :]
|
||||||
|
bin_idx = marker_logits.argmax(axis=-1)
|
||||||
|
bin_size_ms = (
|
||||||
|
float(timestamp_segment_time_ms)
|
||||||
|
if timestamp_segment_time_ms is not None
|
||||||
|
else (audio_duration_ms / classify_num if classify_num > 0 else 0.0)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Repair non-monotonic bins across the whole start/end sequence before
|
||||||
|
# pairing, so each (start, end) pair stays ordered and one bad bin can't
|
||||||
|
# corrupt its neighbours.
|
||||||
|
marker_ms = _processor.fix_timestamp([int(round(int(b) * bin_size_ms)) for b in bin_idx])
|
||||||
|
|
||||||
|
# Pair markers into per-word intervals, enforcing the wire contract here so
|
||||||
|
# callers don't have to: monotonic and non-overlapping across words, with
|
||||||
|
# start <= end and nothing past the audio length.
|
||||||
|
duration_ms = int(round(audio_duration_ms))
|
||||||
|
out: list[WordTimestamp] = []
|
||||||
|
prev_end_ms = 0
|
||||||
|
for i, word in enumerate(words):
|
||||||
|
start_ms = min(max(marker_ms[i * 2], prev_end_ms), duration_ms)
|
||||||
|
end_ms = min(max(marker_ms[i * 2 + 1], start_ms), duration_ms)
|
||||||
|
out.append(WordTimestamp(word=word, start_ms=start_ms, end_ms=end_ms))
|
||||||
|
prev_end_ms = end_ms
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# Test hooks ---------------------------------------------------------------
|
||||||
|
# Tests need a way to reset module state without restarting Python. Not
|
||||||
|
# part of the public API; do not call in production code.
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_for_tests() -> None:
|
||||||
|
"""Drop the cached aligner state so the next call reloads."""
|
||||||
|
global _llm, _classify_num, _timestamp_token_id, _timestamp_segment_time_ms, _loaded_config
|
||||||
|
with _lock:
|
||||||
|
_llm = None
|
||||||
|
_classify_num = None
|
||||||
|
_timestamp_token_id = None
|
||||||
|
_timestamp_segment_time_ms = None
|
||||||
|
_loaded_config = None
|
||||||
|
_processor._reset_for_tests()
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
# Adapted from:
|
||||||
|
# https://github.com/QwenLM/Qwen3-ASR/blob/main/qwen_asr/inference/qwen3_forced_aligner.py
|
||||||
|
#
|
||||||
|
# Copyright 2026 The Alibaba Qwen team. All rights reserved.
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
"""Qwen3 forced-aligner text/timestamp processor.
|
||||||
|
|
||||||
|
This is the model-specific half of upstream's ``Qwen3ForceAlignProcessor``:
|
||||||
|
it turns text into the aligner's word units and prompt, repairs the predicted
|
||||||
|
timestamp bins, and resolves the marker token id. The generic "run a vLLM
|
||||||
|
pooling model" half lives in :mod:`vllm_omni.utils.forced_aligner`, which
|
||||||
|
calls into this module.
|
||||||
|
|
||||||
|
Keeping the Qwen-specific pieces here marks the seam for the model-agnostic
|
||||||
|
aligner the issue asks for: a different aligner family would supply its own
|
||||||
|
processor exposing the same small surface — :func:`segment_words`,
|
||||||
|
:func:`build_prompt`, :func:`fix_timestamp`, :func:`resolve_timestamp_token_id`.
|
||||||
|
|
||||||
|
Word segmentation prefers qwen_asr's official ``Qwen3ForceAlignProcessor``
|
||||||
|
when installed (full multilingual fidelity, incl. Japanese/Korean) and
|
||||||
|
otherwise uses the faithful port below: exact for whitespace-delimited and
|
||||||
|
Chinese-mixed text; Japanese/Korean degrade to whitespace splitting.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import unicodedata
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# Prompt tokens for the Qwen3 forced aligner. Baked in (not config) because the
|
||||||
|
# prompt template and the word segmentation are inherently Qwen-specific: a
|
||||||
|
# different aligner family needs a different processor module regardless.
|
||||||
|
AUDIO_PLACEHOLDER = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||||
|
TIMESTAMP_TOKEN = "<timestamp>"
|
||||||
|
|
||||||
|
_CJK_RANGES = (
|
||||||
|
(0x4E00, 0x9FFF),
|
||||||
|
(0x3400, 0x4DBF),
|
||||||
|
(0x20000, 0x2A6DF),
|
||||||
|
(0x2A700, 0x2B73F),
|
||||||
|
(0x2B740, 0x2B81F),
|
||||||
|
(0x2B820, 0x2CEAF),
|
||||||
|
(0xF900, 0xFAFF),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Map common language codes to the names qwen_asr's processor expects.
|
||||||
|
_LANG_ALIASES = {
|
||||||
|
"ja": "japanese",
|
||||||
|
"jp": "japanese",
|
||||||
|
"jpn": "japanese",
|
||||||
|
"ko": "korean",
|
||||||
|
"kr": "korean",
|
||||||
|
"kor": "korean",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_prompt(words: list[str]) -> str:
|
||||||
|
"""Wrap segmented words in the Qwen3 aligner prompt template.
|
||||||
|
|
||||||
|
Each word gets two trailing ``<timestamp>`` markers (start + end); the
|
||||||
|
model classifies each marker into an audio time bin.
|
||||||
|
"""
|
||||||
|
if not words:
|
||||||
|
# Pad with one timestamp so the decoder always has something to
|
||||||
|
# read; an empty result still surfaces as "[]" upstream.
|
||||||
|
body = TIMESTAMP_TOKEN
|
||||||
|
else:
|
||||||
|
body = f"{TIMESTAMP_TOKEN}{TIMESTAMP_TOKEN}".join(words) + f"{TIMESTAMP_TOKEN}{TIMESTAMP_TOKEN}"
|
||||||
|
return f"<|im_start|>user\n{AUDIO_PLACEHOLDER}{body}<|im_end|>\n<|im_start|>assistant\n"
|
||||||
|
|
||||||
|
|
||||||
|
# --- word segmentation (port of Qwen3ForceAlignProcessor) ----------------
|
||||||
|
#
|
||||||
|
# The marker count fed to the model must match the segmentation the aligner
|
||||||
|
# was trained with, or the decode drifts and returns empty.
|
||||||
|
|
||||||
|
|
||||||
|
def _is_cjk_char(ch: str) -> bool:
|
||||||
|
code = ord(ch)
|
||||||
|
return any(lo <= code <= hi for lo, hi in _CJK_RANGES)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_kept_char(ch: str) -> bool:
|
||||||
|
"""Keep letters/numbers (any script) and the apostrophe; drop punctuation."""
|
||||||
|
if ch == "'":
|
||||||
|
return True
|
||||||
|
cat = unicodedata.category(ch)
|
||||||
|
return cat.startswith("L") or cat.startswith("N")
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_token(token: str) -> str:
|
||||||
|
return "".join(ch for ch in token if _is_kept_char(ch))
|
||||||
|
|
||||||
|
|
||||||
|
def _split_segment_with_chinese(seg: str) -> list[str]:
|
||||||
|
"""Split a cleaned segment so each CJK char is its own token."""
|
||||||
|
tokens: list[str] = []
|
||||||
|
buf: list[str] = []
|
||||||
|
|
||||||
|
def flush() -> None:
|
||||||
|
if buf:
|
||||||
|
tokens.append("".join(buf))
|
||||||
|
buf.clear()
|
||||||
|
|
||||||
|
for ch in seg:
|
||||||
|
if _is_cjk_char(ch):
|
||||||
|
flush()
|
||||||
|
tokens.append(ch)
|
||||||
|
else:
|
||||||
|
buf.append(ch)
|
||||||
|
flush()
|
||||||
|
return tokens
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenize_space_lang(text: str) -> list[str]:
|
||||||
|
"""Whitespace + Chinese-mixed segmentation (Qwen3ForceAlignProcessor port).
|
||||||
|
|
||||||
|
Splits on whitespace, strips punctuation from each segment (keeping
|
||||||
|
cross-script letters/digits and the apostrophe), then peels off CJK
|
||||||
|
characters as individual tokens.
|
||||||
|
"""
|
||||||
|
tokens: list[str] = []
|
||||||
|
for seg in text.split():
|
||||||
|
cleaned = _clean_token(seg)
|
||||||
|
if cleaned:
|
||||||
|
tokens.extend(_split_segment_with_chinese(cleaned))
|
||||||
|
return tokens
|
||||||
|
|
||||||
|
|
||||||
|
# Cached qwen_asr processor (or a sentinel that it is unavailable). Importing
|
||||||
|
# it eagerly pulls nagisa + the Qwen3ASR transformers backend, so we only try
|
||||||
|
# once, lazily, and remember the outcome.
|
||||||
|
_official_processor: Any = None
|
||||||
|
_official_processor_unavailable = False
|
||||||
|
|
||||||
|
|
||||||
|
def _get_official_processor() -> Any | None:
|
||||||
|
global _official_processor, _official_processor_unavailable
|
||||||
|
if _official_processor is not None:
|
||||||
|
return _official_processor
|
||||||
|
if _official_processor_unavailable:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from qwen_asr.inference.qwen3_forced_aligner import ( # type: ignore[import-not-found]
|
||||||
|
Qwen3ForceAlignProcessor,
|
||||||
|
)
|
||||||
|
|
||||||
|
_official_processor = Qwen3ForceAlignProcessor()
|
||||||
|
except Exception: # noqa: BLE001 - any import/init error means "use the port"
|
||||||
|
_official_processor_unavailable = True
|
||||||
|
logger.info(
|
||||||
|
"qwen_asr not available; using the built-in word segmentation "
|
||||||
|
"(exact for whitespace/Chinese-mixed text; Japanese/Korean fall "
|
||||||
|
"back to whitespace splitting). Install qwen-asr for full fidelity."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
return _official_processor
|
||||||
|
|
||||||
|
|
||||||
|
def segment_words(text: str, language: str | None = None) -> list[str]:
|
||||||
|
"""Split ``text`` into the aligner's word units.
|
||||||
|
|
||||||
|
Prefers qwen_asr's official processor (full multilingual fidelity); falls
|
||||||
|
back to the built-in port otherwise. ``language`` follows the official
|
||||||
|
naming — ``"japanese"`` / ``"korean"`` (or codes like ``ja`` / ``ko``)
|
||||||
|
trigger language-specific tokenisers; anything else (incl. ``None`` /
|
||||||
|
``"auto"``) uses the whitespace + Chinese-mixed path.
|
||||||
|
"""
|
||||||
|
lang = (language or "auto").strip().lower()
|
||||||
|
lang = _LANG_ALIASES.get(lang, lang)
|
||||||
|
processor = _get_official_processor()
|
||||||
|
if processor is not None:
|
||||||
|
try:
|
||||||
|
word_list, _ = processor.encode_timestamp(text, lang)
|
||||||
|
return list(word_list)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
logger.warning(
|
||||||
|
"Official Qwen3ForceAlignProcessor failed for language=%r; falling back to built-in segmentation.",
|
||||||
|
lang,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
if lang in ("japanese", "korean"):
|
||||||
|
logger.warning(
|
||||||
|
"language=%r needs qwen_asr (nagisa/soynlp) for faithful word "
|
||||||
|
"segmentation; falling back to whitespace splitting.",
|
||||||
|
lang,
|
||||||
|
)
|
||||||
|
return _tokenize_space_lang(text)
|
||||||
|
|
||||||
|
|
||||||
|
def fix_timestamp(values: list[int]) -> list[int]:
|
||||||
|
"""Repair non-monotonic timestamp bins via LIS + interpolation.
|
||||||
|
|
||||||
|
The model occasionally emits out-of-order bins; this snaps anomalies back
|
||||||
|
onto the longest non-decreasing subsequence (interpolating longer runs)
|
||||||
|
so paired start/end markers stay ordered. Port of
|
||||||
|
``Qwen3ForceAlignProcessor.fix_timestamp`` (Apache-2.0).
|
||||||
|
"""
|
||||||
|
n = len(values)
|
||||||
|
if n == 0:
|
||||||
|
return []
|
||||||
|
|
||||||
|
dp = [1] * n
|
||||||
|
parent = [-1] * n
|
||||||
|
for i in range(1, n):
|
||||||
|
for j in range(i):
|
||||||
|
if values[j] <= values[i] and dp[j] + 1 > dp[i]:
|
||||||
|
dp[i] = dp[j] + 1
|
||||||
|
parent[i] = j
|
||||||
|
|
||||||
|
idx = dp.index(max(dp))
|
||||||
|
lis: list[int] = []
|
||||||
|
while idx != -1:
|
||||||
|
lis.append(idx)
|
||||||
|
idx = parent[idx]
|
||||||
|
is_normal = [False] * n
|
||||||
|
for k in lis:
|
||||||
|
is_normal[k] = True
|
||||||
|
|
||||||
|
result = [float(v) for v in values]
|
||||||
|
i = 0
|
||||||
|
while i < n:
|
||||||
|
if is_normal[i]:
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
j = i
|
||||||
|
while j < n and not is_normal[j]:
|
||||||
|
j += 1
|
||||||
|
count = j - i
|
||||||
|
left = next((result[k] for k in range(i - 1, -1, -1) if is_normal[k]), None)
|
||||||
|
right = next((result[k] for k in range(j, n) if is_normal[k]), None)
|
||||||
|
if count <= 2:
|
||||||
|
for k in range(i, j):
|
||||||
|
if left is None:
|
||||||
|
result[k] = right
|
||||||
|
elif right is None:
|
||||||
|
result[k] = left
|
||||||
|
else:
|
||||||
|
result[k] = left if (k - (i - 1)) <= (j - k) else right
|
||||||
|
elif left is not None and right is not None:
|
||||||
|
step = (right - left) / (count + 1)
|
||||||
|
for k in range(i, j):
|
||||||
|
result[k] = left + step * (k - i + 1)
|
||||||
|
elif left is not None:
|
||||||
|
for k in range(i, j):
|
||||||
|
result[k] = left
|
||||||
|
elif right is not None:
|
||||||
|
for k in range(i, j):
|
||||||
|
result[k] = right
|
||||||
|
i = j
|
||||||
|
|
||||||
|
return [int(r) for r in result]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_timestamp_token_id(tokenizer: Any, timestamp_token: str = TIMESTAMP_TOKEN) -> int:
|
||||||
|
"""Look up the integer id of the timestamp special token."""
|
||||||
|
convert = getattr(tokenizer, "convert_tokens_to_ids", None)
|
||||||
|
if not callable(convert):
|
||||||
|
raise RuntimeError("Aligner tokenizer has no convert_tokens_to_ids method.")
|
||||||
|
tid = convert(timestamp_token)
|
||||||
|
if isinstance(tid, list):
|
||||||
|
tid = tid[0] if tid else None
|
||||||
|
if tid is None or (isinstance(tid, int) and tid < 0):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Aligner tokenizer does not recognise {timestamp_token!r} (got id={tid}). "
|
||||||
|
"Check the model card; the marker token may use a different name."
|
||||||
|
)
|
||||||
|
return int(tid)
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_for_tests() -> None:
|
||||||
|
"""Drop the cached official-processor probe so the next call re-checks."""
|
||||||
|
global _official_processor, _official_processor_unavailable
|
||||||
|
_official_processor = None
|
||||||
|
_official_processor_unavailable = False
|
||||||
Reference in new issue
Block a user