[Feature] Flush the speech WebSocket on input.done without closing it (#5517)

Signed-off-by: iancarrasco-b10 <ian.carrasco@baseten.co>
This commit is contained in:
iancarrasco-b10 authored and GitHub committed 2026-07-28 23:46:19 +00:00
1 parent 0feb92bdfa
commit 23bb4395bb
8 files changed
+688 -302

No files matched your search

@@ -537,12 +537,19 @@ curl -X POST http://localhost:8091/v1/audio/speech \
Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
### Streaming WebSocket
The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
```bash
python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
```
`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
```bash
python qwen3_tts/streaming_speech_client.py \
--text "First utterance." \
--text "Second utterance, same connection."
```
To receive word-level timestamps, launch the server with a forced aligner:
```bash
vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
@@ -7,6 +7,12 @@ Usage:
# Send full text at once
python streaming_speech_client.py --text "Hello world. How are you? I am fine."
# Several utterances over one connection: input.done flushes each of them
# and the connection stays open, so only the first pays the handshake
python streaming_speech_client.py \
--text "First utterance." \
--text "Second utterance, same connection."
# Pick a built-in speaker for CustomVoice models
python streaming_speech_client.py \
--text "打开电子枪,关闭扫描,切换到低倍模式。" \
@@ -64,18 +70,27 @@ except ImportError:
raise SystemExit(1)
def save_audio_file(output_dir: str, sentence_index: int, response_format: str, chunks: list[bytes]) -> str:
def frame_basename(msg: dict) -> str:
"""Name a file after the utterance and sentence a frame belongs to.
Every utterance is sentence 0, so the utterance index is what keeps the
files of one connection from overwriting each other.
"""
return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"
def save_audio_file(output_dir: str, basename: str, response_format: str, chunks: list[bytes]) -> str:
audio_bytes = b"".join(chunks)
if response_format == "pcm":
raw_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.pcm")
raw_path = os.path.join(output_dir, f"{basename}.pcm")
with open(raw_path, "wb") as f:
f.write(audio_bytes)
wav_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.wav")
wav_path = os.path.join(output_dir, f"{basename}.wav")
save_pcm_wav(wav_path, audio_bytes)
return wav_path
filename = os.path.join(output_dir, f"sentence_{sentence_index:03d}.{response_format}")
filename = os.path.join(output_dir, f"{basename}.{response_format}")
with open(filename, "wb") as f:
f.write(audio_bytes)
return filename
@@ -95,183 +110,207 @@ def timestamp_words_text(timestamps: list[dict] | None) -> str:
return " ".join(str(item.get("word", "")) for item in timestamps if item.get("word"))
async def send_utterance(ws, text: str, simulate_stt: bool, stt_delay: float) -> None:
"""Send one utterance's text, then input.done to flush it."""
if simulate_stt:
words = text.split(" ")
for i, word in enumerate(words):
chunk = word + (" " if i < len(words) - 1 else "")
await ws.send(
json.dumps(
{
"type": "input.text",
"text": chunk,
}
)
)
print(f" Sent: {chunk!r}")
await asyncio.sleep(stt_delay)
else:
await ws.send(
json.dumps(
{
"type": "input.text",
"text": text,
}
)
)
print(f"Sent full text: {text!r}")
await ws.send(json.dumps({"type": "input.done"}))
print("Sent input.done")
async def receive_utterance(
ws,
output_dir: str,
response_format: str,
word_timestamps: bool,
save_chunks: bool,
) -> None:
"""Consume frames until session.done marks the flushed utterance complete."""
current_label = "utterance 0"
current_sentence_text = ""
current_chunks: list[bytes] = []
current_timestamps: list[dict] = []
# Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
# the sentence is a failure only if no frame ever carried timestamps.
alignment_frame_seen = False
while True:
message = await ws.recv()
if isinstance(message, bytes):
current_chunks.append(message)
print(f" Received audio chunk for {current_label}: {len(message)} bytes")
else:
# JSON frame
msg = json.loads(message)
msg_type = msg.get("type")
if msg_type == "audio.start":
current_label = f"utterance {msg['utterance_index']}"
current_sentence_text = msg.get("sentence_text", "")
current_chunks = []
current_timestamps = []
alignment_frame_seen = False
print(f" [{current_label}] Generating: {current_sentence_text!r}")
elif msg_type == "audio.chunk":
audio = base64.b64decode(msg["audio_b64"])
current_chunks.append(audio)
chunk_timestamps = msg.get("timestamps")
# Sentence-level alignment sends a trailing timestamp-only
# frame with empty audio; don't write an empty chunk file.
if save_chunks and audio:
chunk_base = os.path.join(
output_dir,
f"{frame_basename(msg)}_chunk_{msg['chunk_id']:03d}",
)
chunk_pcm = f"{chunk_base}.pcm"
chunk_wav = f"{chunk_base}.wav"
chunk_json = f"{chunk_base}_timestamps.json"
with open(chunk_pcm, "wb") as f:
f.write(audio)
save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
with open(chunk_json, "w", encoding="utf-8") as f:
json.dump(
{
"utterance_index": msg["utterance_index"],
"sentence_index": msg["sentence_index"],
"chunk_id": msg["chunk_id"],
"chunk_start_ms": msg.get("chunk_start_ms"),
"chunk_end_ms": msg.get("chunk_end_ms"),
"sentence_text": current_sentence_text,
"covered_text": timestamp_words_text(chunk_timestamps),
"timestamps": chunk_timestamps,
},
f,
ensure_ascii=False,
indent=2,
)
if chunk_timestamps is None:
print(f" [{current_label}] chunk {msg['chunk_id']}: {len(audio)} bytes, timestamps unavailable")
else:
alignment_frame_seen = True
current_timestamps.extend(chunk_timestamps)
print(
f" [{current_label}] chunk {msg['chunk_id']}: "
f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
)
if save_chunks and audio:
print(f" saved chunk -> {chunk_wav}")
elif msg_type == "audio.done":
filename = save_audio_file(
output_dir,
frame_basename(msg),
response_format,
current_chunks,
)
if word_timestamps:
# `null` on failure, else the accumulated list (`[]` for silence).
sentence_timestamps = current_timestamps if alignment_frame_seen else None
ts_filename = os.path.join(
output_dir,
f"{frame_basename(msg)}_timestamps.json",
)
with open(ts_filename, "w", encoding="utf-8") as f:
json.dump(
{
"utterance_index": msg["utterance_index"],
"sentence_index": msg["sentence_index"],
"sentence_text": current_sentence_text,
"covered_text": timestamp_words_text(sentence_timestamps),
"timestamps": sentence_timestamps,
},
f,
ensure_ascii=False,
indent=2,
)
print(
f" [{current_label}] Done"
f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
f" error={msg.get('error', False)}"
f" -> {filename}"
)
if word_timestamps:
print(f" [{current_label}] Timestamps -> {ts_filename}")
current_chunks = []
current_timestamps = []
alignment_frame_seen = False
elif msg_type == "session.done":
print(f"\nUtterance {msg['utterance_index']} complete: {msg['total_sentences']} sentence(s) generated")
return
elif msg_type == "error":
print(f" ERROR: {msg['message']}")
else:
print(f" Unknown message: {msg}")
async def stream_tts(
url: str,
text: str,
texts: list[str],
config: dict,
output_dir: str,
simulate_stt: bool = False,
stt_delay: float = 0.1,
save_chunks: bool = False,
) -> None:
"""Connect to the streaming TTS endpoint and process audio responses."""
"""Connect to the streaming TTS endpoint and process audio responses.
Every text is synthesized over the same connection: input.done flushes an
utterance without closing, so only the first one pays the handshake.
"""
os.makedirs(output_dir, exist_ok=True)
response_format = config.get("response_format", "wav")
word_timestamps = bool(config.get("word_timestamps", False))
async with websockets.connect(url) as ws:
# 1. Send session config
# 1. Send session config; it stays in effect for every utterance below.
config_msg = {"type": "session.config", **config}
await ws.send(json.dumps(config_msg))
print(f"Sent session config: {config}")
# 2. Send text (either all at once or word-by-word)
async def send_text():
if simulate_stt:
words = text.split(" ")
for i, word in enumerate(words):
chunk = word + (" " if i < len(words) - 1 else "")
await ws.send(
json.dumps(
{
"type": "input.text",
"text": chunk,
}
)
)
print(f" Sent: {chunk!r}")
await asyncio.sleep(stt_delay)
else:
await ws.send(
json.dumps(
{
"type": "input.text",
"text": text,
}
)
)
print(f"Sent full text: {text!r}")
# 3. Signal end of input
await ws.send(json.dumps({"type": "input.done"}))
print("Sent input.done")
# Run sender and receiver concurrently
sender_task = asyncio.create_task(send_text())
response_format = config.get("response_format", "wav")
word_timestamps = bool(config.get("word_timestamps", False))
current_sentence_index = 0
current_sentence_text = ""
current_chunks: list[bytes] = []
current_timestamps: list[dict] = []
# Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
# the sentence is a failure only if no frame ever carried timestamps.
alignment_frame_seen = False
try:
while True:
message = await ws.recv()
if isinstance(message, bytes):
current_chunks.append(message)
print(f" Received audio chunk for sentence {current_sentence_index}: {len(message)} bytes")
else:
# JSON frame
msg = json.loads(message)
msg_type = msg.get("type")
if msg_type == "audio.start":
current_sentence_index = msg["sentence_index"]
current_sentence_text = msg.get("sentence_text", "")
current_chunks = []
current_timestamps = []
alignment_frame_seen = False
print(f" [sentence {msg['sentence_index']}] Generating: {current_sentence_text!r}")
elif msg_type == "audio.chunk":
audio = base64.b64decode(msg["audio_b64"])
current_chunks.append(audio)
chunk_timestamps = msg.get("timestamps")
# Sentence-level alignment sends a trailing timestamp-only
# frame with empty audio; don't write an empty chunk file.
if save_chunks and audio:
chunk_base = os.path.join(
output_dir,
f"sentence_{msg['sentence_index']:03d}_chunk_{msg['chunk_id']:03d}",
)
chunk_pcm = f"{chunk_base}.pcm"
chunk_wav = f"{chunk_base}.wav"
chunk_json = f"{chunk_base}_timestamps.json"
with open(chunk_pcm, "wb") as f:
f.write(audio)
save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
with open(chunk_json, "w", encoding="utf-8") as f:
json.dump(
{
"sentence_index": msg["sentence_index"],
"chunk_id": msg["chunk_id"],
"chunk_start_ms": msg.get("chunk_start_ms"),
"chunk_end_ms": msg.get("chunk_end_ms"),
"sentence_text": current_sentence_text,
"covered_text": timestamp_words_text(chunk_timestamps),
"timestamps": chunk_timestamps,
},
f,
ensure_ascii=False,
indent=2,
)
if chunk_timestamps is None:
print(
f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
f"{len(audio)} bytes, timestamps unavailable"
)
else:
alignment_frame_seen = True
current_timestamps.extend(chunk_timestamps)
print(
f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
)
if save_chunks and audio:
print(f" saved chunk -> {chunk_wav}")
elif msg_type == "audio.done":
filename = save_audio_file(
output_dir,
msg["sentence_index"],
response_format,
current_chunks,
)
if word_timestamps:
# `null` on failure, else the accumulated list (`[]` for silence).
sentence_timestamps = current_timestamps if alignment_frame_seen else None
ts_filename = os.path.join(
output_dir,
f"sentence_{msg['sentence_index']:03d}_timestamps.json",
)
with open(ts_filename, "w", encoding="utf-8") as f:
json.dump(
{
"sentence_index": msg["sentence_index"],
"sentence_text": current_sentence_text,
"covered_text": timestamp_words_text(sentence_timestamps),
"timestamps": sentence_timestamps,
},
f,
ensure_ascii=False,
indent=2,
)
print(
f" [sentence {msg['sentence_index']}] Done"
f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
f" error={msg.get('error', False)}"
f" -> {filename}"
)
if word_timestamps:
print(f" [sentence {msg['sentence_index']}] Timestamps -> {ts_filename}")
current_chunks = []
current_timestamps = []
alignment_frame_seen = False
elif msg_type == "session.done":
print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated")
break
elif msg_type == "error":
print(f" ERROR: {msg['message']}")
else:
print(f" Unknown message: {msg}")
finally:
sender_task.cancel()
for text in texts:
# 2. Send text (either all at once or word-by-word), then flush.
sender_task = asyncio.create_task(send_utterance(ws, text, simulate_stt, stt_delay))
try:
await sender_task
except asyncio.CancelledError:
pass # Task cancellation is expected during shutdown
await receive_utterance(
ws,
output_dir=output_dir,
response_format=response_format,
word_timestamps=word_timestamps,
save_chunks=save_chunks,
)
finally:
sender_task.cancel()
try:
await sender_task
except asyncio.CancelledError:
pass # Task cancellation is expected during shutdown
# 3. Release the connection once there is nothing left to say.
await ws.send(json.dumps({"type": "session.close"}))
print("Sent session.close")
print(f"\nAudio files saved to: {output_dir}/")
@@ -286,7 +325,8 @@ def main():
parser.add_argument(
"--text",
required=True,
help="Text to synthesize",
action="append",
help="Text to synthesize; repeat to send several utterances over one connection",
)
parser.add_argument(
"--output-dir",
@@ -385,7 +425,7 @@ def main():
asyncio.run(
stream_tts(
url=args.url,
text=args.text,
texts=args.text,
config=config,
output_dir=args.output_dir,
simulate_stt=args.simulate_stt,