[Feature] Flush the speech WebSocket on input.done without closing it (#5517)
Signed-off-by: iancarrasco-b10 <ian.carrasco@baseten.co>
This commit is contained in:
1 parent
0feb92bdfa
commit
23bb4395bb
8 files changed
+688
-302
No files matched your search
@@ -537,12 +537,19 @@ curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
|
||||
|
||||
### Streaming WebSocket
|
||||
The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
|
||||
The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
|
||||
```bash
|
||||
python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
|
||||
python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
|
||||
```
|
||||
|
||||
`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
|
||||
```bash
|
||||
python qwen3_tts/streaming_speech_client.py \
|
||||
--text "First utterance." \
|
||||
--text "Second utterance, same connection."
|
||||
```
|
||||
|
||||
To receive word-level timestamps, launch the server with a forced aligner:
|
||||
```bash
|
||||
vllm-omni serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||
|
||||
@@ -7,6 +7,12 @@ Usage:
|
||||
# Send full text at once
|
||||
python streaming_speech_client.py --text "Hello world. How are you? I am fine."
|
||||
|
||||
# Several utterances over one connection: input.done flushes each of them
|
||||
# and the connection stays open, so only the first pays the handshake
|
||||
python streaming_speech_client.py \
|
||||
--text "First utterance." \
|
||||
--text "Second utterance, same connection."
|
||||
|
||||
# Pick a built-in speaker for CustomVoice models
|
||||
python streaming_speech_client.py \
|
||||
--text "打开电子枪,关闭扫描,切换到低倍模式。" \
|
||||
@@ -64,18 +70,27 @@ except ImportError:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def save_audio_file(output_dir: str, sentence_index: int, response_format: str, chunks: list[bytes]) -> str:
|
||||
def frame_basename(msg: dict) -> str:
|
||||
"""Name a file after the utterance and sentence a frame belongs to.
|
||||
|
||||
Every utterance is sentence 0, so the utterance index is what keeps the
|
||||
files of one connection from overwriting each other.
|
||||
"""
|
||||
return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"
|
||||
|
||||
|
||||
def save_audio_file(output_dir: str, basename: str, response_format: str, chunks: list[bytes]) -> str:
|
||||
audio_bytes = b"".join(chunks)
|
||||
if response_format == "pcm":
|
||||
raw_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.pcm")
|
||||
raw_path = os.path.join(output_dir, f"{basename}.pcm")
|
||||
with open(raw_path, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
|
||||
wav_path = os.path.join(output_dir, f"sentence_{sentence_index:03d}.wav")
|
||||
wav_path = os.path.join(output_dir, f"{basename}.wav")
|
||||
save_pcm_wav(wav_path, audio_bytes)
|
||||
return wav_path
|
||||
|
||||
filename = os.path.join(output_dir, f"sentence_{sentence_index:03d}.{response_format}")
|
||||
filename = os.path.join(output_dir, f"{basename}.{response_format}")
|
||||
with open(filename, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
return filename
|
||||
@@ -95,183 +110,207 @@ def timestamp_words_text(timestamps: list[dict] | None) -> str:
|
||||
return " ".join(str(item.get("word", "")) for item in timestamps if item.get("word"))
|
||||
|
||||
|
||||
async def send_utterance(ws, text: str, simulate_stt: bool, stt_delay: float) -> None:
|
||||
"""Send one utterance's text, then input.done to flush it."""
|
||||
if simulate_stt:
|
||||
words = text.split(" ")
|
||||
for i, word in enumerate(words):
|
||||
chunk = word + (" " if i < len(words) - 1 else "")
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": chunk,
|
||||
}
|
||||
)
|
||||
)
|
||||
print(f" Sent: {chunk!r}")
|
||||
await asyncio.sleep(stt_delay)
|
||||
else:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
)
|
||||
print(f"Sent full text: {text!r}")
|
||||
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
print("Sent input.done")
|
||||
|
||||
|
||||
async def receive_utterance(
|
||||
ws,
|
||||
output_dir: str,
|
||||
response_format: str,
|
||||
word_timestamps: bool,
|
||||
save_chunks: bool,
|
||||
) -> None:
|
||||
"""Consume frames until session.done marks the flushed utterance complete."""
|
||||
current_label = "utterance 0"
|
||||
current_sentence_text = ""
|
||||
current_chunks: list[bytes] = []
|
||||
current_timestamps: list[dict] = []
|
||||
# Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
|
||||
# the sentence is a failure only if no frame ever carried timestamps.
|
||||
alignment_frame_seen = False
|
||||
|
||||
while True:
|
||||
message = await ws.recv()
|
||||
|
||||
if isinstance(message, bytes):
|
||||
current_chunks.append(message)
|
||||
print(f" Received audio chunk for {current_label}: {len(message)} bytes")
|
||||
else:
|
||||
# JSON frame
|
||||
msg = json.loads(message)
|
||||
msg_type = msg.get("type")
|
||||
|
||||
if msg_type == "audio.start":
|
||||
current_label = f"utterance {msg['utterance_index']}"
|
||||
current_sentence_text = msg.get("sentence_text", "")
|
||||
current_chunks = []
|
||||
current_timestamps = []
|
||||
alignment_frame_seen = False
|
||||
print(f" [{current_label}] Generating: {current_sentence_text!r}")
|
||||
elif msg_type == "audio.chunk":
|
||||
audio = base64.b64decode(msg["audio_b64"])
|
||||
current_chunks.append(audio)
|
||||
chunk_timestamps = msg.get("timestamps")
|
||||
# Sentence-level alignment sends a trailing timestamp-only
|
||||
# frame with empty audio; don't write an empty chunk file.
|
||||
if save_chunks and audio:
|
||||
chunk_base = os.path.join(
|
||||
output_dir,
|
||||
f"{frame_basename(msg)}_chunk_{msg['chunk_id']:03d}",
|
||||
)
|
||||
chunk_pcm = f"{chunk_base}.pcm"
|
||||
chunk_wav = f"{chunk_base}.wav"
|
||||
chunk_json = f"{chunk_base}_timestamps.json"
|
||||
with open(chunk_pcm, "wb") as f:
|
||||
f.write(audio)
|
||||
save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
|
||||
with open(chunk_json, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"utterance_index": msg["utterance_index"],
|
||||
"sentence_index": msg["sentence_index"],
|
||||
"chunk_id": msg["chunk_id"],
|
||||
"chunk_start_ms": msg.get("chunk_start_ms"),
|
||||
"chunk_end_ms": msg.get("chunk_end_ms"),
|
||||
"sentence_text": current_sentence_text,
|
||||
"covered_text": timestamp_words_text(chunk_timestamps),
|
||||
"timestamps": chunk_timestamps,
|
||||
},
|
||||
f,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
if chunk_timestamps is None:
|
||||
print(f" [{current_label}] chunk {msg['chunk_id']}: {len(audio)} bytes, timestamps unavailable")
|
||||
else:
|
||||
alignment_frame_seen = True
|
||||
current_timestamps.extend(chunk_timestamps)
|
||||
print(
|
||||
f" [{current_label}] chunk {msg['chunk_id']}: "
|
||||
f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
|
||||
)
|
||||
if save_chunks and audio:
|
||||
print(f" saved chunk -> {chunk_wav}")
|
||||
elif msg_type == "audio.done":
|
||||
filename = save_audio_file(
|
||||
output_dir,
|
||||
frame_basename(msg),
|
||||
response_format,
|
||||
current_chunks,
|
||||
)
|
||||
if word_timestamps:
|
||||
# `null` on failure, else the accumulated list (`[]` for silence).
|
||||
sentence_timestamps = current_timestamps if alignment_frame_seen else None
|
||||
ts_filename = os.path.join(
|
||||
output_dir,
|
||||
f"{frame_basename(msg)}_timestamps.json",
|
||||
)
|
||||
with open(ts_filename, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"utterance_index": msg["utterance_index"],
|
||||
"sentence_index": msg["sentence_index"],
|
||||
"sentence_text": current_sentence_text,
|
||||
"covered_text": timestamp_words_text(sentence_timestamps),
|
||||
"timestamps": sentence_timestamps,
|
||||
},
|
||||
f,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
print(
|
||||
f" [{current_label}] Done"
|
||||
f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
|
||||
f" error={msg.get('error', False)}"
|
||||
f" -> {filename}"
|
||||
)
|
||||
if word_timestamps:
|
||||
print(f" [{current_label}] Timestamps -> {ts_filename}")
|
||||
current_chunks = []
|
||||
current_timestamps = []
|
||||
alignment_frame_seen = False
|
||||
elif msg_type == "session.done":
|
||||
print(f"\nUtterance {msg['utterance_index']} complete: {msg['total_sentences']} sentence(s) generated")
|
||||
return
|
||||
elif msg_type == "error":
|
||||
print(f" ERROR: {msg['message']}")
|
||||
else:
|
||||
print(f" Unknown message: {msg}")
|
||||
|
||||
|
||||
async def stream_tts(
|
||||
url: str,
|
||||
text: str,
|
||||
texts: list[str],
|
||||
config: dict,
|
||||
output_dir: str,
|
||||
simulate_stt: bool = False,
|
||||
stt_delay: float = 0.1,
|
||||
save_chunks: bool = False,
|
||||
) -> None:
|
||||
"""Connect to the streaming TTS endpoint and process audio responses."""
|
||||
"""Connect to the streaming TTS endpoint and process audio responses.
|
||||
|
||||
Every text is synthesized over the same connection: input.done flushes an
|
||||
utterance without closing, so only the first one pays the handshake.
|
||||
"""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
response_format = config.get("response_format", "wav")
|
||||
word_timestamps = bool(config.get("word_timestamps", False))
|
||||
|
||||
async with websockets.connect(url) as ws:
|
||||
# 1. Send session config
|
||||
# 1. Send session config; it stays in effect for every utterance below.
|
||||
config_msg = {"type": "session.config", **config}
|
||||
await ws.send(json.dumps(config_msg))
|
||||
print(f"Sent session config: {config}")
|
||||
|
||||
# 2. Send text (either all at once or word-by-word)
|
||||
async def send_text():
|
||||
if simulate_stt:
|
||||
words = text.split(" ")
|
||||
for i, word in enumerate(words):
|
||||
chunk = word + (" " if i < len(words) - 1 else "")
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": chunk,
|
||||
}
|
||||
)
|
||||
)
|
||||
print(f" Sent: {chunk!r}")
|
||||
await asyncio.sleep(stt_delay)
|
||||
else:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input.text",
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
)
|
||||
print(f"Sent full text: {text!r}")
|
||||
|
||||
# 3. Signal end of input
|
||||
await ws.send(json.dumps({"type": "input.done"}))
|
||||
print("Sent input.done")
|
||||
|
||||
# Run sender and receiver concurrently
|
||||
sender_task = asyncio.create_task(send_text())
|
||||
|
||||
response_format = config.get("response_format", "wav")
|
||||
word_timestamps = bool(config.get("word_timestamps", False))
|
||||
current_sentence_index = 0
|
||||
current_sentence_text = ""
|
||||
current_chunks: list[bytes] = []
|
||||
current_timestamps: list[dict] = []
|
||||
# Distinguish `null` (aligner failed) from `[]` (silence / no tokens):
|
||||
# the sentence is a failure only if no frame ever carried timestamps.
|
||||
alignment_frame_seen = False
|
||||
|
||||
try:
|
||||
while True:
|
||||
message = await ws.recv()
|
||||
|
||||
if isinstance(message, bytes):
|
||||
current_chunks.append(message)
|
||||
print(f" Received audio chunk for sentence {current_sentence_index}: {len(message)} bytes")
|
||||
else:
|
||||
# JSON frame
|
||||
msg = json.loads(message)
|
||||
msg_type = msg.get("type")
|
||||
|
||||
if msg_type == "audio.start":
|
||||
current_sentence_index = msg["sentence_index"]
|
||||
current_sentence_text = msg.get("sentence_text", "")
|
||||
current_chunks = []
|
||||
current_timestamps = []
|
||||
alignment_frame_seen = False
|
||||
print(f" [sentence {msg['sentence_index']}] Generating: {current_sentence_text!r}")
|
||||
elif msg_type == "audio.chunk":
|
||||
audio = base64.b64decode(msg["audio_b64"])
|
||||
current_chunks.append(audio)
|
||||
chunk_timestamps = msg.get("timestamps")
|
||||
# Sentence-level alignment sends a trailing timestamp-only
|
||||
# frame with empty audio; don't write an empty chunk file.
|
||||
if save_chunks and audio:
|
||||
chunk_base = os.path.join(
|
||||
output_dir,
|
||||
f"sentence_{msg['sentence_index']:03d}_chunk_{msg['chunk_id']:03d}",
|
||||
)
|
||||
chunk_pcm = f"{chunk_base}.pcm"
|
||||
chunk_wav = f"{chunk_base}.wav"
|
||||
chunk_json = f"{chunk_base}_timestamps.json"
|
||||
with open(chunk_pcm, "wb") as f:
|
||||
f.write(audio)
|
||||
save_pcm_wav(chunk_wav, audio, int(msg.get("sample_rate", 24000)))
|
||||
with open(chunk_json, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"sentence_index": msg["sentence_index"],
|
||||
"chunk_id": msg["chunk_id"],
|
||||
"chunk_start_ms": msg.get("chunk_start_ms"),
|
||||
"chunk_end_ms": msg.get("chunk_end_ms"),
|
||||
"sentence_text": current_sentence_text,
|
||||
"covered_text": timestamp_words_text(chunk_timestamps),
|
||||
"timestamps": chunk_timestamps,
|
||||
},
|
||||
f,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
if chunk_timestamps is None:
|
||||
print(
|
||||
f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
|
||||
f"{len(audio)} bytes, timestamps unavailable"
|
||||
)
|
||||
else:
|
||||
alignment_frame_seen = True
|
||||
current_timestamps.extend(chunk_timestamps)
|
||||
print(
|
||||
f" [sentence {msg['sentence_index']}] chunk {msg['chunk_id']}: "
|
||||
f"{len(audio)} bytes, {len(chunk_timestamps)} timestamp(s)"
|
||||
)
|
||||
if save_chunks and audio:
|
||||
print(f" saved chunk -> {chunk_wav}")
|
||||
elif msg_type == "audio.done":
|
||||
filename = save_audio_file(
|
||||
output_dir,
|
||||
msg["sentence_index"],
|
||||
response_format,
|
||||
current_chunks,
|
||||
)
|
||||
if word_timestamps:
|
||||
# `null` on failure, else the accumulated list (`[]` for silence).
|
||||
sentence_timestamps = current_timestamps if alignment_frame_seen else None
|
||||
ts_filename = os.path.join(
|
||||
output_dir,
|
||||
f"sentence_{msg['sentence_index']:03d}_timestamps.json",
|
||||
)
|
||||
with open(ts_filename, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"sentence_index": msg["sentence_index"],
|
||||
"sentence_text": current_sentence_text,
|
||||
"covered_text": timestamp_words_text(sentence_timestamps),
|
||||
"timestamps": sentence_timestamps,
|
||||
},
|
||||
f,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
print(
|
||||
f" [sentence {msg['sentence_index']}] Done"
|
||||
f" bytes={msg.get('total_bytes', len(b''.join(current_chunks)))}"
|
||||
f" error={msg.get('error', False)}"
|
||||
f" -> {filename}"
|
||||
)
|
||||
if word_timestamps:
|
||||
print(f" [sentence {msg['sentence_index']}] Timestamps -> {ts_filename}")
|
||||
current_chunks = []
|
||||
current_timestamps = []
|
||||
alignment_frame_seen = False
|
||||
elif msg_type == "session.done":
|
||||
print(f"\nSession complete: {msg['total_sentences']} sentence(s) generated")
|
||||
break
|
||||
elif msg_type == "error":
|
||||
print(f" ERROR: {msg['message']}")
|
||||
else:
|
||||
print(f" Unknown message: {msg}")
|
||||
finally:
|
||||
sender_task.cancel()
|
||||
for text in texts:
|
||||
# 2. Send text (either all at once or word-by-word), then flush.
|
||||
sender_task = asyncio.create_task(send_utterance(ws, text, simulate_stt, stt_delay))
|
||||
try:
|
||||
await sender_task
|
||||
except asyncio.CancelledError:
|
||||
pass # Task cancellation is expected during shutdown
|
||||
await receive_utterance(
|
||||
ws,
|
||||
output_dir=output_dir,
|
||||
response_format=response_format,
|
||||
word_timestamps=word_timestamps,
|
||||
save_chunks=save_chunks,
|
||||
)
|
||||
finally:
|
||||
sender_task.cancel()
|
||||
try:
|
||||
await sender_task
|
||||
except asyncio.CancelledError:
|
||||
pass # Task cancellation is expected during shutdown
|
||||
|
||||
# 3. Release the connection once there is nothing left to say.
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
print("Sent session.close")
|
||||
|
||||
print(f"\nAudio files saved to: {output_dir}/")
|
||||
|
||||
@@ -286,7 +325,8 @@ def main():
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
required=True,
|
||||
help="Text to synthesize",
|
||||
action="append",
|
||||
help="Text to synthesize; repeat to send several utterances over one connection",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
@@ -385,7 +425,7 @@ def main():
|
||||
asyncio.run(
|
||||
stream_tts(
|
||||
url=args.url,
|
||||
text=args.text,
|
||||
texts=args.text,
|
||||
config=config,
|
||||
output_dir=args.output_dir,
|
||||
simulate_stt=args.simulate_stt,
|
||||
|
||||
Reference in new issue
Block a user