Merge remote-tracking branch 'origin/main' into qwen-image-2.1
# Conflicts: # vllm_omni/diffusion/executor/multiproc_executor.py # vllm_omni/diffusion/offloader/sequential_backend.py
This commit is contained in:
commit
9174554e40
1332 files changed
+160051
-60504
No files matched your search
@@ -2,7 +2,6 @@
|
||||
# The docs generator combines these names with the serving mode for each page H1.
|
||||
aura_omni: AURA Omni
|
||||
covo_audio: Covo-Audio-Chat
|
||||
dynin_omni: Dynin-Omni
|
||||
lance: Lance
|
||||
mimo_audio: MiMo-Audio
|
||||
ming_flash_omni: Ming-flash-omni 2.0
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Stream LingBot-World 2.0 pixels from one stepwise request.
|
||||
|
||||
One ``AsyncOmni.generate()`` drives the whole rollout: each AR block leaves
|
||||
``post_decode()`` as one video chunk, decoded through that session's own
|
||||
temporal VAE cache rather than as an isolated clip. That is what this script
|
||||
shows end to end -- chunk *N + 1* continues chunk *N*, so the streamed timeline
|
||||
carries the same frame count as an offline whole-clip decode of the same
|
||||
latents instead of losing each block's opening frame to a restarted decoder.
|
||||
Only the session's first latent frame expands to a single raw frame; on the
|
||||
default checkpoint, whose block is three latent frames at a temporal factor of
|
||||
four, that is 9 frames for the opening chunk and 12 for every later one.
|
||||
|
||||
The decode lives inside the pipeline, keyed by ``request_id``, so nothing here
|
||||
owns a VAE: this file is a client of the served path, and the same chunks reach
|
||||
a WebSocket client of ``/v1/realtime/video`` unchanged.
|
||||
|
||||
Examples::
|
||||
|
||||
python examples/offline_inference/diffusion/ar_diffusion_streaming_decode.py \
|
||||
--image scene.png --prompt "a lit hallway" \
|
||||
--action-script actions.json --output-dir /tmp/lingbot-stream
|
||||
|
||||
``--action-script`` is a JSON list with one camera action list per AR block,
|
||||
for example ``[[["w"], ["w"], ["w"]], [["a"], [], []]]`` for a checkpoint that
|
||||
generates three latent frames per block. The frames per block and temporal
|
||||
compression are read off the checkpoint rather than assumed, so the script
|
||||
can be repointed with ``--model``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Only for type hints -- the real imports stay deferred inside run() so
|
||||
# --help and the offline validation helpers don't require a CUDA-enabled
|
||||
# vLLM install.
|
||||
import numpy as np
|
||||
|
||||
_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Stream decoded LingBot-World 2.0 video chunks from a single stepwise request.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
parser.add_argument("--model", default=_MODEL, help="Hugging Face model ID or local checkpoint path.")
|
||||
parser.add_argument("--image", required=True, help="Initial RGB image.")
|
||||
parser.add_argument("--prompt", required=True, help="Scene prompt for the rollout.")
|
||||
parser.add_argument(
|
||||
"--action-script",
|
||||
required=True,
|
||||
help=(
|
||||
"JSON file holding one camera action list per AR block. "
|
||||
"The per-block frame count is read off the checkpoint, not assumed."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--output-dir", required=True, help="Directory for decoded frames and metadata.")
|
||||
parser.add_argument("--request-id", default="lingbot-world-stream", help="Request id; also the session id.")
|
||||
parser.add_argument("--height", type=int, default=480)
|
||||
parser.add_argument("--width", type=int, default=832)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--gpu-memory-fraction", type=float, default=0.6)
|
||||
parser.add_argument("--tensor-parallel-size", type=int, default=1)
|
||||
parser.add_argument("--enforce-eager", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def _load_action_script(path: Path) -> list[list[list[str]]]:
|
||||
"""Parse the per-block camera action script, without assuming its geometry.
|
||||
|
||||
The shape is the one ``sampling_params.extra_args["camera_action_script"]``
|
||||
takes: one entry per AR block, each entry one per-frame action list. How
|
||||
many frames a block holds is a property of the checkpoint, so it is checked
|
||||
in :func:`_validate_against_checkpoint` once the configs have been read.
|
||||
"""
|
||||
try:
|
||||
value = json.loads(path.read_text())
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"--action-script is not valid JSON: {exc.msg}.") from None
|
||||
if not isinstance(value, list) or not value:
|
||||
raise ValueError("--action-script must be a non-empty JSON list of per-block action lists.")
|
||||
script: list[list[list[str]]] = []
|
||||
for block_index, block in enumerate(value):
|
||||
if not isinstance(block, list) or not block:
|
||||
raise ValueError(f"--action-script block {block_index} must be a non-empty list of frame action lists.")
|
||||
frames: list[list[str]] = []
|
||||
for frame in block:
|
||||
if not isinstance(frame, list) or any(not isinstance(action, str) for action in frame):
|
||||
raise ValueError(f"--action-script block {block_index} frames must contain only action strings.")
|
||||
frames.append(list(frame))
|
||||
script.append(frames)
|
||||
return script
|
||||
|
||||
|
||||
def _model_config_json(model: str, relative_path: str) -> dict[str, Any]:
|
||||
"""Read one config file out of a local checkpoint or a Hub repo."""
|
||||
if os.path.isdir(model):
|
||||
return json.loads((Path(model) / relative_path).read_text())
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
return json.loads(Path(hf_hub_download(repo_id=model, filename=relative_path)).read_text())
|
||||
|
||||
|
||||
def _checkpoint_geometry(model: str) -> tuple[int, int]:
|
||||
"""Read ``(frames_per_block, temporal_compression)`` off the checkpoint.
|
||||
|
||||
The pipeline takes both from the model --
|
||||
``transformer.config.num_frames_per_block`` and
|
||||
``vae.config.scale_factor_temporal`` -- so hardcoding 3 and 4 here would
|
||||
silently build a wrong action script and an inconsistent ``num_frames`` the
|
||||
moment ``--model`` points at a checkpoint with different geometry.
|
||||
"""
|
||||
transformer_config = _model_config_json(model, "transformer/config.json")
|
||||
vae_config = _model_config_json(model, "vae/config.json")
|
||||
if "num_frames_per_block" not in transformer_config:
|
||||
raise ValueError(
|
||||
f"{model}'s transformer/config.json declares no num_frames_per_block, "
|
||||
"so this checkpoint does not generate AR blocks and cannot be streamed this way."
|
||||
)
|
||||
frames_per_block = int(transformer_config["num_frames_per_block"])
|
||||
temporal_compression = int(vae_config.get("scale_factor_temporal", 4))
|
||||
if frames_per_block <= 0 or temporal_compression <= 0:
|
||||
raise ValueError(
|
||||
"the checkpoint reports a non-positive block geometry "
|
||||
f"(num_frames_per_block={frames_per_block}, scale_factor_temporal={temporal_compression})."
|
||||
)
|
||||
return frames_per_block, temporal_compression
|
||||
|
||||
|
||||
def _validate_against_checkpoint(
|
||||
script: list[list[list[str]]], *, frames_per_block: int, temporal_compression: int
|
||||
) -> int:
|
||||
"""Check the script against the checkpoint and return the frame count to request."""
|
||||
for block_index, block in enumerate(script):
|
||||
if len(block) != frames_per_block:
|
||||
raise ValueError(
|
||||
f"--action-script block {block_index} holds {len(block)} frame action lists, but this "
|
||||
f"checkpoint generates {frames_per_block} latent frames per AR block."
|
||||
)
|
||||
return (len(script) * frames_per_block - 1) * temporal_compression + 1
|
||||
|
||||
|
||||
def _validate_args(args: argparse.Namespace) -> tuple[Path, Path, Path]:
|
||||
image = Path(args.image).expanduser().resolve()
|
||||
action_script = Path(args.action_script).expanduser().resolve()
|
||||
output_dir = Path(args.output_dir).expanduser().resolve()
|
||||
if not image.is_file():
|
||||
raise ValueError("--image must point to an existing file.")
|
||||
if not action_script.is_file():
|
||||
raise ValueError("--action-script must point to an existing JSON file.")
|
||||
if not args.prompt.strip():
|
||||
raise ValueError("--prompt must contain non-whitespace text.")
|
||||
if args.height <= 0 or args.width <= 0 or args.height % 16 or args.width % 16:
|
||||
raise ValueError("--height and --width must be positive multiples of 16.")
|
||||
if args.tensor_parallel_size <= 0:
|
||||
raise ValueError("--tensor-parallel-size must be positive.")
|
||||
if not math.isfinite(args.gpu_memory_fraction) or not 0 < args.gpu_memory_fraction <= 1:
|
||||
raise ValueError("--gpu-memory-fraction must be in (0, 1].")
|
||||
if not args.request_id.strip():
|
||||
raise ValueError("--request-id must contain non-whitespace text.")
|
||||
return image, action_script, output_dir
|
||||
|
||||
|
||||
def _as_frames(value: Any) -> np.ndarray:
|
||||
"""Normalize one streamed chunk to ``[T, H, W, 3]`` uint8.
|
||||
|
||||
``output_type="np"`` reaches the client as a float array in ``[0, 1]``,
|
||||
with or without the leading batch axis depending on how the formatter
|
||||
unwrapped it; a nested list is the ``"pil"`` shape. Anything else means
|
||||
the chunk is not pixels -- most likely the request still asked for
|
||||
latents -- and is worth failing on rather than saving.
|
||||
"""
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import torch
|
||||
|
||||
if isinstance(value, list):
|
||||
if len(value) == 1 and isinstance(value[0], list):
|
||||
value = value[0]
|
||||
if value and all(isinstance(frame, PIL.Image.Image) for frame in value):
|
||||
return np.stack([np.asarray(frame.convert("RGB")) for frame in value])
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = value.detach().float().cpu().numpy()
|
||||
if not isinstance(value, np.ndarray):
|
||||
raise RuntimeError(f"Expected decoded pixels from each AR block, got {type(value).__name__}.")
|
||||
if value.ndim == 5 and value.shape[0] == 1:
|
||||
value = value[0]
|
||||
if value.ndim != 4 or value.shape[-1] != 3:
|
||||
raise RuntimeError(f"Expected one [T, H, W, 3] pixel chunk per AR block, got shape {tuple(value.shape)}.")
|
||||
if value.dtype != np.uint8:
|
||||
value = (np.clip(value, 0.0, 1.0) * 255.0).round().astype(np.uint8)
|
||||
return value
|
||||
|
||||
|
||||
def _chunk_metadata(output: Any) -> dict[str, Any]:
|
||||
multimodal = getattr(output, "multimodal_output", None) or {}
|
||||
metadata = multimodal.get("metadata") if isinstance(multimodal, dict) else None
|
||||
ar_diffusion = metadata.get("ar_diffusion") if isinstance(metadata, dict) else None
|
||||
if not isinstance(ar_diffusion, dict):
|
||||
raise RuntimeError("Streamed chunk is missing its ar_diffusion metadata envelope.")
|
||||
return dict(ar_diffusion)
|
||||
|
||||
|
||||
async def run(argv: Sequence[str] | None = None) -> Path:
|
||||
args = parse_args(argv)
|
||||
image, action_script_path, output_dir = _validate_args(args)
|
||||
camera_action_script = _load_action_script(action_script_path)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Imports stay below pure input validation so --help and helper tests do not
|
||||
# require a CUDA-enabled vLLM installation.
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
frames_per_block, temporal_compression = _checkpoint_geometry(args.model)
|
||||
num_chunks = len(camera_action_script)
|
||||
num_frames = _validate_against_checkpoint(
|
||||
camera_action_script,
|
||||
frames_per_block=frames_per_block,
|
||||
temporal_compression=temporal_compression,
|
||||
)
|
||||
|
||||
engine = AsyncOmni(
|
||||
model=args.model,
|
||||
engine_backend="vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine",
|
||||
enforce_eager=args.enforce_eager,
|
||||
tensor_parallel_size=args.tensor_parallel_size,
|
||||
max_num_seqs=1,
|
||||
# One request, many chunks: the stepwise contract from #6844. Streaming
|
||||
# output is what turns each post_decode() into a delivered chunk.
|
||||
step_execution=True,
|
||||
diffusion_streaming_output=True,
|
||||
model_config={
|
||||
"ar_diffusion_height": args.height,
|
||||
"ar_diffusion_width": args.width,
|
||||
"ar_diffusion_kv_config": {
|
||||
"gpu_memory_fraction": args.gpu_memory_fraction,
|
||||
"warmup_cudagraph": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_frames=num_frames,
|
||||
num_inference_steps=4,
|
||||
max_sequence_length=512,
|
||||
seed=args.seed,
|
||||
# Pixels, not latents: this is what routes the chunk through the
|
||||
# session's streaming VAE decode instead of handing back a tensor.
|
||||
output_type="np",
|
||||
extra_args={"flow_shift": 5.0, "camera_action_script": camera_action_script},
|
||||
)
|
||||
prompt = {"prompt": args.prompt, "multi_modal_data": {"image": str(image)}}
|
||||
|
||||
measurements: list[dict[str, Any]] = []
|
||||
t_start = time.perf_counter()
|
||||
first_frame_at: float | None = None
|
||||
chunk_started = t_start
|
||||
try:
|
||||
async for output in engine.generate(prompt, sampling, request_id=args.request_id):
|
||||
images = getattr(output, "images", None)
|
||||
if not images:
|
||||
continue
|
||||
if len(images) != 1:
|
||||
raise RuntimeError("Expected exactly one decoded chunk per AR block.")
|
||||
frames = _as_frames(images[0])
|
||||
now = time.perf_counter()
|
||||
if first_frame_at is None:
|
||||
first_frame_at = now - t_start
|
||||
chunk_index = len(measurements)
|
||||
metadata = _chunk_metadata(output)
|
||||
|
||||
np.save(output_dir / f"chunk_{chunk_index:03d}.npy", frames)
|
||||
(output_dir / f"chunk_{chunk_index:03d}.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
measurements.append(
|
||||
{
|
||||
"chunk_index": chunk_index,
|
||||
"chunk_latency_seconds": now - chunk_started,
|
||||
"frames": int(frames.shape[0]),
|
||||
"frame_shape": list(frames.shape),
|
||||
"metadata": metadata,
|
||||
}
|
||||
)
|
||||
chunk_started = now
|
||||
print(json.dumps(measurements[-1], sort_keys=True), flush=True)
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
delivered_frames = sum(int(chunk["frames"]) for chunk in measurements)
|
||||
summary_path = output_dir / "summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"request_id": args.request_id,
|
||||
"chunks_delivered": len(measurements),
|
||||
"chunks_expected": num_chunks,
|
||||
"frames_delivered": delivered_frames,
|
||||
"frames_expected": num_frames,
|
||||
# The point of session-owned decode: a streamed rollout carries
|
||||
# the same frame timeline as an offline decode of the same
|
||||
# latents. Per-chunk decode would deliver fewer, because every
|
||||
# block would re-expand its own opening frame.
|
||||
"matches_offline_frame_timeline": delivered_frames == num_frames,
|
||||
"time_to_first_frame_seconds": first_frame_at,
|
||||
"total_seconds": time.perf_counter() - t_start,
|
||||
"chunks": measurements,
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
return summary_path
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> Path:
|
||||
return asyncio.run(run(argv))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Generate a LingBot-World v2 video from an image and camera trajectory.
|
||||
|
||||
The official checkpoint is licensed separately under CC BY-NC-SA and is
|
||||
@@ -32,7 +32,6 @@ import numpy as np
|
||||
_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
|
||||
_MAX_SEQUENCE_LENGTH = 512
|
||||
_MAX_PIXEL_AREA = 480 * 832
|
||||
_MAX_RAW_FRAMES = 117
|
||||
_MAX_SOURCE_CAMERA_FRAMES = 4096
|
||||
_TEMPORAL_COMPRESSION = 4
|
||||
_LATENT_FRAMES_PER_BLOCK = 3
|
||||
@@ -99,6 +98,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of GPUs used for tensor parallelism inside the DiT.",
|
||||
)
|
||||
parser.add_argument("--ulysses-degree", type=int, default=1, help="Pure Ulysses sequence parallel degree.")
|
||||
parser.add_argument("--flow-shift", type=float, default=5.0, help="Positive FlowUniPC scheduler shift.")
|
||||
parser.add_argument("--fps", type=int, default=16, help="Frames per second in the exported MP4.")
|
||||
parser.add_argument("--output", default="lingbot_world_v2.mp4", help="Output MP4 path.")
|
||||
@@ -158,6 +158,8 @@ def build_omni_kwargs(
|
||||
|
||||
if args.tensor_parallel_size <= 0:
|
||||
raise ValueError("--tensor-parallel-size must be a positive integer.")
|
||||
if args.ulysses_degree <= 0:
|
||||
raise ValueError("--ulysses-degree must be a positive integer.")
|
||||
flow_shift = _positive_finite(args.flow_shift, "--flow-shift")
|
||||
model_path = Path(args.model).expanduser()
|
||||
model = str(model_path.resolve()) if model_path.exists() else args.model
|
||||
@@ -165,6 +167,7 @@ def build_omni_kwargs(
|
||||
"model": model,
|
||||
"flow_shift": flow_shift,
|
||||
"tensor_parallel_size": args.tensor_parallel_size,
|
||||
"ulysses_degree": args.ulysses_degree,
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"model_config": {"lingbot_action_root": str(paths.action_root)},
|
||||
}
|
||||
@@ -187,8 +190,6 @@ def build_request(
|
||||
raise ValueError("--height * --width must not exceed 480 * 832 pixels.")
|
||||
if args.num_frames <= 0:
|
||||
raise ValueError("--num-frames must be a positive integer.")
|
||||
if args.num_frames > _MAX_RAW_FRAMES:
|
||||
raise ValueError("--num-frames must not exceed 117 raw frames.")
|
||||
if args.num_frames > paths.camera_frames:
|
||||
raise ValueError("Camera arrays must contain at least --num-frames frames.")
|
||||
if (args.num_frames - 1) % _TEMPORAL_COMPRESSION:
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Run LingBot-World 2.0 as an in-process realtime AR-Diffusion session.
|
||||
|
||||
This example exercises the internal session API. It is not a public HTTP or
|
||||
WebSocket protocol. Each JSONL event advances the world by one three-latent
|
||||
frame AR block and writes the latent plus its identity metadata. The current
|
||||
scoped implementation supports at most ten ticks per generation epoch because
|
||||
its image condition is bounded to 117 pixel frames; reset or create a session
|
||||
to start a new world.
|
||||
frame AR block and writes the latent plus its identity metadata. Session-owned
|
||||
causal VAE encoder state produces each condition block without retaining or
|
||||
repeating a full image-condition horizon.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,8 +23,6 @@ from typing import Any
|
||||
_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
|
||||
_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1"
|
||||
_FRAMES_PER_BLOCK = 3
|
||||
# ((117 pixel frames - 1) / VAE temporal factor 4 + 1) / 3 latent frames.
|
||||
_MAX_REALTIME_TICKS = 10
|
||||
|
||||
|
||||
def _camera_event_data(frames: list[list[str]]) -> dict[str, Any]:
|
||||
@@ -41,7 +38,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--events",
|
||||
required=True,
|
||||
help="JSONL file with one prompt/action event per AR block; at most 10 events.",
|
||||
help="JSONL file with one prompt/action event per AR block.",
|
||||
)
|
||||
parser.add_argument("--output-dir", required=True, help="Directory for chunk latents and metadata.")
|
||||
parser.add_argument("--session-id", default="lingbot-world", help="Persistent world session identifier.")
|
||||
@@ -84,12 +81,6 @@ def _load_events(path: Path) -> list[dict[str, Any]]:
|
||||
events.append({"event_id": event_id, "prompt": prompt, "frames": frames})
|
||||
if not events:
|
||||
raise ValueError("events file must contain at least one event.")
|
||||
if len(events) > _MAX_REALTIME_TICKS:
|
||||
raise ValueError(
|
||||
"events file must contain at most "
|
||||
f"{_MAX_REALTIME_TICKS} events because the current LingBot realtime "
|
||||
"image-condition horizon is 117 pixel frames."
|
||||
)
|
||||
event_ids = [event["event_id"] for event in events]
|
||||
if event_ids != sorted(set(event_ids)):
|
||||
raise ValueError("event_id values must be unique and strictly increasing.")
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
# Dynin-Omni: Offline inference
|
||||
|
||||
This folder contains a unified offline inference entrypoint:
|
||||
|
||||
- `end2end.py`
|
||||
|
||||
## 1. Environment Setup
|
||||
|
||||
Run from repository root:
|
||||
|
||||
```bash
|
||||
cd <REPO_ROOT>
|
||||
```
|
||||
|
||||
If needed, install this repo in editable mode:
|
||||
|
||||
```bash
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## 2. Extra Dependencies (EMOVA)
|
||||
|
||||
Install the following packages for EMOVA-related components:
|
||||
|
||||
```bash
|
||||
pip install \
|
||||
"phonemizer==3.3.0" \
|
||||
"Unidecode==1.4.0" \
|
||||
"hydra-core==1.3.2" \
|
||||
"pytorch-lightning==1.1.0" \
|
||||
"wget==3.2" \
|
||||
"wrapt==2.1.1" \
|
||||
"onnx==1.20.1" \
|
||||
"frozendict==2.4.7" \
|
||||
"inflect==7.5.0" \
|
||||
"braceexpand==0.1.7" \
|
||||
"webdataset==1.0.2" \
|
||||
"torch-stft==0.1.4" \
|
||||
"editdistance==0.8.1"
|
||||
```
|
||||
|
||||
## 3. Hardware and VRAM Requirements
|
||||
|
||||
This example uses a 3-stage pipeline on one GPU by default
|
||||
([`dynin_omni.yaml`](../../../vllm_omni/deploy/dynin_omni.yaml)):
|
||||
|
||||
- Stage-0 (`token2text`): `gpu_memory_utilization: 0.5`
|
||||
- Stage-1 (`token2image`): `gpu_memory_utilization: 0.1`
|
||||
- Stage-2 (`token2audio`): `gpu_memory_utilization: 0.1`
|
||||
|
||||
### Requested GPU Memory Budget from `gpu_memory_utilization`
|
||||
|
||||
| Stage | Utilization | A100 80GB | H200 141GB |
|
||||
| :-- | :-- | :-- | :-- |
|
||||
| Stage-0 (token2text) | 0.5 | ~40.0 GB | ~70.5 GB |
|
||||
| Stage-1 (token2image) | 0.1 | ~8.0 GB | ~14.1 GB |
|
||||
| Stage-2 (token2audio) | 0.1 | ~8.0 GB | ~14.1 GB |
|
||||
| Total requested budget | 0.7 | ~56.0 GB | ~98.7 GB |
|
||||
|
||||
### Observed Runtime Signal (from your log)
|
||||
|
||||
- Stage-0 reported: `Model loading took 15.12 GiB memory` (weights footprint signal).
|
||||
- Stages 1/2 can still add runtime memory depending on task path and backend allocations.
|
||||
- Keep extra headroom for CUDA/PyTorch overhead and temporary allocations.
|
||||
|
||||
### GPU Compatibility
|
||||
|
||||
- Confirmed target GPUs for this setup: **NVIDIA H200**, **NVIDIA A100**.
|
||||
- CI/e2e coverage in this repo also includes CUDA **L4** markers for Dynin tests.
|
||||
|
||||
## 4. End2End Run Examples
|
||||
|
||||
```bash
|
||||
# t2t
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task t2t --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
|
||||
|
||||
# i2t
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task i2t --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text "Please describe this image in detail."
|
||||
|
||||
# s2t
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task s2t --model snu-aidas/Dynin-Omni --audio <AUDIO_PATH> --text "Transcribe the given audio."
|
||||
|
||||
# t2i
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task t2i --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
|
||||
|
||||
# v2t
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task v2t --model snu-aidas/Dynin-Omni --video <VIDEO_PATH> --text "Describe this video in detail."
|
||||
|
||||
# i2i
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task i2i --model snu-aidas/Dynin-Omni --image <IMAGE_PATH> --text <INSTRUCTION_TEXT>
|
||||
|
||||
# t2s
|
||||
python <REPO_ROOT>/examples/offline_inference/dynin_omni/end2end.py \
|
||||
--task t2s --model snu-aidas/Dynin-Omni --text <INSTRUCTION_TEXT>
|
||||
```
|
||||
|
||||
## 5. Notes
|
||||
|
||||
- Outputs are saved under task-specific directories in `/tmp` by default.
|
||||
- You can override output path with `--output-dir`.
|
||||
- If you want to force local config resolution, pass `--dynin-config-path <PATH_TO_DYNIN_OMNI_YAML>`.
|
||||
- If you see the warning
|
||||
`max_num_batched_tokens (32768) exceeds max_num_seqs * max_model_len (4096)`,
|
||||
reduce `max_num_batched_tokens` in the deploy config (for example, `4096` in the CI config).
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,154 +0,0 @@
|
||||
# HunyuanImage-3.0-Instruct
|
||||
|
||||
This example runs HunyuanImage-3.0-Instruct offline with the unified deploy
|
||||
YAMLs under `vllm_omni/deploy/`.
|
||||
|
||||
## Deploy Configs
|
||||
|
||||
| File | Topology | Default use |
|
||||
| :--- | :--- | :--- |
|
||||
| `vllm_omni/deploy/hunyuan_image_3_moe.yaml` | AR + DiT | Default for `text2img` and `img2img`. |
|
||||
| `vllm_omni/deploy/hunyuan_image3_ar.yaml` | AR only | Default for `img2text` and `text2text`. |
|
||||
| `vllm_omni/deploy/hunyuan_image3_dit.yaml` | DiT only | Standalone diffusion stage. Pass it explicitly with `--deploy-config`. |
|
||||
|
||||
The example chooses a deploy config automatically when `--deploy-config` is
|
||||
omitted:
|
||||
|
||||
| `--modality` | `mode` passed to Omni | Default deploy |
|
||||
| :--- | :--- | :--- |
|
||||
| `text2img` | `text-to-image` | `hunyuan_image_3_moe.yaml` |
|
||||
| `img2img` | `image-editing` | `hunyuan_image_3_moe.yaml` |
|
||||
| `img2text` | `image-to-text` | `hunyuan_image3_ar.yaml` |
|
||||
| `text2text` | `text-to-text` | `hunyuan_image3_ar.yaml` |
|
||||
|
||||
`--modality` is an offline example convenience flag. It maps to the internal
|
||||
`mode` argument passed to `Omni(...)` by this script. HunyuanImage3 uses
|
||||
separate deploy YAMLs for AR + DiT, AR-only, and DiT-only topologies, so the
|
||||
stage topology is selected by the deploy file rather than by YAML mode
|
||||
overrides.
|
||||
|
||||
Online serving does not expose a `--modality` flag or accept `mode` as an API
|
||||
request field. Choose the deploy topology when starting the server with
|
||||
`--deploy-config`, then use the OpenAI-compatible endpoint and request shape for
|
||||
the scenario. The `modalities` request field is used by the chat completions
|
||||
path; the image endpoints infer the image task from the endpoint and payload.
|
||||
|
||||
| Online scenario | Server deploy | Request |
|
||||
| :--- | :--- | :--- |
|
||||
| Text to image | `--deploy-config vllm_omni/deploy/hunyuan_image_3_moe.yaml` | `POST /v1/images/generations`, or `POST /v1/chat/completions` with `"modalities": ["image"]`. |
|
||||
| Image editing | `--deploy-config vllm_omni/deploy/hunyuan_image_3_moe.yaml` | `POST /v1/images/edits`. |
|
||||
| Image/text to text | `--deploy-config vllm_omni/deploy/hunyuan_image3_ar.yaml` | `POST /v1/chat/completions` for text output, for example with `"modalities": ["text"]`. |
|
||||
| DiT-only image generation | `--deploy-config vllm_omni/deploy/hunyuan_image3_dit.yaml` | `POST /v1/images/generations`. |
|
||||
|
||||
## Run Examples
|
||||
|
||||
Text to image, using the default AR + DiT deploy:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/hunyuan_image3/end2end.py \
|
||||
--model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality text2img \
|
||||
--prompts "A cute cat sitting on a windowsill watching the sunset"
|
||||
```
|
||||
|
||||
Image editing, using the default AR + DiT deploy:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/hunyuan_image3/end2end.py \
|
||||
--model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality img2img \
|
||||
--image-path /path/to/image.png \
|
||||
--prompts "Make the petals neon pink"
|
||||
```
|
||||
|
||||
Image to text, using the AR-only deploy:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/hunyuan_image3/end2end.py \
|
||||
--model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality img2text \
|
||||
--image-path /path/to/image.jpg \
|
||||
--prompts "Describe the content of the picture."
|
||||
```
|
||||
|
||||
Text to text, using the AR-only deploy:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/hunyuan_image3/end2end.py \
|
||||
--model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality text2text \
|
||||
--prompts "What is the capital of France?"
|
||||
```
|
||||
|
||||
Standalone DiT, using the DiT-only deploy explicitly:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/hunyuan_image3/end2end.py \
|
||||
--model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality text2img \
|
||||
--deploy-config vllm_omni/deploy/hunyuan_image3_dit.yaml \
|
||||
--prompts "A cinematic portrait of an astronaut in a greenhouse"
|
||||
```
|
||||
|
||||
Override the default full AR + DiT deploy explicitly:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/hunyuan_image3/end2end.py \
|
||||
--model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality text2img \
|
||||
--deploy-config vllm_omni/deploy/hunyuan_image_3_moe.yaml \
|
||||
--prompts "A cute cat"
|
||||
```
|
||||
|
||||
## Additional Config
|
||||
|
||||
You can pass diffusion worker `additional_config` from the offline example as a JSON object.
|
||||
This maps to the upstream vLLM `VllmConfig.additional_config` platform extension field:
|
||||
https://docs.vllm.ai/en/stable/api/vllm/config/#vllm.config.VllmConfig.additional_config
|
||||
|
||||
```bash
|
||||
python end2end.py --modality text2img \
|
||||
--prompts "A cute cat" \
|
||||
--additional-config '{"torchair_graph_config":{"enabled":true}}'
|
||||
```
|
||||
|
||||
|
||||
## Key Arguments
|
||||
|
||||
| Argument | Description |
|
||||
| :--- | :--- |
|
||||
| `--deploy-config` | Optional path for unified deploy YAML overrides. |
|
||||
| `--additional-config` | JSON object forwarded to diffusion worker `additional_config`. |
|
||||
| `--modality` | Offline-only convenience flag. One of `text2img`, `img2img`, `img2text`, `text2text`. It selects prompt formatting, internal `mode`, and default deploy config for this script. Online serving uses `--deploy-config` plus the endpoint and, for chat completions, request `modalities` instead. |
|
||||
| `--steps` | Number of diffusion inference steps for image generation. |
|
||||
| `--guidance-scale` | Classifier-free guidance scale for image generation. |
|
||||
| `--height`, `--width` | Output image size for `text2img`. |
|
||||
| `--bot-task` | Override prompt mode. `none`, `think`, `recaption`, `think_recaption`, or `vanilla`. |
|
||||
| `--sys-type` | Override the system prompt type, for example `en_unified` or `en_vanilla`. |
|
||||
| `--vae-use-tiling` | Enable VAE tiling for memory reduction. |
|
||||
|
||||
## Notes
|
||||
|
||||
- `hunyuan_image3_ar.yaml` is a 4-card AR-only text/comprehension deploy.
|
||||
- `hunyuan_image3_dit.yaml` is a single-stage DiT deploy with `stage_id: 0`.
|
||||
- Platform-specific overrides are included in the unified deploy YAMLs.
|
||||
|
||||
## Prompt Format
|
||||
|
||||
HunyuanImage-3.0-Instruct uses an instruct chat template:
|
||||
|
||||
```text
|
||||
<|startoftext|>{system_prompt}
|
||||
|
||||
User: {<img>?}{user_prompt}
|
||||
|
||||
Assistant: {trigger_tag?}
|
||||
```
|
||||
|
||||
- `<img>`: Placeholder for each input image (single token; expanded by the multimodal pipeline).
|
||||
- Trigger tags: `<think>` for CoT and `<recaption>` for recaptioning, placed after `Assistant: `.
|
||||
- System prompt: Auto-selected from `task` and `bot_task`.
|
||||
- `bot_task='vanilla'` with `task='t2i'` uses the bare pretrain template.
|
||||
|
||||
The shared `vllm_omni.diffusion.models.hunyuan_image3.prompt_utils.build_prompt_tokens()`
|
||||
helper handles segment-by-segment tokenization and matches HF `apply_chat_template`.
|
||||
@@ -1,321 +0,0 @@
|
||||
"""
|
||||
HunyuanImage-3.0-Instruct unified end-to-end inference script.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
||||
MAX_IMAGES_PER_REQUEST,
|
||||
build_prompt_tokens,
|
||||
resolve_stop_token_ids,
|
||||
resolve_sys_type,
|
||||
)
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniPromptType
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_DEFAULT_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image_3_moe.yaml")
|
||||
_DEFAULT_AR_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml")
|
||||
|
||||
_MODALITY_TASK_MAP: dict[str, tuple[str, str | None]] = {
|
||||
"text2img": ("t2i", "think"),
|
||||
"img2img": ("it2i", "think"),
|
||||
"img2text": ("i2t", None),
|
||||
"text2text": ("t2t", None),
|
||||
}
|
||||
|
||||
_MODALITY_DEFAULT_DEPLOY_CONFIG = {
|
||||
"text2img": _DEFAULT_DEPLOY_CONFIG,
|
||||
"img2img": _DEFAULT_DEPLOY_CONFIG,
|
||||
"img2text": _DEFAULT_AR_DEPLOY_CONFIG,
|
||||
"text2text": _DEFAULT_AR_DEPLOY_CONFIG,
|
||||
}
|
||||
|
||||
_MODALITY_MODE = {
|
||||
"text2img": "text-to-image",
|
||||
"img2img": "image-editing",
|
||||
"img2text": "image-to-text",
|
||||
"text2text": "text-to-text",
|
||||
}
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="HunyuanImage-3.0-Instruct end-to-end inference.")
|
||||
parser.add_argument("--model", default="tencent/HunyuanImage-3.0-Instruct", help="Model name or local path.")
|
||||
parser.add_argument(
|
||||
"--modality",
|
||||
default="text2img",
|
||||
choices=list(_MODALITY_TASK_MAP),
|
||||
)
|
||||
parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
|
||||
parser.add_argument(
|
||||
"--image-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Input image path(s) for img2img/img2text. Comma-separated for multi-image (up to 3).",
|
||||
)
|
||||
parser.add_argument("--output", type=str, default=".", help="Output directory to save results.")
|
||||
parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=5.0, help="Classifier-free guidance scale.")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--height", type=int, default=None, help="Output image height.")
|
||||
parser.add_argument("--width", type=int, default=None, help="Output image width.")
|
||||
parser.add_argument("--vae-use-tiling", action="store_true", help="Enable VAE tiling.")
|
||||
parser.add_argument(
|
||||
"--bot-task",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["none", "think", "recaption", "think_recaption", "vanilla"],
|
||||
help="Override prompt mode. Default: auto from --modality.",
|
||||
)
|
||||
parser.add_argument("--sys-type", type=str, default=None, help="Override system prompt type.")
|
||||
parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.")
|
||||
parser.add_argument(
|
||||
"--stream",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable streaming CoT display; print AR text token-by-token in real time.",
|
||||
)
|
||||
parser.add_argument("--log-stats", action="store_true", default=False)
|
||||
parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
|
||||
parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
|
||||
parser.add_argument(
|
||||
"--diffusion-kv-cache-dtype",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Diffusion attention KV cache dtype, for example 'fp8'. Separate from vLLM --kv-cache-dtype.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--diffusion-kv-cache-skip-steps",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Denoising step selector to keep diffusion KV cache in native dtype, for example '0,1,4-6'.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--diffusion-kv-cache-skip-layers",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Transformer layer selector to keep diffusion KV cache in native dtype, for example '0-2,10'.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--additional-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"JSON object forwarded to Omni/additional_config, for example "
|
||||
'\'{"torchair_graph_config":{"enabled":true}}\'. Different platforms may support different '
|
||||
"configs. Make sure the configs are valid for the platform you are using. "
|
||||
"Contents must be hashable."
|
||||
),
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def parse_additional_config(raw_value: str | None) -> dict | None:
|
||||
"""Parse a JSON string into an additional_config mapping."""
|
||||
if raw_value is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
additional_config = json.loads(raw_value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid --additional-config JSON: {exc}") from exc
|
||||
|
||||
if additional_config is None:
|
||||
return None
|
||||
if not isinstance(additional_config, dict):
|
||||
raise ValueError(f"--additional-config must decode to a JSON object, got {type(additional_config).__name__}")
|
||||
return additional_config
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
additional_config = parse_additional_config(args.additional_config)
|
||||
|
||||
task, default_bot_task = _MODALITY_TASK_MAP[args.modality]
|
||||
if args.bot_task is None:
|
||||
bot_task: str | None = default_bot_task
|
||||
elif args.bot_task == "none":
|
||||
bot_task = None
|
||||
else:
|
||||
bot_task = args.bot_task
|
||||
|
||||
deploy_config = args.deploy_config or _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality]
|
||||
|
||||
omni_kwargs = {
|
||||
"model": args.model,
|
||||
"vae_use_tiling": args.vae_use_tiling,
|
||||
"log_stats": args.log_stats,
|
||||
"init_timeout": args.init_timeout,
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"mode": _MODALITY_MODE[args.modality],
|
||||
"diffusion_kv_cache_dtype": args.diffusion_kv_cache_dtype,
|
||||
"diffusion_kv_cache_skip_steps": args.diffusion_kv_cache_skip_steps,
|
||||
"diffusion_kv_cache_skip_layers": args.diffusion_kv_cache_skip_layers,
|
||||
}
|
||||
|
||||
if additional_config is not None:
|
||||
omni_kwargs["additional_config"] = additional_config
|
||||
omni_kwargs["deploy_config"] = deploy_config
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
|
||||
prompts = args.prompts or ["A cute cat"]
|
||||
input_images: list = []
|
||||
if args.modality in ("img2img", "img2text"):
|
||||
if not args.image_path:
|
||||
raise ValueError(f"--image-path required for {args.modality}, got: {args.image_path}")
|
||||
from PIL import Image
|
||||
|
||||
image_paths = [p.strip() for p in args.image_path.split(",") if p.strip()]
|
||||
if len(image_paths) > MAX_IMAGES_PER_REQUEST:
|
||||
raise ValueError(
|
||||
f"--image-path accepts at most {MAX_IMAGES_PER_REQUEST} images for "
|
||||
f"HunyuanImage-3.0 IT2I, got {len(image_paths)}: {args.image_path}"
|
||||
)
|
||||
for image_path in image_paths:
|
||||
if not os.path.exists(image_path):
|
||||
raise ValueError(f"Image path does not exist: {image_path}")
|
||||
input_images.append(Image.open(image_path).convert("RGB"))
|
||||
if not input_images:
|
||||
raise ValueError(f"--image-path produced no usable paths: {args.image_path!r}")
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
||||
mm_image_payload = (input_images[0] if len(input_images) == 1 else input_images) if input_images else None
|
||||
|
||||
formatted_prompts: list[OmniPromptType] = []
|
||||
for prompt in prompts:
|
||||
build_kwargs: dict = {"task": task, "bot_task": bot_task, "sys_type": args.sys_type}
|
||||
if input_images:
|
||||
build_kwargs["num_images"] = len(input_images)
|
||||
result = build_prompt_tokens(prompt, tokenizer, **build_kwargs)
|
||||
token_ids = result.token_ids
|
||||
effective_sys_type = args.sys_type or resolve_sys_type(bot_task)
|
||||
|
||||
prompt_dict: dict = {
|
||||
"prompt_token_ids": token_ids,
|
||||
"prompt": prompt,
|
||||
"use_system_prompt": effective_sys_type,
|
||||
}
|
||||
if args.modality == "text2img":
|
||||
prompt_dict["modalities"] = ["image"]
|
||||
elif args.modality == "img2img":
|
||||
prompt_dict["modalities"] = ["image"]
|
||||
prompt_dict["multi_modal_data"] = {"image": mm_image_payload}
|
||||
prompt_dict["height"] = input_images[0].height
|
||||
prompt_dict["width"] = input_images[0].width
|
||||
elif args.modality == "img2text":
|
||||
prompt_dict["modalities"] = ["text"]
|
||||
prompt_dict["multi_modal_data"] = {"image": mm_image_payload}
|
||||
else:
|
||||
prompt_dict["modalities"] = ["text"]
|
||||
formatted_prompts.append(prompt_dict)
|
||||
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
|
||||
params_list = list(omni.default_sampling_params_list)
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
if (args.height is None) != (args.width is None):
|
||||
raise ValueError("--height and --width must both be specified or both omitted.")
|
||||
user_specified_size = args.height is not None and args.width is not None
|
||||
if args.modality in ("img2text", "text2text"):
|
||||
ar_image_size = "auto"
|
||||
elif user_specified_size:
|
||||
ar_image_size = f"{args.width}x{args.height}"
|
||||
else:
|
||||
ar_image_size = None
|
||||
ar_stop_token_ids = resolve_stop_token_ids(
|
||||
task=task, bot_task=bot_task, tokenizer=tokenizer, image_size=ar_image_size
|
||||
)
|
||||
print(
|
||||
f"[AR Config] task={task}, bot_task={bot_task}, image_size={ar_image_size}, stop_token_ids={ar_stop_token_ids}"
|
||||
)
|
||||
for sp in params_list:
|
||||
if isinstance(sp, OmniDiffusionSamplingParams):
|
||||
sp.num_inference_steps = args.steps
|
||||
sp.guidance_scale = args.guidance_scale
|
||||
sp.guidance_scale_provided = True
|
||||
if args.seed is not None:
|
||||
sp.seed = args.seed
|
||||
if args.modality == "text2img":
|
||||
sp.height = args.height
|
||||
sp.width = args.width
|
||||
elif hasattr(sp, "stop_token_ids"):
|
||||
sp.stop_token_ids = ar_stop_token_ids
|
||||
# When --stream is set, request DELTA output from the AR stage
|
||||
# so we can display CoT text token-by-token in real time.
|
||||
if args.stream and hasattr(sp, "output_kind"):
|
||||
sp.output_kind = RequestOutputKind.DELTA
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("HunyuanImage-3.0 Generation Configuration:")
|
||||
print(f" Model: {args.model}")
|
||||
print(f" Modality: {args.modality}")
|
||||
print(f" Prompt task: {task}")
|
||||
print(f" Bot task: {bot_task}")
|
||||
print(f" Deploy config: {deploy_config}")
|
||||
print(f" Num stages: {omni.num_stages}")
|
||||
if args.modality in ("text2img", "img2img"):
|
||||
print(f" Inference steps: {args.steps}")
|
||||
print(f" Guidance scale: {args.guidance_scale}")
|
||||
print(f" Seed: {args.seed}")
|
||||
print(f" diffusion_kv_cache_dtype: {args.diffusion_kv_cache_dtype}")
|
||||
print(f" diffusion_kv_cache_skip_steps: {args.diffusion_kv_cache_skip_steps}")
|
||||
print(f" diffusion_kv_cache_skip_layers: {args.diffusion_kv_cache_skip_layers}")
|
||||
if args.modality == "text2img":
|
||||
print(f" Output size: {args.width}x{args.height}")
|
||||
if args.image_path:
|
||||
print(f" Input image: {args.image_path}")
|
||||
if additional_config is not None:
|
||||
print(f" Additional config: {additional_config}")
|
||||
print(f" Prompts: {prompts}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
# When --stream is set, print AR CoT text token-by-token in real time.
|
||||
# Otherwise, collect and print the full AR text once when stage 0 finishes.
|
||||
omni_outputs = omni.generate(
|
||||
prompts=formatted_prompts,
|
||||
sampling_params_list=params_list,
|
||||
py_generator=True,
|
||||
use_tqdm=False,
|
||||
)
|
||||
img_idx = 0
|
||||
for req_output in omni_outputs:
|
||||
ro = req_output
|
||||
stage_id = getattr(req_output, "stage_id", None)
|
||||
|
||||
# AR stage text — each CompletionOutput.text is already a delta when
|
||||
# output_kind=DELTA, so we can print it directly (matching the pattern
|
||||
# in serving_chat.py).
|
||||
if stage_id == 0 and ro and getattr(ro, "outputs", None):
|
||||
for o in ro.outputs:
|
||||
text = getattr(o, "text", "") or ""
|
||||
if text:
|
||||
print(text, end="", flush=True)
|
||||
# Non-streaming: one shot with full text — emit a trailing newline.
|
||||
if not args.stream:
|
||||
print(flush=True)
|
||||
|
||||
# Collect images from diffusion stage
|
||||
images = getattr(req_output, "images", None)
|
||||
if not images and ro and hasattr(ro, "images"):
|
||||
images = ro.images
|
||||
if images:
|
||||
for j, img in enumerate(images):
|
||||
save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
|
||||
img.save(save_path)
|
||||
print(f"\n[Output] Saved image to {save_path}")
|
||||
img_idx += 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -96,6 +96,7 @@ from typing import Any
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.data import logger
|
||||
from vllm_omni.diffusion.utils.image_output import extract_images_from_outputs
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -103,6 +104,8 @@ from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_extras import (
|
||||
build_image_to_image_prompt,
|
||||
get_ar_input_builder,
|
||||
get_ar_tokenizer_validator,
|
||||
get_extra_body_params,
|
||||
get_model_class_name,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
@@ -110,6 +113,91 @@ from vllm_omni.model_extras import (
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def _apply_ar_stage_inputs(
|
||||
ar_input_builder: Any,
|
||||
*,
|
||||
model: str,
|
||||
prompt_text: str,
|
||||
extra_body: dict[str, Any],
|
||||
num_images: int,
|
||||
height: int | None,
|
||||
width: int | None,
|
||||
prompt_dict: dict[str, Any],
|
||||
sampling_params_list: list[Any],
|
||||
text_output: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
validate_tokenizer: Any | None = None,
|
||||
allow_tokenizer_fallback: bool = False,
|
||||
) -> None:
|
||||
"""Apply a model's declared AR-stage inputs to the request in place.
|
||||
|
||||
Mirrors the helper in ``text_to_image.py``: loads the tokenizer for
|
||||
byte-for-byte HF-parity AR tokenization, asks the model's
|
||||
``ar_input_builder`` for the AR prefill + stop tokens, and writes them
|
||||
onto ``prompt_dict`` and the non-diffusion (AR) stage params.
|
||||
Model-agnostic -- only the declared-hook contract is assumed.
|
||||
|
||||
``trust_remote_code`` must be threaded in from the caller's own resolved
|
||||
``--trust-remote-code`` flag -- this is a separate tokenizer load outside
|
||||
the main engine, so it needs the same explicit user opt-in rather than
|
||||
defaulting to trusting whatever ``--model`` was passed.
|
||||
|
||||
``validate_tokenizer``, when the model declares one via
|
||||
``get_ar_tokenizer_validator``, is called on a successfully-loaded real
|
||||
tokenizer -- outside the load's try/except, so a validation failure (a
|
||||
model/tokenizer revision drifting from hardcoded special-token ids)
|
||||
raises instead of being swallowed into the string-prompt fallback.
|
||||
|
||||
``allow_tokenizer_fallback`` (default ``False``, i.e. fail fast): a
|
||||
tokenizer load failure normally raises, since silently degrading to the
|
||||
string-prompt form can produce a request that completes with subtly
|
||||
wrong stop tokens instead of surfacing the real problem (missing
|
||||
``--trust-remote-code``, network/cache issue, etc.). Set ``True`` only
|
||||
for explicit unit tests or a deliberate offline-compat run where the
|
||||
caller has already decided a degraded prompt is acceptable.
|
||||
"""
|
||||
try:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
ar_tokenizer: Any | None = AutoTokenizer.from_pretrained(model, trust_remote_code=trust_remote_code)
|
||||
except Exception as exc: # noqa: BLE001 - re-raised unless the caller opts into the fallback
|
||||
if not allow_tokenizer_fallback:
|
||||
raise
|
||||
logger.warning(f"AR tokenizer load failed ({exc}); falling back to string prompt (no BPE parity).")
|
||||
ar_tokenizer = None
|
||||
|
||||
if ar_tokenizer is not None and validate_tokenizer is not None:
|
||||
validate_tokenizer(ar_tokenizer)
|
||||
|
||||
ar_inputs = ar_input_builder(
|
||||
prompt=prompt_text,
|
||||
tokenizer=ar_tokenizer,
|
||||
extra_body=extra_body,
|
||||
num_images=num_images,
|
||||
height=height,
|
||||
width=width,
|
||||
text_output=text_output,
|
||||
)
|
||||
|
||||
if ar_inputs.prompt_token_ids is not None:
|
||||
prompt_dict["prompt_token_ids"] = ar_inputs.prompt_token_ids
|
||||
elif ar_inputs.prompt is not None:
|
||||
prompt_dict["prompt"] = ar_inputs.prompt
|
||||
if ar_inputs.use_system_prompt:
|
||||
prompt_dict["use_system_prompt"] = ar_inputs.use_system_prompt
|
||||
prompt_dict["modalities"] = ar_inputs.modalities
|
||||
|
||||
# Apply stop_token_ids to exactly the stage(s) the builder names -- not
|
||||
# "whichever stage isn't a diffusion stage," which would misfire on any
|
||||
# future topology with more than one non-diffusion stage.
|
||||
for stage_index in ar_inputs.stage_indices:
|
||||
if stage_index >= len(sampling_params_list):
|
||||
continue
|
||||
stage_params = sampling_params_list[stage_index]
|
||||
if not isinstance(stage_params, OmniDiffusionSamplingParams) and hasattr(stage_params, "stop_token_ids"):
|
||||
stage_params.stop_token_ids = ar_inputs.stop_token_ids
|
||||
|
||||
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
config = json.loads(value)
|
||||
@@ -419,6 +507,22 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Trust and execute custom modeling code from the model repo (required by e.g. HunyuanImage-3.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-tokenizer-fallback",
|
||||
action="store_true",
|
||||
help=(
|
||||
"For models with a declared ar_input_builder (e.g. HunyuanImage-3.0): if loading "
|
||||
"the AR tokenizer fails, degrade to the string-prompt form (no BPE parity) instead "
|
||||
"of failing the run. Off by default -- a tokenizer load failure usually means a "
|
||||
"missing --trust-remote-code or a network/cache issue that's worth surfacing, not "
|
||||
"silently masking with a possibly-wrong prompt."
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -492,6 +596,8 @@ def main():
|
||||
)
|
||||
if args.enforce_eager is not None:
|
||||
omni_kwargs["enforce_eager"] = args.enforce_eager
|
||||
if args.trust_remote_code:
|
||||
omni_kwargs["trust_remote_code"] = True
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
omni = Omni(**omni_kwargs)
|
||||
@@ -585,6 +691,26 @@ def main():
|
||||
if not diffusion_replaced and len(sampling_params_list) == 1:
|
||||
sampling_params_list = [diffusion_params]
|
||||
|
||||
# Models with an AR text stage (e.g. HunyuanImage3 image-editing) declare an
|
||||
# ar_input_builder; build the AR prefill + stop tokens declaratively from the
|
||||
# plain prompt + reference image count + extra_args. Others are untouched.
|
||||
ar_input_builder = get_ar_input_builder(model_class_name)
|
||||
if ar_input_builder is not None:
|
||||
_apply_ar_stage_inputs(
|
||||
ar_input_builder,
|
||||
model=args.model,
|
||||
prompt_text=args.prompt,
|
||||
extra_body=extra_args_from_cli,
|
||||
num_images=len(input_images),
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
prompt_dict=prompt_dict,
|
||||
sampling_params_list=sampling_params_list,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
validate_tokenizer=get_ar_tokenizer_validator(model_class_name),
|
||||
allow_tokenizer_fallback=args.allow_tokenizer_fallback,
|
||||
)
|
||||
|
||||
outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
|
||||
generation_end = time.perf_counter()
|
||||
generation_time = generation_end - generation_start
|
||||
|
||||
@@ -19,23 +19,23 @@ This folder provides several entrypoints for experimenting with text-to-image di
|
||||
|
||||
### Supported Models
|
||||
|
||||
| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) |
|
||||
| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) |
|
||||
| ----- | ----------- | ----------- | ----------------- |
|
||||
| `Qwen/Qwen-Image` | 1024 x 1024 | 60.0 | 53.7 |
|
||||
| `Qwen/Qwen-Image-2512` |1024 x 1024 | 60.0 | 53.7 |
|
||||
| `Qwen/Qwen-Image-2512` | 1024 x 1024 | 60.0 | 53.7 |
|
||||
| `Tongyi-MAI/Z-Image-Turbo` | 1024 x 1024 | 24.8 | 19.2 |
|
||||
| `stepfun-ai/NextStep-1.1` | 512 x 512 | 71.8 | 28.1 |
|
||||
| `meituan-longcat/LongCat-Image` | 1024 x 1024 | 71.2 | 27.3 |
|
||||
| `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 |
|
||||
| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
|
||||
| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 |
|
||||
| `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 |
|
||||
| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 33.9 | 31.4 |
|
||||
| `black-forest-labs/FLUX.1-schnell` | 1024 x 1024 | 33.9 | 31.4 |
|
||||
| `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
|
||||
| `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
|
||||
| `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
|
||||
| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3) | 160 |
|
||||
| `HiDream-I1-Full` | 1024 x 1024 | 63.7 | 57.7 |
|
||||
| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3) | 160 |
|
||||
| `HiDream-I1-Full` | 1024 x 1024 | 63.7 | 57.7 |
|
||||
| `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | 1024 (Raw) / 2048 (Turbo) | — | ~30 |
|
||||
|
||||
!!! info
|
||||
@@ -112,9 +112,9 @@ python text_to_image.py \
|
||||
| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
|
||||
| `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
|
||||
| `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
|
||||
| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
|
||||
| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
|
||||
| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
|
||||
| `--lora-backend` | str | `"peft"` | LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
|
||||
| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0. |
|
||||
| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0. |
|
||||
| `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |
|
||||
|
||||
**NextStep-1.1 specific arguments:**
|
||||
@@ -127,7 +127,7 @@ python text_to_image.py \
|
||||
| `--use-norm` | flag | off | Apply layer normalization to sampled tokens |
|
||||
|
||||
> If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage.
|
||||
|
||||
>
|
||||
> Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
|
||||
|
||||
## More CLI Examples
|
||||
@@ -216,6 +216,26 @@ python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--output /output.png
|
||||
```
|
||||
|
||||
### Anima Single-File Checkpoints
|
||||
|
||||
Anima uses the official model dimensions by default. A checkpoint may provide a JSON file with the same stem (for example, `anima.json` beside `anima.safetensors`) containing `transformer` and `text_conditioner` configuration objects. These override the default dimensions and must match the checkpoint weights. The shared tiny-model tests use this to load small random-weight models. Components in `text_encoder/`, `vae/`, `tokenizer/`, `t5_tokenizer/`, and `scheduler/` beside the checkpoint are discovered automatically.
|
||||
|
||||
To load Anima, point `--model` to the single-file checkpoint path, pass the native pipeline class name using `--model-class-name`, and supply the converted components directory using `--custom-pipeline-args`:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--model /path/to/models/anima-official/split_files/diffusion_models/anima-base-v1.0.safetensors \
|
||||
--model-class-name AnimaPipeline \
|
||||
--custom-pipeline-args '{"components_path": "/path/to/models/anima-components"}' \
|
||||
--prompt "A cinematic close-up of a glass teapot on a wooden table." \
|
||||
--seed 42 \
|
||||
--guidance-scale 4.0 \
|
||||
--num-inference-steps 50 \
|
||||
--height 1024 \
|
||||
--width 1024 \
|
||||
--output anima_output.png
|
||||
```
|
||||
|
||||
### Krea 2
|
||||
|
||||
Krea 2 is published as diffusers-format checkpoints: `krea/Krea-2-Turbo` (few-step distilled) and `krea/Krea-2-Raw`.
|
||||
@@ -336,7 +356,7 @@ python text_to_image.py \
|
||||
|
||||
LoRA adapters must be in PEFT format. A typical adapter directory structure:
|
||||
|
||||
```
|
||||
```text
|
||||
lora_adapter/
|
||||
├── adapter_config.json
|
||||
└── adapter_model.safetensors
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
@@ -24,6 +24,8 @@ from vllm_omni.model_extras import (
|
||||
build_text_to_image_prompt as build_model_text_to_image_prompt,
|
||||
)
|
||||
from vllm_omni.model_extras import (
|
||||
get_ar_input_builder,
|
||||
get_ar_tokenizer_validator,
|
||||
get_extra_body_params,
|
||||
get_model_class_name,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
@@ -56,6 +58,7 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
parse_custom_pipeline_args = functools.partial(parse_json_object, flag_name="--custom-pipeline-args")
|
||||
|
||||
|
||||
def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
|
||||
@@ -378,10 +381,123 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help="Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-class-name",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override the diffusion pipeline class name (e.g. AnimaPipeline).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--custom-pipeline-args",
|
||||
type=parse_custom_pipeline_args,
|
||||
default=None,
|
||||
help='JSON object passed to native/custom pipelines (e.g. \'{"components_path": "/path"}\').',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Trust and execute custom modeling code from the model repo (required by e.g. HunyuanImage-3.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-tokenizer-fallback",
|
||||
action="store_true",
|
||||
help=(
|
||||
"For models with a declared ar_input_builder (e.g. HunyuanImage-3.0): if loading "
|
||||
"the AR tokenizer fails, degrade to the string-prompt form (no BPE parity) instead "
|
||||
"of failing the run. Off by default -- a tokenizer load failure usually means a "
|
||||
"missing --trust-remote-code or a network/cache issue that's worth surfacing, not "
|
||||
"silently masking with a possibly-wrong prompt."
|
||||
),
|
||||
)
|
||||
current_omni_platform.pre_register_and_update(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _apply_ar_stage_inputs(
|
||||
ar_input_builder: Any,
|
||||
*,
|
||||
model: str,
|
||||
prompt_text: str,
|
||||
extra_body: dict[str, Any],
|
||||
num_images: int,
|
||||
height: int | None,
|
||||
width: int | None,
|
||||
prompt_dict: dict[str, Any],
|
||||
sampling_params_list: list[Any],
|
||||
text_output: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
validate_tokenizer: Any | None = None,
|
||||
allow_tokenizer_fallback: bool = False,
|
||||
) -> None:
|
||||
"""Apply a model's declared AR-stage inputs to the request in place.
|
||||
|
||||
Loads the model tokenizer (for byte-for-byte HF-parity segment
|
||||
tokenization) and asks the model's ``ar_input_builder`` for the AR
|
||||
prefill + stop tokens, then writes them onto ``prompt_dict`` and the
|
||||
non-diffusion (AR) stage sampling params. Kept model-agnostic: the
|
||||
example only knows the declared-hook contract.
|
||||
|
||||
``trust_remote_code`` must be threaded in from the caller's own resolved
|
||||
``--trust-remote-code`` flag -- this is a separate tokenizer load outside
|
||||
the main engine, so it needs the same explicit user opt-in rather than
|
||||
defaulting to trusting whatever ``--model`` was passed.
|
||||
|
||||
``validate_tokenizer``, when the model declares one via
|
||||
``get_ar_tokenizer_validator``, is called on a successfully-loaded real
|
||||
tokenizer -- outside the load's try/except, so a validation failure (a
|
||||
model/tokenizer revision drifting from hardcoded special-token ids)
|
||||
raises instead of being swallowed into the string-prompt fallback.
|
||||
|
||||
``allow_tokenizer_fallback`` (default ``False``, i.e. fail fast): a
|
||||
tokenizer load failure normally raises, since silently degrading to the
|
||||
string-prompt form can produce a request that completes with subtly
|
||||
wrong stop tokens instead of surfacing the real problem (missing
|
||||
``--trust-remote-code``, network/cache issue, etc.). Set ``True`` only
|
||||
for explicit unit tests or a deliberate offline-compat run where the
|
||||
caller has already decided a degraded prompt is acceptable.
|
||||
"""
|
||||
try:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
ar_tokenizer: Any | None = AutoTokenizer.from_pretrained(model, trust_remote_code=trust_remote_code)
|
||||
except Exception as exc: # noqa: BLE001 - re-raised unless the caller opts into the fallback
|
||||
if not allow_tokenizer_fallback:
|
||||
raise
|
||||
logger.warning(f"AR tokenizer load failed ({exc}); falling back to string prompt (no BPE parity).")
|
||||
ar_tokenizer = None
|
||||
|
||||
if ar_tokenizer is not None and validate_tokenizer is not None:
|
||||
validate_tokenizer(ar_tokenizer)
|
||||
|
||||
ar_inputs = ar_input_builder(
|
||||
prompt=prompt_text,
|
||||
tokenizer=ar_tokenizer,
|
||||
extra_body=extra_body,
|
||||
num_images=num_images,
|
||||
height=height,
|
||||
width=width,
|
||||
text_output=text_output,
|
||||
)
|
||||
|
||||
if ar_inputs.prompt_token_ids is not None:
|
||||
prompt_dict["prompt_token_ids"] = ar_inputs.prompt_token_ids
|
||||
elif ar_inputs.prompt is not None:
|
||||
prompt_dict["prompt"] = ar_inputs.prompt
|
||||
if ar_inputs.use_system_prompt:
|
||||
prompt_dict["use_system_prompt"] = ar_inputs.use_system_prompt
|
||||
prompt_dict["modalities"] = ar_inputs.modalities
|
||||
|
||||
# Apply stop_token_ids to exactly the stage(s) the builder names -- not
|
||||
# "whichever stage isn't a diffusion stage," which would misfire on any
|
||||
# future topology with more than one non-diffusion stage.
|
||||
for stage_index in ar_inputs.stage_indices:
|
||||
if stage_index >= len(sampling_params_list):
|
||||
continue
|
||||
stage_params = sampling_params_list[stage_index]
|
||||
if not isinstance(stage_params, OmniDiffusionSamplingParams) and hasattr(stage_params, "stop_token_ids"):
|
||||
stage_params.stop_token_ids = ar_inputs.stop_token_ids
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
@@ -456,7 +572,6 @@ def main():
|
||||
"vae_patch_parallel_size": args.vae_patch_parallel_size,
|
||||
"enable_expert_parallel": args.enable_expert_parallel,
|
||||
"enable_cpu_offload": args.enable_cpu_offload,
|
||||
"mode": "text-to-image",
|
||||
"log_stats": args.log_stats,
|
||||
"enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
|
||||
"profiler_config": args.profiler_config,
|
||||
@@ -470,11 +585,17 @@ def main():
|
||||
omni_kwargs["tensor_parallel_size"] = args.tensor_parallel_size
|
||||
if args.enforce_eager is not None:
|
||||
omni_kwargs["enforce_eager"] = args.enforce_eager
|
||||
if args.trust_remote_code:
|
||||
omni_kwargs["trust_remote_code"] = True
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
if use_nextstep:
|
||||
if args.model_class_name:
|
||||
omni_kwargs["model_class_name"] = args.model_class_name
|
||||
elif use_nextstep:
|
||||
# NextStep-1.1 requires explicit pipeline class
|
||||
omni_kwargs["model_class_name"] = "NextStep11Pipeline"
|
||||
if args.custom_pipeline_args is not None:
|
||||
omni_kwargs["custom_pipeline_args"] = args.custom_pipeline_args
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
@@ -510,6 +631,10 @@ def main():
|
||||
print(f" LoRA: scale={args.lora_scale}")
|
||||
if args.deploy_config:
|
||||
print(f" deploy-config: {args.deploy_config}")
|
||||
if args.model_class_name:
|
||||
print(f" Model class name: {args.model_class_name}")
|
||||
if args.custom_pipeline_args is not None:
|
||||
print(f" Custom pipeline args: {args.custom_pipeline_args}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
# Build LoRA request when --lora-path is set
|
||||
@@ -614,6 +739,30 @@ def main():
|
||||
if not diffusion_replaced and len(sampling_params_list) == 1:
|
||||
sampling_params_list = [diffusion_params]
|
||||
|
||||
# Models with an AR text stage (e.g. HunyuanImage3) declare an
|
||||
# ar_input_builder. When present, build the AR prefill token-ids and AR
|
||||
# stop-token-ids declaratively from the plain prompt + extra_body, so this
|
||||
# example stays model-agnostic. Models without one are untouched.
|
||||
ar_input_builder = get_ar_input_builder(model_class_name)
|
||||
# A model can also be deployed with only its diffusion stage. Keep those
|
||||
# requests on the string-prompt path, as in the single-stage images API.
|
||||
has_ar_stage = any(not isinstance(params, OmniDiffusionSamplingParams) for params in sampling_params_list)
|
||||
if ar_input_builder is not None and has_ar_stage:
|
||||
_apply_ar_stage_inputs(
|
||||
ar_input_builder,
|
||||
model=args.model,
|
||||
prompt_text=args.prompt,
|
||||
extra_body=user_extra,
|
||||
num_images=0,
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
prompt_dict=prompt_dict,
|
||||
sampling_params_list=sampling_params_list,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
validate_tokenizer=get_ar_tokenizer_validator(model_class_name),
|
||||
allow_tokenizer_fallback=args.allow_tokenizer_fallback,
|
||||
)
|
||||
|
||||
outputs = omni.generate(prompt_dict, sampling_params_list=sampling_params_list)
|
||||
|
||||
generation_end = time.perf_counter()
|
||||
|
||||
@@ -13,11 +13,13 @@ list of supported architectures across all modalities, see
|
||||
## Supported Models
|
||||
|
||||
| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
|
||||
|---|---|---|---|---|---|---|
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| AuK | `tencent/AuK`, `tencent/AuK-Flash` (assembled with `tools/prepare_auk_checkpoint.py`) | 2 (thinker encoder + DiT) | ✓ | — | instruction-driven editing, enhancement, separation; no example script (use `vllm_omni.model_extras.auk`) | 24 kHz |
|
||||
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
|
||||
| Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | 2 (talker + codec) | ✓ | ✓ (async chunk) | voice design / voice direction (`--instruction`) | 24 kHz |
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
|
||||
| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz |
|
||||
| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning later) | ✓ (online) | zero-shot | 22.05 kHz |
|
||||
| GLM-TTS | `zai-org/GLM-TTS` | 2 (AR + DiT) | ✓ (required) | ✓ | — | 24 kHz |
|
||||
| Ming-omni-tts | `inclusionAI/Ming-omni-tts-0.5B` | 2 (AR + audio VAE) | ✓ | ✓ | style / IP / dialect / TTA / podcast | 44.1 kHz |
|
||||
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
|
||||
@@ -25,7 +27,6 @@ list of supported architectures across all modalities, see
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
|
||||
| dots.tts | `dots-studio/dots.tts-soar` | single (native AR) | — (text-only) | ✓ (online) | no-reference synthesis | 48 kHz |
|
||||
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
|
||||
| IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
|
||||
@@ -52,32 +53,38 @@ codec: a slow AR transformer predicts one semantic token per codec frame, and a
|
||||
4-layer fast AR predicts that frame's remaining 9 codebooks.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
No extra packages: the codec architecture ships in tree and its weights
|
||||
(`codec.pth`) come with the checkpoint.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
--text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
--text "Welcome to Audio8 TTS." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript of the reference recording."
|
||||
```
|
||||
|
||||
The reference transcript must match what is actually spoken in the clip;
|
||||
a mismatch degrades both stability and speaker similarity.
|
||||
|
||||
### Streaming
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
--text "Welcome to Audio8 TTS." --streaming
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
|
||||
- Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
|
||||
- Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
|
||||
@@ -90,6 +97,7 @@ python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
- `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
|
||||
|
||||
### Measured behaviour (1x H20, shared GPU)
|
||||
|
||||
`--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
|
||||
chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
|
||||
smoke-test magnitudes, not a benchmark.
|
||||
@@ -101,12 +109,14 @@ smoke-test magnitudes, not a benchmark.
|
||||
2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
```bash
|
||||
uv pip install -e .
|
||||
# Includes soundfile, onnxruntime, x-transformers, einops via requirements.
|
||||
```
|
||||
|
||||
Download the model snapshot:
|
||||
|
||||
```python
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
|
||||
@@ -114,15 +124,18 @@ snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
|
||||
```
|
||||
|
||||
If your downloaded checkpoint lacks `config.json`, add it:
|
||||
|
||||
```json
|
||||
{
|
||||
"model_type": "cosyvoice3",
|
||||
"architectures": ["CosyVoice3Model"]
|
||||
}
|
||||
```
|
||||
|
||||
This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
|
||||
--model pretrained_models/Fun-CosyVoice3-0.5B \
|
||||
@@ -130,9 +143,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
If `--ref-audio` is omitted, the script downloads the upstream
|
||||
[`zero_shot_prompt.wav`](https://github.com/FunAudioLLM/CosyVoice/blob/main/asset/zero_shot_prompt.wav) from the CosyVoice repo into the current directory.
|
||||
To use your own clip, pass `--ref-audio /path/to/reference.wav`, and modify `--prompt-text` correspondingly.
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
|
||||
--model pretrained_models/Fun-CosyVoice3-0.5B \
|
||||
@@ -142,9 +157,11 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` on `vllm serve` to switch to the legacy synchronous path.
|
||||
|
||||
### Notes
|
||||
|
||||
- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
|
||||
- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
|
||||
|
||||
@@ -155,6 +172,7 @@ Streaming is enabled by default via `async_chunk: true` in `vllm_omni/deploy/cos
|
||||
2-stage TTS pipeline (AR + DiT flow-matching) at 24 kHz. Every request requires reference audio and its transcript for zero-shot voice cloning.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
|
||||
--model zai-org/GLM-TTS \
|
||||
@@ -165,12 +183,14 @@ python examples/offline_inference/text_to_speech/glm_tts/end2end.py \
|
||||
```
|
||||
|
||||
### Architecture
|
||||
```
|
||||
|
||||
```text
|
||||
Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24 kHz)
|
||||
(Llama-based) (32k vocab) (Flow Matching)
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- `--ref-audio` and `--ref-text` are **required** together; GLM-TTS does not support text-only synthesis.
|
||||
- Reference audio should be 3-10 seconds.
|
||||
- First run may be slow due to lazy loading of WhisperVQ tokenizer and CampPlus ONNX speaker embedder.
|
||||
@@ -185,12 +205,14 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
|
||||
No extra packages are required; the DAC codec is vendored in vLLM-Omni.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
|
||||
--text "Hello, this is a test of the Fish Speech text to speech system."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
|
||||
--text "Hello, this is a cloned voice." \
|
||||
@@ -199,14 +221,17 @@ python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
|
||||
--text "Hello, this is a streaming test." \
|
||||
--streaming
|
||||
```
|
||||
|
||||
Streaming requires `async_chunk: true` in the deploy config.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 44.1 kHz mono WAV.
|
||||
- DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
|
||||
|
||||
@@ -217,6 +242,7 @@ Streaming requires `async_chunk: true` in the deploy config.
|
||||
Dense 0.5B two-stage TTS pipeline (`AR + flow` + audio VAE) at 44.1 kHz. The example covers style, IP voice, music-only generation, text-to-audio events, emotion, dialect, zero-shot cloning, podcast, speech+BGM, and speech+environment-sound cases.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
--case style \
|
||||
@@ -225,6 +251,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
--case zero_shot \
|
||||
@@ -235,6 +262,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
--case basic \
|
||||
@@ -245,6 +273,7 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- `style`, `ip`, `bgm`, and `tta` do not require reference audio.
|
||||
- Reference-audio cases use `--ref-audio`; `zero_shot` also requires `--ref-text`.
|
||||
- `podcast` uses multiple references via `--ref-audio-paths`.
|
||||
@@ -257,14 +286,17 @@ python examples/offline_inference/text_to_speech/ming_tts/end2end.py \
|
||||
Standalone talker-only deployment of Ming-flash-omni-2.0 at 44.1 kHz. Voice is controlled through caption fields (`风格` / `IP` / `语速`/`基频`/`音量`) rather than reference audio.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
The example calls into `vllm_omni.model_executor.models.ming_flash_omni.prompt_utils` for the default prompt and instruction builder; no extra pip install on top of the base vLLM-Omni install.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
|
||||
```
|
||||
|
||||
### Cases
|
||||
|
||||
```bash
|
||||
# ASMR-style whisper (caption-driven)
|
||||
python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py --case style
|
||||
@@ -279,6 +311,7 @@ python examples/offline_inference/text_to_speech/ming_flash_omni_tts/end2end.py
|
||||
Override the default text per case with `--text`, write to a custom path with `--output`.
|
||||
|
||||
### Notes
|
||||
|
||||
- Talker-only deployment — for the multimodal Ming-flash-omni example, see [`examples/offline_inference/ming_flash_omni/`](../../ming_flash_omni/).
|
||||
- Deploy config: `vllm_omni/deploy/ming_flash_omni_tts.yaml` (single GPU, `enforce_eager`, `max_num_seqs: 1`).
|
||||
- Decode defaults from the Ming cookbook: `max_decode_steps=200`, `cfg=2.0`, `sigma=0.25`, `temperature=0.0`, `use_zero_spk_emb=True`.
|
||||
@@ -292,6 +325,7 @@ Single-stage 0.1B AR LM + MOSS-Audio-Tokenizer-Nano codec at 48 kHz mono (mixed
|
||||
> **No built-in speaker presets.** `--ref-audio` is required on every call. Default `--mode voice_clone` matches upstream's recommended workflow; `--mode continuation` is exposed for completeness but upstream's continuation-with-prompt path emits very short / near-silent output, so it is rarely useful in practice. Sample reference clips ship in the upstream repo under [`assets/audio/`](https://github.com/OpenMOSS/MOSS-TTS-Nano/tree/main/assets/audio) (e.g. `zh_1.wav`, `en_2.wav`, `jp_2.wav`).
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
# Fetch a sample reference clip (one-off, user-scoped cache).
|
||||
REF_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/moss-tts-nano"
|
||||
@@ -303,9 +337,11 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
|
||||
--text "你好,这是MOSS-TTS-Nano的语音合成演示。" \
|
||||
--ref-audio "$REF_DIR/zh_1.wav"
|
||||
```
|
||||
|
||||
The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
|
||||
|
||||
### Reproducible runs
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
|
||||
--text "Deterministic test." \
|
||||
@@ -314,6 +350,7 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 48 kHz mono WAV (the tokenizer is internally stereo at 48 kHz; the wrapper averages to mono before reaching the engine).
|
||||
- Deploy config: `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded; override with `--deploy-config`).
|
||||
- Default `--max-new-frames 375` ≈ 14 s of audio; raise for longer outputs.
|
||||
@@ -327,12 +364,15 @@ python examples/offline_inference/text_to_speech/moss_tts_nano/end2end.py \
|
||||
Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
```bash
|
||||
huggingface-cli download k2-fsa/OmniVoice
|
||||
```
|
||||
|
||||
Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
|
||||
|
||||
### Quick start (auto voice)
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
--model k2-fsa/OmniVoice \
|
||||
@@ -340,6 +380,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
--model k2-fsa/OmniVoice \
|
||||
@@ -349,6 +390,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
```
|
||||
|
||||
### Voice design
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
--model k2-fsa/OmniVoice \
|
||||
@@ -357,6 +399,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
```
|
||||
|
||||
### Language hint
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
--model k2-fsa/OmniVoice \
|
||||
@@ -365,6 +408,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
```
|
||||
|
||||
### Seed for Reproducibility
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
--model k2-fsa/OmniVoice \
|
||||
@@ -373,6 +417,7 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
|
||||
- Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
|
||||
|
||||
@@ -383,13 +428,16 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
|
||||
3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
|
||||
|
||||
```bash
|
||||
pip uninstall onnxruntime
|
||||
pip install onnxruntime-rocm
|
||||
```
|
||||
|
||||
### Task variants
|
||||
|
||||
- `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
|
||||
- `VoiceDesign`: text + descriptive instruction designs a new voice.
|
||||
- `Base`: voice cloning from reference audio + transcript.
|
||||
@@ -413,22 +461,27 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-ty
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
|
||||
--query-type CustomVoice \
|
||||
--streaming \
|
||||
--output-dir /tmp/out_stream
|
||||
```
|
||||
|
||||
Streaming requires `async_chunk: true` in the deploy config.
|
||||
|
||||
### Word Timestamps
|
||||
|
||||
Word-level timestamps are currently a serving-path feature: launch
|
||||
`vllm-omni serve` with `--forced-aligner` and request `word_timestamps`
|
||||
(see `examples/online_serving/text_to_speech/README.md`). An offline
|
||||
example is not provided in this release.
|
||||
|
||||
### Batched decoding
|
||||
|
||||
The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, create an overlay based on `vllm_omni/deploy/qwen3_tts.yaml` and pass it with `--deploy-config`:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
|
||||
--query-type CustomVoice \
|
||||
@@ -436,9 +489,11 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
|
||||
--batch-size 4 \
|
||||
--deploy-config /path/to/qwen3_tts_batched.yaml
|
||||
```
|
||||
|
||||
`--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
|
||||
|
||||
### Notes
|
||||
|
||||
- Run `--help` for the full argument surface.
|
||||
- See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
|
||||
|
||||
@@ -449,6 +504,7 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
|
||||
Single-stage native AR TTS at 22.05 kHz. Pipeline: `Qwen3.5 backbone → 32 FSQ codebook heads (one frame/step) → NeMo NanoCodec`. Backbone runs under vLLM paged attention; the 32-head sampling + learned embedding feedback ride the native-AR runner hooks. `enforce_eager` (CUDA graph is a perf follow-up).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
The NanoCodec decoder needs NeMo (NVIDIA Open Model License), installed separately:
|
||||
|
||||
```bash
|
||||
@@ -478,19 +534,22 @@ failure rather than an environment one. The PyTorch sampler it falls back to cos
|
||||
here.
|
||||
|
||||
### Quick start (zero-shot, default voice)
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/gepard/end2end.py \
|
||||
--text "Hello, this is a Gepard demo."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Reference-audio cloning (`ref_compressor` speaker prefix) is a follow-up PR.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 22.05 kHz mono WAV.
|
||||
- First run downloads two checkpoints, not one: the model itself, and the NanoCodec decoder it names in `codec_id` (`nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps` by default). Both are fetched automatically; expect the first startup to be correspondingly slower.
|
||||
- Offline only for now; the `/v1/audio/speech` adapter is a follow-up PR.
|
||||
- Deploy config: `vllm_omni/deploy/gepard.yaml` (the example's default; copy it and pass `--deploy-config` to change it).
|
||||
- Offline and online: `examples/offline_inference/text_to_speech/gepard/end2end.py` and `/v1/audio/speech` (see the [online Gepard section](../../online_serving/text_to_speech/README.md#gepard-10)).
|
||||
- Deploy config: `vllm_omni/deploy/gepard.yaml` (the example's default; copy it and pass `--deploy-config` to change it). Online serving **must** pass `--deploy-config vllm_omni/deploy/gepard.yaml` because the checkpoint self-identifies as `qwen3_5_text`.
|
||||
- Generation length and reproducibility are stage settings, not script flags. One output token is one audio frame, so `max_tokens` in the YAML is the frame budget; `seed` makes the in-model 32-head sampling reproducible. The script deliberately passes no `SamplingParams`: one supplied by a caller replaces the stage defaults wholesale rather than merging, which would drop the pipeline's stop token and run every request to `max_tokens`.
|
||||
- Text length: short texts are repeated internally to match the training layout (the checkpoint's `text_repetition` block); the upper bound is the stage's `max_model_len`, enforced by the engine. Empty text is rejected rather than voiced.
|
||||
- `VLLM_GEPARD_GREEDY=1` swaps the 32-head Gumbel-max sampling for argmax, for reproducible comparisons that do not depend on a seed.
|
||||
@@ -500,6 +559,7 @@ Not yet — PR1 is zero-shot only (the learned `null_prefix` default voice). Ref
|
||||
Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
```bash
|
||||
pip install voxcpm
|
||||
# or, for a local source checkout:
|
||||
@@ -507,6 +567,7 @@ export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
|
||||
```
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
--model openbmb/VoxCPM2 \
|
||||
@@ -514,7 +575,9 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
--text "Hello, this is a voice clone demo." \
|
||||
@@ -523,39 +586,16 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [`examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py`](../../online_serving/text_to_speech/voxcpm2/gradio_demo.py) for an AudioWorklet-based gapless streaming player; the offline `end2end.py` script does not expose a streaming path.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 48 kHz mono WAV.
|
||||
- Deploy config: `vllm_omni/deploy/voxcpm2.yaml` (auto-loaded by HF `model_type`).
|
||||
|
||||
---
|
||||
|
||||
## dots.tts
|
||||
|
||||
Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR loopback → AudioVAE (streaming decode)`. Same "vLLM-native base LM + side-path computation" pattern as VoxCPM2, with a plain Qwen2 backbone instead of MiniCPM4 and no FSQ / residual LM stage.
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/dots_tts/end2end.py \
|
||||
--model dots-studio/dots.tts-soar \
|
||||
--text "Hello, this is a test of dots TTS running on vLLM Omni."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
Not wired in this release. Generation is text-only and does not consume reference audio, a named voice, or a precomputed speaker embedding. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet.
|
||||
|
||||
### Streaming
|
||||
The AudioVAE decoder emits incremental audio through its internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`). Online streaming is exposed through the OpenAI-compatible `/v1/audio/speech` endpoint with `stream=true`; see the [online serving guide](../../online_serving/text_to_speech/README.md#dotstts). The offline `end2end.py` script returns the consolidated waveform.
|
||||
|
||||
### Notes
|
||||
- Output: 48 kHz mono WAV.
|
||||
- Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`).
|
||||
- Checkpoints: `dots-studio/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet.
|
||||
- Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap.
|
||||
|
||||
---
|
||||
|
||||
## IndexTTS-2 and IndexTTS-2.5
|
||||
|
||||
Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker. IndexTTS-2 Stage 1 uses RepCodec semantic embeddings, GPT latent, S2Mel CFM/DiT, and BigVGAN. IndexTTS-2.5 replaces RepCodec with EnhancedCodec and uses the official code-only path (`use_gpt_latent=false`). Every request requires reference audio for zero-shot voice cloning.
|
||||
@@ -590,6 +630,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
```
|
||||
|
||||
### Emotion control
|
||||
|
||||
```bash
|
||||
# Emotion from reference audio
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
@@ -614,6 +655,7 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- `--ref-audio` is **required** — neither version provides text-only synthesis.
|
||||
- IndexTTS-2.5 Stage 0 (AR Talker) generates mel codes from text + reference audio using plain vLLM sampling. It does not reproduce the official default `num_beams=3` beam search. For parity comparisons, run upstream with `num_beams=1`; output quality can differ from the official beam-search result.
|
||||
- IndexTTS-2.5 accepts language codes such as `zh`, `en`, `zhen` (mixed Chinese/English), `ja`, and `yue`. `Mandarin` is a vLLM-Omni convenience alias for `zh`. Japanese (`ja`) uses `fugashi` tokenization and produces audio, but it does not automatically expand numbers, dates, or percentages; write those inputs as readable Japanese text before calling the model. Use `--no-text-normalization` only when the input is already normalized.
|
||||
@@ -629,12 +671,15 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Latest `mistral_common` with `SpeechRequest` support:
|
||||
|
||||
```bash
|
||||
pip install -e /path/to/mistral-common # or upgrade from PyPI when available
|
||||
```
|
||||
|
||||
### Quick start (voice preset)
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
|
||||
--write-audio --voice cheerful_female \
|
||||
@@ -643,6 +688,7 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
|
||||
```
|
||||
|
||||
### Voice cloning (capability gated upstream)
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
|
||||
--write-audio \
|
||||
@@ -652,15 +698,48 @@ python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
|
||||
```
|
||||
|
||||
### Streaming + concurrency
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
|
||||
--num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
|
||||
--model mistralai/Voxtral-4B-TTS-2603 \
|
||||
--text "..."
|
||||
```
|
||||
|
||||
Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
|
||||
|
||||
### Notes
|
||||
|
||||
- `--num-prompts N` replicates the prompt for performance measurement.
|
||||
- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
|
||||
- Run `--help` for the full argument surface.
|
||||
|
||||
---
|
||||
|
||||
## Breeze-TTS-2
|
||||
|
||||
Two-stage AR TTS (T5Gemma2 + Qwen3 talker with a depth decoder → bundled Qwen3-TTS codec) at 24 kHz mono. The prompt builder picks one of four templates automatically: plain, voice design (`--instruction`), clone (`--ref-audio` + `--ref-text`), or voice direction (reference + `--instruction`).
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/breeze_tts_2/end2end.py \
|
||||
--model BreezeBlue/Breeze-TTS-2 \
|
||||
--text "Hello, this is Breeze TTS 2 running on vLLM Omni."
|
||||
```
|
||||
|
||||
### Voice direction
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/breeze_tts_2/end2end.py \
|
||||
--text "We need to discuss what happened last night." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript of the reference audio." \
|
||||
--instruction "Speak slowly with a restrained, serious tone."
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 24 kHz mono WAV; `--max-new-tokens` caps generated frames (80 ms each).
|
||||
- Speaker tags `--voice S0`..`S9` (default `S0`) select the timbre in non-reference modes; with `--ref-audio` the timbre comes from the reference clip.
|
||||
- Deploy config: `vllm_omni/deploy/breeze_tts_2.yaml` (auto-loaded by HF `model_type`); see [`recipes/BreezeBlue/Breeze-TTS-2.md`](../../../recipes/BreezeBlue/Breeze-TTS-2.md) for the online-serving recipe.
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Offline Breeze-TTS-2 inference example (two-stage talker + codec pipeline).
|
||||
|
||||
Uses the async-chunk deploy config (vllm_omni/deploy/breeze_tts_2.yaml).
|
||||
Reference checkpoint: BreezeBlue/Breeze-TTS-2.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from transformers import AutoConfig
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.breeze_tts_2.prompt_builder import (
|
||||
BreezeTTS2PromptBuilder,
|
||||
)
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SAMPLE_RATE = 24_000
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Offline Breeze-TTS-2 inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="BreezeBlue/Breeze-TTS-2",
|
||||
help="Breeze-TTS-2 model path or HuggingFace repo ID.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="Hello, this is Breeze TTS 2 running on vLLM Omni.",
|
||||
help="Text to synthesize.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--voice",
|
||||
type=str,
|
||||
default="S0",
|
||||
help="Speaker tag such as S0..S9.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instruction",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional style instruction (voice design / voice direction).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional reference wav path for cloning (requires --ref-text).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Exact transcript of the reference audio.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-new-tokens",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Cap on generated codec frames (one frame is 80 ms of audio).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default="output_audio",
|
||||
help="Directory for output WAV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override the deploy config path. If unset, auto-loads "
|
||||
"vllm_omni/deploy/breeze_tts_2.yaml based on the HF model_type.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_template(args) -> str:
|
||||
if args.ref_audio is not None:
|
||||
return "ref_edit_tata" if args.instruction else "ref_clone_tata"
|
||||
return "tts_instruction" if args.instruction else "tts_plain"
|
||||
|
||||
|
||||
def reference_payload(ref_audio_path: str | None) -> dict:
|
||||
"""Load the reference clip into the payload keys the prompt builder reads.
|
||||
|
||||
``BreezeTTS2PromptBuilder`` consumes ``ref_audio`` (a waveform) plus
|
||||
``ref_audio_sample_rate``; a bare path field is not read.
|
||||
"""
|
||||
if ref_audio_path is None:
|
||||
return {}
|
||||
waveform, sample_rate = sf.read(ref_audio_path, dtype="float32", always_2d=True)
|
||||
# Down-mix to the single 1-D clip the reference encoder expects.
|
||||
waveform = waveform.mean(axis=1)
|
||||
return {"ref_audio": waveform, "ref_audio_sample_rate": int(sample_rate)}
|
||||
|
||||
|
||||
def build_prompt(args) -> dict:
|
||||
config = AutoConfig.from_pretrained(args.model, trust_remote_code=True)
|
||||
reference_encoder = None
|
||||
if args.ref_audio is not None:
|
||||
from vllm_omni.model_executor.models.breeze_tts_2.audio_tokenizer import (
|
||||
BreezeReferenceAudioTokenizer,
|
||||
)
|
||||
|
||||
reference_encoder = BreezeReferenceAudioTokenizer.from_pretrained(
|
||||
args.model,
|
||||
num_codebooks=int(getattr(config, "num_codebooks", 16)),
|
||||
device_map="cpu",
|
||||
)
|
||||
builder = BreezeTTS2PromptBuilder.from_pretrained(
|
||||
args.model,
|
||||
config,
|
||||
reference_audio_encoder=reference_encoder,
|
||||
)
|
||||
payload = {
|
||||
"text": args.text,
|
||||
"speaker": args.voice,
|
||||
**reference_payload(args.ref_audio),
|
||||
}
|
||||
if args.instruction:
|
||||
payload["instruction"] = args.instruction
|
||||
if args.ref_text:
|
||||
payload["ref_text"] = args.ref_text
|
||||
prompt = builder.build(payload, template=resolve_template(args))
|
||||
if args.max_new_tokens is not None:
|
||||
prompt["additional_information"]["breeze_max_new_frames"] = int(args.max_new_tokens)
|
||||
return prompt
|
||||
|
||||
|
||||
def extract_audio(multimodal_output: dict) -> torch.Tensor:
|
||||
"""Extract the final complete audio tensor from multimodal output.
|
||||
|
||||
The output processor concatenates per-step delta tensors under
|
||||
``model_outputs``. Falls back to ``audio`` for backwards compat.
|
||||
"""
|
||||
audio = multimodal_output.get("model_outputs")
|
||||
if audio is None:
|
||||
audio = multimodal_output.get("audio")
|
||||
if audio is None:
|
||||
raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")
|
||||
|
||||
if isinstance(audio, list):
|
||||
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
|
||||
if not valid:
|
||||
raise ValueError("Audio list is empty or all elements are None.")
|
||||
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
||||
|
||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
engine = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
)
|
||||
|
||||
prompt = build_prompt(args)
|
||||
|
||||
print(f"Model : {args.model}")
|
||||
print(f"Text : {args.text}")
|
||||
print(f"Template : {resolve_template(args)}")
|
||||
print(f"Output dir : {output_dir}")
|
||||
print(f"Prompt len : {len(prompt['prompt_token_ids'])} tokens")
|
||||
|
||||
t_start = time.perf_counter()
|
||||
outputs = engine.generate([prompt])
|
||||
elapsed = time.perf_counter() - t_start
|
||||
|
||||
request_output = outputs[0]
|
||||
mm = request_output.outputs[0].multimodal_output
|
||||
audio = extract_audio(mm)
|
||||
|
||||
duration = audio.numel() / SAMPLE_RATE
|
||||
rtf = elapsed / duration if duration > 0 else float("inf")
|
||||
|
||||
output_path = output_dir / "output.wav"
|
||||
sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV")
|
||||
|
||||
print(f"Saved : {output_path}")
|
||||
print(f"Duration : {duration:.2f}s")
|
||||
print(f"Inference : {elapsed:.2f}s")
|
||||
print(f"RTF : {rtf:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
main()
|
||||
@@ -1,125 +0,0 @@
|
||||
"""Offline dots.tts inference example (single-stage native AR pipeline).
|
||||
|
||||
Uses the single-stage native AR config (vllm_omni/deploy/dots_tts.yaml).
|
||||
Reference checkpoint: dots-studio/dots.tts-soar.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SAMPLE_RATE = 48_000
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Offline dots.tts native AR inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="dots-studio/dots.tts-soar",
|
||||
help="dots.tts model path or HuggingFace repo ID.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="Hello, this is a test of dots TTS running on vLLM Omni.",
|
||||
help="Text to synthesize.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default="output_audio",
|
||||
help="Directory for output WAV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override the deploy config path. If unset, auto-loads "
|
||||
"vllm_omni/deploy/dots_tts.yaml based on the HF model_type.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def extract_audio(multimodal_output: dict) -> torch.Tensor:
|
||||
"""Extract the final complete audio tensor from multimodal output.
|
||||
|
||||
The output processor concatenates per-step delta tensors under
|
||||
``model_outputs``. Falls back to ``audio`` for backwards compat.
|
||||
"""
|
||||
audio = multimodal_output.get("model_outputs")
|
||||
if audio is None:
|
||||
audio = multimodal_output.get("audio")
|
||||
if audio is None:
|
||||
raise ValueError(f"No audio key in multimodal_output: {list(multimodal_output.keys())}")
|
||||
|
||||
if isinstance(audio, list):
|
||||
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
|
||||
if not valid:
|
||||
raise ValueError("Audio list is empty or all elements are None.")
|
||||
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
||||
|
||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
engine = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
)
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.model_executor.models.dots_tts.dots_tts_prompt import (
|
||||
build_dots_tts_prompt,
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
||||
|
||||
prompt = build_dots_tts_prompt(
|
||||
tokenizer=tokenizer,
|
||||
text=args.text,
|
||||
)
|
||||
|
||||
print(f"Model : {args.model}")
|
||||
print(f"Text : {args.text}")
|
||||
print(f"Output dir : {output_dir}")
|
||||
print(f"Prompt len : {len(prompt['prompt_token_ids'])} tokens")
|
||||
|
||||
t_start = time.perf_counter()
|
||||
outputs = engine.generate([prompt])
|
||||
elapsed = time.perf_counter() - t_start
|
||||
|
||||
request_output = outputs[0]
|
||||
mm = request_output.outputs[0].multimodal_output
|
||||
audio = extract_audio(mm)
|
||||
|
||||
duration = audio.numel() / SAMPLE_RATE
|
||||
rtf = elapsed / duration if duration > 0 else float("inf")
|
||||
|
||||
output_path = output_dir / "output.wav"
|
||||
sf.write(str(output_path), audio.numpy(), SAMPLE_RATE, format="WAV")
|
||||
|
||||
print(f"Saved : {output_path}")
|
||||
print(f"Duration : {duration:.2f}s")
|
||||
print(f"Inference : {elapsed:.2f}s")
|
||||
print(f"RTF : {rtf:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
main()
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Offline inference demo for Qwen3 TTS via vLLM Omni.
|
||||
|
||||
Provides single and batch sample inputs for CustomVoice, VoiceDesign, and Base
|
||||
@@ -166,6 +169,9 @@ def get_custom_voice_query(use_batch_sample: bool = False) -> QueryResult:
|
||||
"language": [language],
|
||||
"speaker": [speaker],
|
||||
"max_new_tokens": [2048],
|
||||
# Optional Code2Wav emit opt-in under deploy.async_chunk.
|
||||
# Independent of prompt-mode non_streaming_mode (#4198 / #6898).
|
||||
"full_utterance_decode": [True],
|
||||
}
|
||||
inputs.append(
|
||||
{
|
||||
@@ -185,6 +191,7 @@ def get_custom_voice_query(use_batch_sample: bool = False) -> QueryResult:
|
||||
"speaker": [speaker],
|
||||
"instruct": [instruct],
|
||||
"max_new_tokens": [2048],
|
||||
"full_utterance_decode": [True],
|
||||
}
|
||||
inputs = {
|
||||
"prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
|
||||
@@ -226,6 +233,7 @@ def get_voice_design_query(use_batch_sample: bool = False) -> QueryResult:
|
||||
"instruct": [instruct],
|
||||
"max_new_tokens": [2048],
|
||||
"non_streaming_mode": [True],
|
||||
"full_utterance_decode": [True],
|
||||
}
|
||||
inputs.append(
|
||||
{
|
||||
@@ -244,6 +252,7 @@ def get_voice_design_query(use_batch_sample: bool = False) -> QueryResult:
|
||||
"instruct": [instruct],
|
||||
"max_new_tokens": [2048],
|
||||
"non_streaming_mode": [True],
|
||||
"full_utterance_decode": [True],
|
||||
}
|
||||
inputs = {
|
||||
"prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
|
||||
@@ -291,6 +300,7 @@ def get_base_query(use_batch_sample: bool = False, mode_tag: str = "icl") -> Que
|
||||
"language": [language],
|
||||
"x_vector_only_mode": [x_vector_only_mode],
|
||||
"max_new_tokens": [2048],
|
||||
"full_utterance_decode": [True],
|
||||
}
|
||||
inputs.append(
|
||||
{
|
||||
@@ -307,6 +317,7 @@ def get_base_query(use_batch_sample: bool = False, mode_tag: str = "icl") -> Que
|
||||
"language": [syn_lang_single],
|
||||
"x_vector_only_mode": [x_vector_only_mode],
|
||||
"max_new_tokens": [2048],
|
||||
"full_utterance_decode": [True],
|
||||
}
|
||||
inputs = {
|
||||
"prompt_token_ids": [0] * _estimate_prompt_len(additional_information, model_name),
|
||||
|
||||
@@ -384,13 +384,20 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
parser.add_argument(
|
||||
quantization_group = parser.add_mutually_exclusive_group()
|
||||
quantization_group.add_argument(
|
||||
"--quantization",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["fp8", "mxfp8", "mxfp4", "mxfp4_dualscale", "int8"],
|
||||
help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 (NPU). mxfp4: W4A4 MXFP4 (NPU). mxfp4_dualscale: W4A4 MXFP4 dual-scale + BF16 fallback mixed (NPU). fp8: online FP8 (GPU).",
|
||||
)
|
||||
quantization_group.add_argument(
|
||||
"--quantization-config",
|
||||
type=json.loads,
|
||||
default=None,
|
||||
help='Quantization JSON, e.g. \'{"method":"mxfp4","w4a8_fallback_steps":[0,2]}\'.',
|
||||
)
|
||||
|
||||
# Distributed and parallel execution
|
||||
parser.add_argument(
|
||||
@@ -572,6 +579,8 @@ def main():
|
||||
omni_kwargs["flow_shift"] = args.flow_shift
|
||||
if args.quantization is not None:
|
||||
omni_kwargs["quantization"] = args.quantization
|
||||
if args.quantization_config is not None:
|
||||
omni_kwargs["quantization_config"] = args.quantization_config
|
||||
if args.cache_backend is not None:
|
||||
omni_kwargs["cache_backend"] = args.cache_backend
|
||||
omni_kwargs["cache_config"] = cache_config
|
||||
|
||||
@@ -14,7 +14,10 @@ Scenario (see ``barge_in_client_flow.md`` for the diagram)
|
||||
|
||||
The demo is model-neutral: it drives any duplex model through the public
|
||||
:class:`vllm_omni.clients.duplex.DuplexClient` over
|
||||
``/v1/realtime?duplex=1``. Model-specific session shape comes from the
|
||||
``/v1/realtime?duplex=1``, or — with ``--inline`` — through
|
||||
:class:`vllm_omni.clients.inline_duplex.InlineDuplexClient` on an in-process
|
||||
:class:`vllm_omni.entrypoints.duplex_omni.DuplexOmni` (no server needed; the
|
||||
model loads in this process). Model-specific session shape comes from the
|
||||
per-model presets under ``vllm_omni.clients.<model>`` selected with
|
||||
``--preset`` (default ``minicpmo_4_5``, whose model decides listen/speak on
|
||||
its own — no VAD).
|
||||
@@ -31,6 +34,15 @@ then:
|
||||
--interrupt-wav follow_up_16k.wav \
|
||||
--output-dir ./duplex_out
|
||||
|
||||
or, without a server, in-process (``--deploy-config`` is optional and defaults
|
||||
to the model's bundled duplex profile):
|
||||
|
||||
python barge_in_client.py --inline \
|
||||
--model /path/to/MiniCPM-o-4_5 \
|
||||
--ref-audio /path/to/reference_voice.wav \
|
||||
--question-wav question_16k.wav \
|
||||
--interrupt-wav follow_up_16k.wav
|
||||
|
||||
Input WAVs must be mono 16 kHz PCM16; when a preset's session takes a
|
||||
different capture format (the PersonaPlex preset streams 24 kHz float32),
|
||||
the audio is converted before streaming. ``--ref-audio`` is required by the
|
||||
@@ -55,6 +67,7 @@ if str(REPO_ROOT) not in sys.path:
|
||||
from vllm_omni.clients.duplex import ( # noqa: E402
|
||||
AudioFormat,
|
||||
DuplexClient,
|
||||
DuplexClientBase,
|
||||
EventCollector,
|
||||
SessionConfig,
|
||||
acknowledge_collected_playback,
|
||||
@@ -65,7 +78,7 @@ from vllm_omni.clients.duplex import ( # noqa: E402
|
||||
)
|
||||
|
||||
_TRANSCRIPT_DELTA_EVENTS = {
|
||||
"response.audio_transcript.delta",
|
||||
"response.output_audio_transcript.delta",
|
||||
"response.output_text.delta",
|
||||
"response.text.delta",
|
||||
}
|
||||
@@ -139,6 +152,39 @@ def _fold_responses(collector: EventCollector) -> dict[str, dict[str, object]]:
|
||||
return out
|
||||
|
||||
|
||||
def _make_client(args: argparse.Namespace, config: SessionConfig) -> DuplexClientBase:
|
||||
"""Websocket client by default; ``--inline`` drives an in-process DuplexOmni."""
|
||||
if not args.inline:
|
||||
return DuplexClient(
|
||||
args.url,
|
||||
model=args.model,
|
||||
config=config,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
)
|
||||
# Imported lazily: the in-process path loads the model (and its engine
|
||||
# dependencies) in this process, which the websocket path never needs.
|
||||
from vllm_omni.clients.inline_duplex import InlineDuplexClient
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
|
||||
# Models shipped with custom HF code (MiniCPM-o) need trust_remote_code to
|
||||
# resolve their pipeline; harmless for the others.
|
||||
omni_kwargs: dict[str, object] = {"model": args.model, "trust_remote_code": True}
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
omni = DuplexOmni(**omni_kwargs)
|
||||
client = InlineDuplexClient(
|
||||
omni,
|
||||
model=args.model,
|
||||
config=config,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
)
|
||||
# The client owns no engine; stop the in-process one when the run ends.
|
||||
client.owned_omni = omni # type: ignore[attr-defined]
|
||||
return client
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
config = _session_config(args.preset, ref_audio_path=args.ref_audio)
|
||||
question = _convert_to_session_format(read_pcm16_wav(Path(args.question_wav)), config.input_audio)
|
||||
@@ -148,105 +194,117 @@ async def run(args: argparse.Namespace) -> int:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
collector = EventCollector()
|
||||
client = DuplexClient(
|
||||
args.url,
|
||||
model=args.model,
|
||||
config=config,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
)
|
||||
client = _make_client(args, config)
|
||||
# GREET: open the voice session (handshake happens on enter).
|
||||
async with client:
|
||||
consume_task = asyncio.create_task(collector.consume(client))
|
||||
try:
|
||||
async with client:
|
||||
consume_task = asyncio.create_task(collector.consume(client))
|
||||
|
||||
# ASK: stream the question at realtime pace, then commit the turn.
|
||||
# The model decides on its own to answer (no VAD).
|
||||
print("[1] streaming question ...", file=sys.stderr)
|
||||
await client.stream_pcm(question, chunk_ms=chunk_ms)
|
||||
await client.commit(create_response=False)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("response.created") > 0,
|
||||
timeout_s=args.timeout_s,
|
||||
label="first response.created",
|
||||
)
|
||||
print("[1] assistant is answering", file=sys.stderr)
|
||||
# ASK: stream the question at realtime pace, then commit the turn.
|
||||
# The model decides on its own to answer (no VAD).
|
||||
print("[1] streaming question ...", file=sys.stderr)
|
||||
await client.stream_pcm(question, chunk_ms=chunk_ms)
|
||||
await client.commit(create_response=False)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("response.created") > 0,
|
||||
timeout_s=args.timeout_s,
|
||||
label="first response.created",
|
||||
)
|
||||
print("[1] assistant is answering", file=sys.stderr)
|
||||
|
||||
# Let the answer play for a moment, as a listening user would.
|
||||
await asyncio.sleep(args.listen_s)
|
||||
# Let the answer play for a moment, as a listening user would.
|
||||
await asyncio.sleep(args.listen_s)
|
||||
|
||||
# INTERRUPT: talk over the assistant while its audio still streams.
|
||||
# The serving overlap policy decides: enough overlapped speech barges
|
||||
# in (the first response is cancelled) or a short remark is deferred
|
||||
# until the model yields the turn.
|
||||
print("[2] interrupting mid-answer ...", file=sys.stderr)
|
||||
ids_before_interrupt = set(collector.response_ids)
|
||||
await client.stream_pcm(interrupt, chunk_ms=chunk_ms)
|
||||
await client.commit(create_response=False)
|
||||
# INTERRUPT: talk over the assistant while its audio still streams.
|
||||
# The serving overlap policy decides: enough overlapped speech barges
|
||||
# in (the first response is cancelled) or a short remark is deferred
|
||||
# until the model yields the turn.
|
||||
print("[2] interrupting mid-answer ...", file=sys.stderr)
|
||||
ids_before_interrupt = set(collector.response_ids)
|
||||
await client.stream_pcm(interrupt, chunk_ms=chunk_ms)
|
||||
await client.commit(create_response=False)
|
||||
|
||||
# ANSWER AGAIN: wait for a response created *after* the interruption
|
||||
# to finish. The first answer may still be streaming here; matching on
|
||||
# any response.done would let its completion satisfy the wait and cut
|
||||
# off the actual follow-up answer at close.
|
||||
def follow_up_answer_done() -> bool:
|
||||
return any(
|
||||
event.get("type") == "response.done"
|
||||
and collector.response_id(event) is not None
|
||||
and collector.response_id(event) not in ids_before_interrupt
|
||||
for event in collector.events
|
||||
# ANSWER AGAIN: wait for a response created *after* the interruption
|
||||
# to finish. The first answer may still be streaming here; matching on
|
||||
# any response.done would let its completion satisfy the wait and cut
|
||||
# off the actual follow-up answer at close.
|
||||
def follow_up_answer_done() -> bool:
|
||||
return any(
|
||||
event.get("type") == "response.done"
|
||||
and collector.response_id(event) is not None
|
||||
and collector.response_id(event) not in ids_before_interrupt
|
||||
for event in collector.events
|
||||
)
|
||||
|
||||
await wait_for_condition(
|
||||
follow_up_answer_done,
|
||||
timeout_s=args.timeout_s,
|
||||
label="response.done for the response answering the interruption",
|
||||
)
|
||||
|
||||
await wait_for_condition(
|
||||
follow_up_answer_done,
|
||||
timeout_s=args.timeout_s,
|
||||
label="response.done for the response answering the interruption",
|
||||
)
|
||||
# HANG UP: report playback, close the session cleanly.
|
||||
await acknowledge_collected_playback(client, collector)
|
||||
await client.close(timeout_s=args.timeout_s)
|
||||
try:
|
||||
await asyncio.wait_for(consume_task, timeout=5.0)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
consume_task.cancel()
|
||||
|
||||
# HANG UP: report playback, close the session cleanly.
|
||||
await acknowledge_collected_playback(client, collector)
|
||||
await client.close(timeout_s=args.timeout_s)
|
||||
try:
|
||||
await asyncio.wait_for(consume_task, timeout=5.0)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
consume_task.cancel()
|
||||
|
||||
# Report: one WAV per response; statuses show what the overlap did.
|
||||
responses = _fold_responses(collector)
|
||||
summary: list[dict[str, object]] = []
|
||||
for index, (response_id, item) in enumerate(responses.items(), start=1):
|
||||
audio = item["audio"]
|
||||
assert isinstance(audio, bytes)
|
||||
sample_rate_hz = int(item["sample_rate_hz"]) # type: ignore[arg-type]
|
||||
wav_path = output_dir / f"response_{index}_{item['status']}.wav"
|
||||
if audio:
|
||||
write_pcm16_wav(wav_path, audio, sample_rate_hz=sample_rate_hz)
|
||||
summary.append(
|
||||
{
|
||||
"response_id": response_id,
|
||||
"status": item["status"],
|
||||
"audio_s": round(len(audio) / (sample_rate_hz * 2), 2),
|
||||
"text": item["text"],
|
||||
"wav": wav_path.name if audio else None,
|
||||
}
|
||||
# Report: one WAV per response; statuses show what the overlap did.
|
||||
responses = _fold_responses(collector)
|
||||
summary: list[dict[str, object]] = []
|
||||
for index, (response_id, item) in enumerate(responses.items(), start=1):
|
||||
audio = item["audio"]
|
||||
assert isinstance(audio, bytes)
|
||||
sample_rate_hz = int(item["sample_rate_hz"]) # type: ignore[arg-type]
|
||||
wav_path = output_dir / f"response_{index}_{item['status']}.wav"
|
||||
if audio:
|
||||
write_pcm16_wav(wav_path, audio, sample_rate_hz=sample_rate_hz)
|
||||
summary.append(
|
||||
{
|
||||
"response_id": response_id,
|
||||
"status": item["status"],
|
||||
"audio_s": round(len(audio) / (sample_rate_hz * 2), 2),
|
||||
"text": item["text"],
|
||||
"wav": wav_path.name if audio else None,
|
||||
}
|
||||
)
|
||||
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
(output_dir / "events.jsonl").write_text(
|
||||
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in collector.events),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
(output_dir / "events.jsonl").write_text(
|
||||
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in collector.events),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
cancelled = sum(1 for item in summary if item["status"] == "cancelled")
|
||||
print(
|
||||
f"\n{len(summary)} responses ({cancelled} interrupted); outputs in {output_dir}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
cancelled = sum(1 for item in summary if item["status"] == "cancelled")
|
||||
print(
|
||||
f"\n{len(summary)} responses ({cancelled} interrupted); outputs in {output_dir}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
finally:
|
||||
owned_omni = getattr(client, "owned_omni", None)
|
||||
if owned_omni is not None:
|
||||
owned_omni.shutdown()
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="openbmb/MiniCPM-o-4_5",
|
||||
help="served model name, or the model path with --inline",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--inline",
|
||||
action="store_true",
|
||||
help="run the model in this process through DuplexOmni + InlineDuplexClient instead of connecting to --url",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
default=None,
|
||||
help="deploy YAML for --inline (default: the model's bundled profile)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preset",
|
||||
choices=["minicpmo_4_5", "personaplex", "none"],
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
# `barge_in_client.py` — what happens, at a glance
|
||||
|
||||
The demo plays one everyday conversation pattern against the full-duplex
|
||||
server: ask something, get talked-over mid-answer, and see the assistant
|
||||
cope. High-level flow below — for the exact wire events, read the
|
||||
The demo plays one everyday conversation pattern against a full-duplex
|
||||
model: ask something, get talked-over mid-answer, and see the assistant
|
||||
cope. It runs either against a server over `/v1/realtime?duplex=1` or, with
|
||||
`--inline`, against an in-process `DuplexOmni` — the conversation below is the
|
||||
same in both modes. High-level flow — for the exact wire events, read the
|
||||
docstring in `barge_in_client.py`.
|
||||
|
||||
```mermaid
|
||||
@@ -48,7 +50,8 @@ answer is saved as `response_1_cancelled.wav`, a completed one as
|
||||
duration per answer.
|
||||
|
||||
Companion: the runtime architecture lives in
|
||||
[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client
|
||||
API is `vllm_omni.clients.duplex.DuplexClient`, documented with the wire
|
||||
protocol in
|
||||
[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client API is
|
||||
`vllm_omni.clients.duplex.DuplexClient` over the websocket and
|
||||
`vllm_omni.clients.inline_duplex.InlineDuplexClient` in-process — both are
|
||||
documented with the wire protocol in
|
||||
[`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md).
|
||||
@@ -1,97 +0,0 @@
|
||||
# Dynin-Omni: Online serving
|
||||
|
||||
## Installation
|
||||
|
||||
Please refer to [README.md](../../../README.md).
|
||||
|
||||
## Launch the Server
|
||||
|
||||
First, find the `transformers_modules` path:
|
||||
|
||||
```bash
|
||||
python - <<'PY'
|
||||
from transformers.utils.hub import HF_MODULES_CACHE
|
||||
print(HF_MODULES_CACHE)
|
||||
PY
|
||||
```
|
||||
|
||||
Then export it for both `PYTHONPATH` and `HF_MODULES_CACHE`:
|
||||
|
||||
```bash
|
||||
export PYTHONPATH=<transformers_modules_path>:$PYTHONPATH
|
||||
export HF_MODULES_CACHE=<transformers_modules_path>
|
||||
```
|
||||
|
||||
Run from repository root. The 3-stage pipeline is resolved automatically from
|
||||
the model type (`vllm_omni/deploy/dynin_omni.yaml`), so no config path is
|
||||
required. Pass `--deploy-config` only to override the defaults:
|
||||
|
||||
```bash
|
||||
vllm-omni serve snu-aidas/Dynin-Omni \
|
||||
--omni \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
If `vllm-omni` is not in PATH, run:
|
||||
|
||||
```bash
|
||||
PYTHONPATH="$(pwd)" python -m vllm_omni.entrypoints.cli.main serve snu-aidas/Dynin-Omni \
|
||||
--omni \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
Wait until the server logs show both `All stages initialized successfully` and
|
||||
`Application startup complete.` before sending requests.
|
||||
|
||||
## Send Requests via Python Client
|
||||
|
||||
Move to the example directory:
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/dynin_omni
|
||||
```
|
||||
|
||||
### Text -> Image
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type t2i \
|
||||
--prompt "A realistic indoor living room with natural daylight."
|
||||
```
|
||||
|
||||
### Image -> Image
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type i2i \
|
||||
--image-path ../../offline_inference/dynin_omni/data/image/sofa_under_water.jpg \
|
||||
--prompt "Transform this surreal underwater setting into a realistic indoor living room while preserving the sofa layout."
|
||||
```
|
||||
|
||||
### Text -> Speech
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type t2s \
|
||||
--prompt "Hello. This is Dynin-omni."
|
||||
```
|
||||
|
||||
## CLI Arguments
|
||||
|
||||
- `--query-type` (`t2i|t2s|i2i`)
|
||||
- `--model` (default: `snu-aidas/Dynin-Omni`)
|
||||
- `--host` / `--port` (OpenAI-compatible vLLM endpoint)
|
||||
- `--prompt` (custom text)
|
||||
- `--image-path` (required for `i2i`)
|
||||
- `--modalities` (optional output modalities override)
|
||||
- `--output-dir` (default: `/tmp/dynin_online_outputs`)
|
||||
|
||||
## Notes
|
||||
|
||||
- This client currently supports only `t2i`, `t2s`, and `i2i`.
|
||||
- `t2t` is intentionally not exposed in this online example.
|
||||
- This example intentionally uses the OpenAI-compatible chat completion endpoint.
|
||||
- Task routing for non-text outputs relies on Dynin task trigger tokens (`<|t2i|>`, `<|i2i|>`, `<|t2s|>`) injected by the client.
|
||||
- Outputs are saved under `/tmp/dynin_online_outputs` by default.
|
||||
- Dynin stage-0 warmup can take a while on first startup; do not send requests before startup completes.
|
||||
- Dynin itself can execute text-returning tasks such as `t2t`, `s2t`, `i2t`, and `v2t`, but this online serving example currently runs stage-0 in `generation` mode. In that path, the generation worker does not surface the final text as `output.text`, so OpenAI chat responses for those text-output tasks may complete internally but still return empty text.
|
||||
-342
@@ -1,342 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_MODEL = "snu-aidas/Dynin-Omni"
|
||||
DEFAULT_OUTPUT_DIR = "/tmp/dynin_online_outputs"
|
||||
|
||||
QUERY_CHOICES = ("t2i", "t2s", "i2i")
|
||||
DEFAULT_PROMPT_BY_QUERY = {
|
||||
"t2i": "A high quality detailed living room interior photo.",
|
||||
"t2s": "Please read this sentence naturally: Hello from Dynin-Omni online serving.",
|
||||
"i2i": "Transform this image into a realistic indoor living room while preserving layout.",
|
||||
}
|
||||
DEFAULT_MODALITIES_BY_QUERY = {
|
||||
"t2i": ["image"],
|
||||
"t2s": ["audio"],
|
||||
"i2i": ["image"],
|
||||
}
|
||||
OFFLINE_PARITY_STAGE_COUNT = 3
|
||||
OFFLINE_PARITY_STAGE_SAMPLING = {
|
||||
"max_tokens": 1,
|
||||
"temperature": 0.0,
|
||||
"top_p": 1.0,
|
||||
"detokenize": False,
|
||||
}
|
||||
|
||||
|
||||
def _infer_mime_type(path: Path) -> str:
|
||||
mime_type, _ = mimetypes.guess_type(str(path))
|
||||
return mime_type or "application/octet-stream"
|
||||
|
||||
|
||||
def _encode_file_as_data_url(path: Path) -> str:
|
||||
mime_type = _infer_mime_type(path)
|
||||
raw = path.read_bytes()
|
||||
encoded = base64.b64encode(raw).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{encoded}"
|
||||
|
||||
|
||||
def _to_image_url(path_or_url: str) -> str:
|
||||
value = str(path_or_url)
|
||||
if value.startswith(("http://", "https://", "data:image/")):
|
||||
return value
|
||||
path = Path(value).expanduser().resolve()
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Image file not found: {path}")
|
||||
return _encode_file_as_data_url(path)
|
||||
|
||||
|
||||
def _build_user_content(query_type: str, prompt: str, image_path: str | None) -> list[dict[str, Any]]:
|
||||
if query_type == "t2i":
|
||||
return [{"type": "text", "text": f"<|t2i|> {prompt}"}]
|
||||
|
||||
if query_type == "t2s":
|
||||
return [{"type": "text", "text": f"<|t2s|> {prompt}"}]
|
||||
|
||||
if query_type == "i2i":
|
||||
if not image_path:
|
||||
raise ValueError("--image-path is required for query type i2i")
|
||||
return [
|
||||
{"type": "text", "text": f"<|i2i|> {prompt}"},
|
||||
{"type": "image_url", "image_url": {"url": _to_image_url(image_path)}},
|
||||
]
|
||||
|
||||
raise ValueError(f"Unsupported query_type: {query_type}")
|
||||
|
||||
|
||||
def _collect_text_from_content(content: Any) -> list[str]:
|
||||
texts: list[str] = []
|
||||
if isinstance(content, str):
|
||||
stripped = content.strip()
|
||||
if stripped:
|
||||
texts.append(stripped)
|
||||
return texts
|
||||
|
||||
if isinstance(content, dict):
|
||||
for key in ("text", "content", "value", "output_text"):
|
||||
text_value = content.get(key)
|
||||
if isinstance(text_value, str) and text_value.strip():
|
||||
texts.append(text_value.strip())
|
||||
return texts
|
||||
|
||||
if isinstance(content, list):
|
||||
for item in content:
|
||||
texts.extend(_collect_text_from_content(item))
|
||||
return texts
|
||||
|
||||
content_text = getattr(content, "text", None)
|
||||
if isinstance(content_text, str) and content_text.strip():
|
||||
texts.append(content_text.strip())
|
||||
content_value = getattr(content, "content", None)
|
||||
if isinstance(content_value, str) and content_value.strip():
|
||||
texts.append(content_value.strip())
|
||||
output_text = getattr(content, "output_text", None)
|
||||
if isinstance(output_text, str) and output_text.strip():
|
||||
texts.append(output_text.strip())
|
||||
return texts
|
||||
|
||||
|
||||
def _extract_text_outputs(chat_completion: Any) -> list[str]:
|
||||
texts: list[str] = []
|
||||
for choice in getattr(chat_completion, "choices", []) or []:
|
||||
message = getattr(choice, "message", None)
|
||||
if message is None:
|
||||
continue
|
||||
content = getattr(message, "content", None)
|
||||
texts.extend(_collect_text_from_content(content))
|
||||
reasoning_content = getattr(message, "reasoning_content", None)
|
||||
if isinstance(reasoning_content, str) and reasoning_content.strip():
|
||||
texts.append(reasoning_content.strip())
|
||||
choice_text = getattr(choice, "text", None)
|
||||
if isinstance(choice_text, str) and choice_text.strip():
|
||||
texts.append(choice_text.strip())
|
||||
top_level_output_text = getattr(chat_completion, "output_text", None)
|
||||
if isinstance(top_level_output_text, str) and top_level_output_text.strip():
|
||||
texts.append(top_level_output_text.strip())
|
||||
return texts
|
||||
|
||||
|
||||
def _extract_image_data_urls(chat_completion: Any) -> list[str]:
|
||||
urls: list[str] = []
|
||||
for choice in getattr(chat_completion, "choices", []) or []:
|
||||
message = getattr(choice, "message", None)
|
||||
if message is None:
|
||||
continue
|
||||
content = getattr(message, "content", None)
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get("type") != "image_url":
|
||||
continue
|
||||
image_url = (item.get("image_url") or {}).get("url")
|
||||
if isinstance(image_url, str) and image_url.startswith("data:image"):
|
||||
urls.append(image_url)
|
||||
return urls
|
||||
|
||||
|
||||
def _extract_audio_payloads(chat_completion: Any) -> list[bytes]:
|
||||
payloads: list[bytes] = []
|
||||
for choice in getattr(chat_completion, "choices", []) or []:
|
||||
message = getattr(choice, "message", None)
|
||||
if message is None:
|
||||
continue
|
||||
message_audio = getattr(message, "audio", None)
|
||||
if message_audio is None:
|
||||
continue
|
||||
data_b64 = getattr(message_audio, "data", None)
|
||||
if isinstance(data_b64, str) and data_b64:
|
||||
try:
|
||||
payloads.append(base64.b64decode(data_b64))
|
||||
except Exception:
|
||||
continue
|
||||
return payloads
|
||||
|
||||
|
||||
def _decode_data_url(data_url: str) -> tuple[bytes, str]:
|
||||
header, data = data_url.split(",", 1)
|
||||
mime_type = "image/png"
|
||||
if ";" in header and ":" in header:
|
||||
mime_type = header.split(":", 1)[1].split(";", 1)[0]
|
||||
return base64.b64decode(data), mime_type
|
||||
|
||||
|
||||
def _image_extension_from_mime(mime_type: str) -> str:
|
||||
if mime_type == "image/jpeg":
|
||||
return ".jpg"
|
||||
if mime_type == "image/webp":
|
||||
return ".webp"
|
||||
if mime_type == "image/gif":
|
||||
return ".gif"
|
||||
return ".png"
|
||||
|
||||
|
||||
def _save_outputs(
|
||||
*,
|
||||
query_type: str,
|
||||
chat_completion: Any,
|
||||
output_dir: Path,
|
||||
) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
stamp = time.strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
text_outputs = _extract_text_outputs(chat_completion)
|
||||
image_data_urls = _extract_image_data_urls(chat_completion)
|
||||
audio_payloads = _extract_audio_payloads(chat_completion)
|
||||
|
||||
if text_outputs:
|
||||
text_path = output_dir / f"{query_type}_{stamp}.txt"
|
||||
text_path.write_text("\n\n".join(text_outputs) + "\n", encoding="utf-8")
|
||||
print(f"[dynin-online] text saved: {text_path}")
|
||||
print(text_outputs[0])
|
||||
|
||||
for idx, image_url in enumerate(image_data_urls):
|
||||
image_bytes, mime_type = _decode_data_url(image_url)
|
||||
ext = _image_extension_from_mime(mime_type)
|
||||
image_path = output_dir / f"{query_type}_{stamp}_{idx}{ext}"
|
||||
image_path.write_bytes(image_bytes)
|
||||
print(f"[dynin-online] image saved: {image_path}")
|
||||
|
||||
for idx, audio_bytes in enumerate(audio_payloads):
|
||||
audio_path = output_dir / f"{query_type}_{stamp}_{idx}.wav"
|
||||
audio_path.write_bytes(audio_bytes)
|
||||
print(f"[dynin-online] audio saved: {audio_path}")
|
||||
|
||||
if not text_outputs and not image_data_urls and not audio_payloads:
|
||||
print("[dynin-online] no output extracted from response")
|
||||
raw_path = output_dir / f"{query_type}_{stamp}_raw_response.json"
|
||||
try:
|
||||
if hasattr(chat_completion, "model_dump_json"):
|
||||
serialized = chat_completion.model_dump_json(indent=2)
|
||||
else:
|
||||
if hasattr(chat_completion, "model_dump"):
|
||||
raw_payload: Any = chat_completion.model_dump(mode="json")
|
||||
else:
|
||||
raw_payload = chat_completion
|
||||
try:
|
||||
serialized = json.dumps(raw_payload, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
serialized = json.dumps({"repr": repr(raw_payload)}, ensure_ascii=False, indent=2)
|
||||
raw_path.write_text(serialized + "\n", encoding="utf-8")
|
||||
print(f"[dynin-online] raw response saved: {raw_path}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _build_offline_parity_sampling_params_list() -> list[dict[str, Any]]:
|
||||
return [dict(OFFLINE_PARITY_STAGE_SAMPLING) for _ in range(OFFLINE_PARITY_STAGE_COUNT)]
|
||||
|
||||
|
||||
def run_request(args: argparse.Namespace) -> None:
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="EMPTY",
|
||||
base_url=f"http://{args.host}:{args.port}/v1",
|
||||
)
|
||||
prompt = args.prompt.strip() if args.prompt else DEFAULT_PROMPT_BY_QUERY[args.query_type]
|
||||
user_content = _build_user_content(
|
||||
query_type=args.query_type,
|
||||
prompt=prompt,
|
||||
image_path=args.image_path,
|
||||
)
|
||||
if args.modalities:
|
||||
modalities = [item.strip() for item in args.modalities.split(",") if item.strip()]
|
||||
else:
|
||||
modalities = DEFAULT_MODALITIES_BY_QUERY[args.query_type]
|
||||
|
||||
extra_body = {
|
||||
"sampling_params_list": _build_offline_parity_sampling_params_list(),
|
||||
}
|
||||
chat_completion = client.chat.completions.create(
|
||||
model=args.model,
|
||||
messages=[{"role": "user", "content": user_content}],
|
||||
modalities=modalities,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
_save_outputs(
|
||||
query_type=args.query_type,
|
||||
chat_completion=chat_completion,
|
||||
output_dir=Path(args.output_dir).expanduser(),
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Dynin-Omni online chat completion client")
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
type=str,
|
||||
default="t2i",
|
||||
choices=QUERY_CHOICES,
|
||||
help="Dynin query type",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
"-m",
|
||||
type=str,
|
||||
default=DEFAULT_MODEL,
|
||||
help="Model name/path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
help="Host/IP of the vLLM Omni API server",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8091,
|
||||
help="Port of the vLLM Omni API server",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
"-p",
|
||||
type=str,
|
||||
default="",
|
||||
help="Custom prompt text",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image-path",
|
||||
"-i",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Image path/URL for i2i",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--modalities",
|
||||
type=str,
|
||||
default="",
|
||||
help="Comma-separated output modalities override (e.g., text,image,audio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
"-o",
|
||||
type=str,
|
||||
default=DEFAULT_OUTPUT_DIR,
|
||||
help="Directory to save outputs",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
os.environ.setdefault("HF_HUB_DISABLE_XET", "1")
|
||||
run_request(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -183,10 +183,20 @@ python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--output-dir /tmp/minicpmo_realtime_duplex_video_demo
|
||||
```
|
||||
|
||||
Detail inside a composite is capped by `scale_resolution=448` at
|
||||
`max_slice_nums=1`: official suggests HD slicing (`max_slice_nums=[2, 1]`) for
|
||||
stacked frames, which the duplex adapter does not implement yet. Reading small
|
||||
text or digits out of a wide scene is limited by that, not by frame timing.
|
||||
Stage 0 processes a stacked unit the way official HD mode does
|
||||
(`max_slice_nums=[2, 1]`): the base frame keeps its own 66-token block plus the
|
||||
patches `get_sliced_grid` cuts for its size at `scale_resolution=448`, and the
|
||||
composite adds one block. The wire still rejects a caller-supplied
|
||||
`max_slice_nums`, so slicing is Stage 0's decision, not the client's.
|
||||
|
||||
That detail costs context. One 960x540 frame is 66 vision tokens per unit and a
|
||||
stacked pair is 264, on top of the unit's audio, so the Stage 0 prompt grows by
|
||||
277 tokens per second stacked against 79 with a single frame. Against the
|
||||
40960-token context this model ships with, a listen-heavy call reaches the limit
|
||||
after about 2.5 minutes stacked and about 8.5 minutes unstacked (measured on a
|
||||
264 s 960x540 clip). Duplex sessions cannot yet evict old units, so keep
|
||||
multi-minute calls on the default `--stack-frames 1`; a stacked session that
|
||||
outgrows the context ends with a context-length error.
|
||||
|
||||
## Open the experimental browser client
|
||||
|
||||
@@ -306,3 +316,12 @@ them unless `--allow-invalid-clock` is explicit.
|
||||
[`examples/offline_inference/minicpmo/`](../../offline_inference/minicpmo/)
|
||||
- Recipe:
|
||||
[`recipes/OpenBMB/MiniCPM-o-4_5.md`](../../../recipes/OpenBMB/MiniCPM-o-4_5.md)
|
||||
|
||||
### Shared realtime UI implementation
|
||||
|
||||
The `python -m examples.online_serving.minicpmo.realtime_web` command remains
|
||||
available with the same MiniCPM defaults and required `--ref-audio`. Its assets
|
||||
now live in [the shared realtime UI](../realtime_web/README.md), with a dedicated
|
||||
`minicpm-native` profile preserving native duplex input, playback ACKs and camera
|
||||
frames. Qwen3 uses a separate profile in the same shell, supporting manual STT
|
||||
turns or Server VAD turns with speech interruption.
|
||||
@@ -5,10 +5,10 @@
|
||||
# Server side for barge_in_client.py: start MiniCPM-o 4.5 in native
|
||||
# full-duplex mode.
|
||||
#
|
||||
# The duplex deploy profile registers the WS /v1/duplex endpoint (with
|
||||
# /v1/realtime?duplex=1 projecting onto the same handler), enables the
|
||||
# engine duplex control plane, and bounds the Thinker/Talker/Code2Wav
|
||||
# stages to four live duplex sessions on one GPU. See
|
||||
# MiniCPM-o 4.5 declares a duplex plugin, so vllm-omni serve runs it through
|
||||
# DuplexOmni: the server is duplex-only (WS /v1/realtime?duplex=1, alias
|
||||
# /v1/duplex, plus /v1/models and /health) and bounds the
|
||||
# Thinker/Talker/Code2Wav stages to four live duplex sessions on one GPU. See
|
||||
# vllm_omni/deploy/minicpmo_4_5.yaml (session_mode: duplex) for the session
|
||||
# limits; every other duplex_session knob keeps its runtime default
|
||||
# (idle TTL 300s, disconnect grace 30s, 16MiB pending input per session).
|
||||
@@ -22,4 +22,4 @@ exec vllm-omni serve "$MODEL" \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port "$PORT"
|
||||
# duplex endpoints: ws://<host>:$PORT/v1/duplex and /v1/realtime?duplex=1
|
||||
# duplex endpoint: ws://<host>:$PORT/v1/realtime?duplex=1 (alias /v1/duplex)
|
||||
@@ -63,14 +63,14 @@ class _StreamingOutputWriter:
|
||||
def handle(self, event: dict[str, object]) -> None:
|
||||
event_type = event.get("type")
|
||||
if event_type in {
|
||||
"response.audio_transcript.delta",
|
||||
"response.output_audio_transcript.delta",
|
||||
"response.output_text.delta",
|
||||
}:
|
||||
delta = event.get("delta")
|
||||
if isinstance(delta, str) and delta:
|
||||
print(delta, end="", file=sys.stderr, flush=True)
|
||||
return
|
||||
if event_type != "response.audio.delta":
|
||||
if event_type != "response.output_audio.delta":
|
||||
return
|
||||
|
||||
delta = event.get("delta") or event.get("audio")
|
||||
@@ -445,7 +445,6 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
ref_audio=_ref_audio_data_url(args.ref_audio),
|
||||
temperature=args.temperature,
|
||||
),
|
||||
session_id=args.session_id,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
@@ -509,12 +508,12 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
|
||||
audio = collector.audio_bytes()
|
||||
first_text_at_s = collector.first_received_at(
|
||||
"response.audio_transcript.delta",
|
||||
"response.output_audio_transcript.delta",
|
||||
"response.output_text.delta",
|
||||
after_s=commit_sent_at_s,
|
||||
)
|
||||
first_audio_at_s = collector.first_received_at(
|
||||
"response.audio.delta",
|
||||
"response.output_audio.delta",
|
||||
after_s=commit_sent_at_s,
|
||||
)
|
||||
response_created_at_s = collector.first_received_at(
|
||||
@@ -536,7 +535,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
for event in collector.events
|
||||
if event.get("type")
|
||||
in {
|
||||
"response.audio_transcript.delta",
|
||||
"response.output_audio_transcript.delta",
|
||||
"response.output_text.delta",
|
||||
}
|
||||
]
|
||||
@@ -638,7 +637,6 @@ def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--session-id")
|
||||
parser.add_argument(
|
||||
"--input-wav",
|
||||
help="Mono 16 kHz PCM16 clip to stream. Optional when --input-video carries the audio track.",
|
||||
|
||||
@@ -1,185 +1,23 @@
|
||||
"""Static host and same-origin Realtime proxy for the full-duplex demo."""
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
"""Backward-compatible MiniCPM entry point for the shared realtime UI."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
from examples.online_serving.realtime_web.server import (
|
||||
APP_DIR,
|
||||
STATIC_DIR,
|
||||
_join_ws_url,
|
||||
build_app,
|
||||
)
|
||||
from examples.online_serving.realtime_web.server import (
|
||||
main as _main,
|
||||
)
|
||||
|
||||
import uvicorn
|
||||
import websockets
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import HTMLResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
APP_DIR = Path(__file__).parent / "app"
|
||||
STATIC_DIR = APP_DIR / "static"
|
||||
|
||||
|
||||
def _join_ws_url(base: str, path: str, query: str) -> str:
|
||||
return base.rstrip("/") + path + (("?" + query) if query else "")
|
||||
|
||||
|
||||
async def _pump_client_to_backend(client: WebSocket, backend) -> None:
|
||||
try:
|
||||
while True:
|
||||
message = await client.receive()
|
||||
if message["type"] == "websocket.disconnect":
|
||||
await backend.close()
|
||||
return
|
||||
if message.get("text") is not None:
|
||||
await backend.send(message["text"])
|
||||
elif message.get("bytes") is not None:
|
||||
await backend.send(message["bytes"])
|
||||
except (WebSocketDisconnect, websockets.ConnectionClosed):
|
||||
with contextlib.suppress(Exception):
|
||||
await backend.close()
|
||||
|
||||
|
||||
async def _pump_backend_to_client(client: WebSocket, backend) -> None:
|
||||
try:
|
||||
async for message in backend:
|
||||
if isinstance(message, bytes):
|
||||
await client.send_bytes(message)
|
||||
else:
|
||||
await client.send_text(message)
|
||||
except (WebSocketDisconnect, websockets.ConnectionClosed):
|
||||
return
|
||||
except RuntimeError as exc:
|
||||
if "websocket.send" in str(exc) and "websocket.close" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def _expected_proxy_close(exc: BaseException) -> bool:
|
||||
if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
|
||||
return True
|
||||
return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
|
||||
|
||||
|
||||
def build_app(
|
||||
*,
|
||||
ws_backend: str = "ws://127.0.0.1:8099",
|
||||
model: str = "openbmb/MiniCPM-o-4_5",
|
||||
public_realtime_url: str | None = None,
|
||||
ref_audio: str | None = None,
|
||||
) -> FastAPI:
|
||||
app = FastAPI(title="Experimental Full-Duplex Web Demo")
|
||||
index_path = APP_DIR / "index.html"
|
||||
app_version_hash = hashlib.sha256()
|
||||
for asset_path in (
|
||||
STATIC_DIR / "app.js",
|
||||
STATIC_DIR / "pcm_worklet.js",
|
||||
STATIC_DIR / "playback_worklet.js",
|
||||
):
|
||||
app_version_hash.update(asset_path.read_bytes())
|
||||
app_version = app_version_hash.hexdigest()[:12]
|
||||
|
||||
ref_audio_uri: str | None = None
|
||||
if ref_audio:
|
||||
ref_path = Path(ref_audio)
|
||||
if not ref_path.is_file():
|
||||
raise SystemExit(f"--ref-audio not found: {ref_audio}")
|
||||
encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
|
||||
ref_audio_uri = f"data:audio/wav;base64,{encoded}"
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index() -> HTMLResponse:
|
||||
config = json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"realtimePath": public_realtime_url or "v1/realtime",
|
||||
"refAudio": ref_audio_uri,
|
||||
"appVersion": app_version,
|
||||
},
|
||||
ensure_ascii=True,
|
||||
)
|
||||
html = (
|
||||
index_path.read_text(encoding="utf-8")
|
||||
.replace("__FULL_DUPLEX_CONFIG__", config)
|
||||
.replace("__FULL_DUPLEX_APP_VERSION__", app_version)
|
||||
)
|
||||
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> Response:
|
||||
return Response(content="ok", media_type="text/plain")
|
||||
|
||||
@app.websocket("/v1/realtime")
|
||||
async def realtime_proxy(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
query = urlencode(websocket.query_params.multi_items())
|
||||
backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
|
||||
logger.info("Proxying Realtime WebSocket to %s", backend_url)
|
||||
try:
|
||||
async with websockets.connect(
|
||||
backend_url,
|
||||
max_size=64 * 1024 * 1024,
|
||||
) as backend:
|
||||
tasks = {
|
||||
asyncio.create_task(_pump_client_to_backend(websocket, backend)),
|
||||
asyncio.create_task(_pump_backend_to_client(websocket, backend)),
|
||||
}
|
||||
done, pending = await asyncio.wait(
|
||||
tasks,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
await asyncio.gather(*pending, return_exceptions=True)
|
||||
for result in await asyncio.gather(*done, return_exceptions=True):
|
||||
if isinstance(result, BaseException) and not _expected_proxy_close(result):
|
||||
raise result
|
||||
except (WebSocketDisconnect, websockets.ConnectionClosed):
|
||||
return
|
||||
except Exception:
|
||||
logger.exception("Realtime WebSocket proxy failed")
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.close(code=1011)
|
||||
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
return app
|
||||
__all__ = ["APP_DIR", "STATIC_DIR", "_join_ws_url", "build_app", "main"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=7862)
|
||||
parser.add_argument("--ws-backend", default="ws://127.0.0.1:8099")
|
||||
parser.add_argument(
|
||||
"--public-realtime-url",
|
||||
help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
|
||||
)
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
required=True,
|
||||
help=(
|
||||
"Reference voice wav for TTS voice cloning, e.g. the "
|
||||
"official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
uvicorn.run(
|
||||
build_app(
|
||||
ws_backend=args.ws_backend,
|
||||
model=args.model,
|
||||
public_realtime_url=args.public_realtime_url,
|
||||
ref_audio=args.ref_audio,
|
||||
),
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level="info",
|
||||
)
|
||||
_main(default_profile="minicpm-native")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Qwen3-Omni: Online serving
|
||||
|
||||
<!-- markdownlint-disable MD028 -->
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](../../../README.md)
|
||||
@@ -12,6 +14,17 @@ Please refer to [README.md](../../../README.md)
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
For a local deployment with multiple API frontend processes sharing one set
|
||||
of stage engines, add `--api-server-count`:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --api-server-count 2 --port 8091
|
||||
```
|
||||
|
||||
This mode currently supports local EngineCore stages. It is not available
|
||||
with headless or remote stage deployments, and pure diffusion stages remain
|
||||
single-frontend.
|
||||
|
||||
The default deployment configuration, situated at `vllm_omni/deploy/qwen3_omni_moe.yaml`, is resolved and loaded
|
||||
automatically via the model registry, obviating the `--deploy-config` flag in standard deployment topologies.
|
||||
Asynchronous chunk streaming operates as **enabled by default** within this bundled configuration.
|
||||
@@ -19,6 +32,7 @@ Additionally, NPU, ROCm, and XPU per-platform configuration deltas are determini
|
||||
`platforms`: section of the corresponding YAML.
|
||||
|
||||
To explicitly utilize a custom deployment YAML, mandate the configuration path accordingly:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config /path/to/your_deploy_config.yaml
|
||||
@@ -53,7 +67,7 @@ Use the stage-based CLI when you want to run one stage per process.
|
||||
The example below pins Stage 0 to GPU 0 and Stage 1/2 to GPU 1 via
|
||||
`CUDA_VISIBLE_DEVICES`.
|
||||
|
||||
**1. Stage 0 (Thinker + API server)**
|
||||
#### 1. Stage 0 (Thinker + API server)
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
@@ -63,7 +77,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
--omni-master-port 26000
|
||||
```
|
||||
|
||||
**2. Stage 1 (Talker)**
|
||||
#### 2. Stage 1 (Talker)
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
@@ -73,7 +87,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
--omni-master-port 26000
|
||||
```
|
||||
|
||||
**3. Stage 2 (Code2Wav)**
|
||||
#### 3. Stage 2 (Code2Wav)
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
@@ -153,7 +167,7 @@ parser defaults). If you don't pass a flag, the YAML value wins.
|
||||
> chunked vs end-to-end modes (e.g. qwen3_tts code2wav) dispatch
|
||||
> automatically based on that bool — no extra flag or variant yaml is
|
||||
> needed.
|
||||
|
||||
>
|
||||
> ⚠️ **For multi-stage models that share GPUs (qwen3_omni_moe by default
|
||||
> shares cuda:1 between stages 1 and 2), avoid using global memory flags.**
|
||||
> A global `--gpu-memory-utilization 0.85` would apply to every stage and
|
||||
@@ -252,11 +266,12 @@ pattern works for Qwen2.5-Omni — replace `base_config:` with the path to
|
||||
### Send Multi-modal Request
|
||||
|
||||
Get into the example folder
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/qwen3_omni
|
||||
```
|
||||
|
||||
#### Send request via python
|
||||
#### Send request via python
|
||||
|
||||
```bash
|
||||
python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type use_image --port 8091 --host "localhost"
|
||||
@@ -264,7 +279,7 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
|
||||
|
||||
#### Realtime WebSocket client (`openai_realtime_client.py`)
|
||||
|
||||
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
|
||||
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.output_audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
|
||||
|
||||
Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.
|
||||
|
||||
@@ -289,7 +304,7 @@ python openai_realtime_client.py \
|
||||
**Arguments:**
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| ------ | --------- | ------------- |
|
||||
| `--url` | `ws://localhost:8091/v1/realtime` | Full WebSocket URL including path |
|
||||
| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (sent in `session.update`) |
|
||||
| `--input-wav` | *(required)* | Input WAV: mono, 16-bit PCM, **16 kHz** |
|
||||
@@ -297,7 +312,7 @@ python openai_realtime_client.py \
|
||||
| `--output-text` | *(optional)* | If set, write final transcription text to this path |
|
||||
| `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
|
||||
| `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
|
||||
| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files |
|
||||
| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.output_audio.delta` WAV files |
|
||||
| `--server-vad` | disabled | Enable server-side Silero VAD instead of sending explicit commits |
|
||||
| `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
|
||||
| `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |
|
||||
@@ -305,24 +320,17 @@ python openai_realtime_client.py \
|
||||
Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
|
||||
|
||||
```yaml
|
||||
base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
|
||||
duplex_session:
|
||||
server_vad_model_path: /models/silero_vad.onnx
|
||||
```
|
||||
|
||||
Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
|
||||
|
||||
```bash
|
||||
cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
|
||||
# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--deploy-config /path/to/qwen3_omni_server_vad.yaml
|
||||
```
|
||||
|
||||
Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
|
||||
semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
|
||||
wire flow; `?duplex=1` remains a supported compatibility alias.
|
||||
For the engine-owned plugin, use an overlay based on
|
||||
`vllm_omni/deploy/qwen3_omni_duplex.yaml`, with `duplex_session.server_vad_model_path`
|
||||
pointing to your local artifact. See the [complete duplex setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
|
||||
The default `qwen3_omni_moe.yaml` remains a turn deployment. Duplex mode serves
|
||||
`/v1/realtime?duplex=1` and `/v1/chat/completions`; the legacy STT handler requires
|
||||
a turn deployment.
|
||||
|
||||
The Python client supports the following command-line arguments:
|
||||
|
||||
@@ -334,7 +342,6 @@ The Python client supports the following command-line arguments:
|
||||
- `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"`
|
||||
- `--speaker`: TTS speaker/voice for audio output when requesting audio (e.g. `ethan`, `chelsie`, `aiden`). Omit to use the model default. Example: `--speaker "chelsie"`
|
||||
|
||||
|
||||
For example, to use a local video file with custom prompt:
|
||||
|
||||
```bash
|
||||
@@ -345,22 +352,22 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
|
||||
--prompt "What are the main activities shown in this video?"
|
||||
```
|
||||
|
||||
#### Send request via curl
|
||||
#### Send request via curl
|
||||
|
||||
```bash
|
||||
bash run_curl_multimodal_generation.sh use_image
|
||||
```
|
||||
|
||||
|
||||
### FAQ
|
||||
|
||||
## Modality control
|
||||
|
||||
You can control output modalities to specify which types of output the model should generate. This is useful when you only need text output and want to skip audio generation stages for better performance.
|
||||
|
||||
### Supported modalities
|
||||
|
||||
| Modalities | Output |
|
||||
|------------|--------|
|
||||
| ------------ | -------- |
|
||||
| `["text"]` | Text only |
|
||||
| `["audio"]` | Audio only |
|
||||
| `["text", "audio"]` | Text + Audio |
|
||||
@@ -496,7 +503,9 @@ print(response.choices[0].message.audio)
|
||||
Supported speaker names depend on the model (e.g. `Ethan`, `Chelsie`, `Aiden`). Omit `speaker` to use the default.
|
||||
|
||||
## Streaming Output
|
||||
|
||||
If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. We support both text streaming output and audio streaming output. Other modalities can output normally.
|
||||
|
||||
```bash
|
||||
python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type use_image \
|
||||
@@ -521,12 +530,14 @@ The convenience script launches both the vLLM server and Gradio demo together:
|
||||
```
|
||||
|
||||
This script will:
|
||||
|
||||
1. Start the vLLM server in the background
|
||||
2. Wait for the server to be ready
|
||||
3. Launch the Gradio demo
|
||||
4. Handle cleanup when you press Ctrl+C
|
||||
|
||||
The script supports the following arguments:
|
||||
|
||||
- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
|
||||
- `--server-port`: Port for vLLM server (default: 8091)
|
||||
- `--gradio-port`: Port for Gradio demo (default: 7861)
|
||||
@@ -537,18 +548,19 @@ The script supports the following arguments:
|
||||
|
||||
#### Option 2: Manual Launch (Two-Step Process)
|
||||
|
||||
**Step 1: Launch the vLLM API server**
|
||||
##### Step 1: Launch the vLLM API server
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
If you have a custom deploy config file:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file
|
||||
```
|
||||
|
||||
**Step 2: Run the Gradio demo**
|
||||
##### Step 2: Run the Gradio demo
|
||||
|
||||
In a separate terminal:
|
||||
|
||||
@@ -561,7 +573,57 @@ Then open `http://localhost:7861/` on your local browser to interact with the we
|
||||
The gradio script supports the following arguments:
|
||||
|
||||
- `--model`: Model name/path (should match the server model)
|
||||
- `--api-base`: Base URL for the vLLM API server (default: http://localhost:8091/v1)
|
||||
- `--api-base`: Base URL for the vLLM API server (default: <http://localhost:8091/v1>)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
|
||||
## Browser voice call (shared realtime UI)
|
||||
|
||||
Run these commands from the repository root with the vLLM-Omni environment activated.
|
||||
|
||||
### Without VAD: manually submit each turn
|
||||
|
||||
Start the backend:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
In another terminal, check backend readiness and start the UI:
|
||||
|
||||
```bash
|
||||
curl --fail http://127.0.0.1:8091/health
|
||||
python -m examples.online_serving.qwen3_omni.realtime_web \
|
||||
--backend ws://127.0.0.1:8091 --stt --port 7863
|
||||
```
|
||||
|
||||
Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
|
||||
`--stt` is the default. Each turn uses a separate connection without shared model
|
||||
history; the conversation shown in the browser is a local log.
|
||||
|
||||
### With VAD: automatically submit after silence
|
||||
|
||||
Start the backend with a Server VAD deployment overlay and a compatible Silero
|
||||
ONNX artifact, following the
|
||||
[complete VAD setup](../realtime_web/README.md#qwen3-with-server-vad-automatic-turns).
|
||||
After its health check succeeds, start the UI:
|
||||
|
||||
```bash
|
||||
python -m examples.online_serving.qwen3_omni.realtime_web \
|
||||
--backend ws://127.0.0.1:8091 --vad --port 7863
|
||||
```
|
||||
|
||||
Speak and pause for 500 ms to submit automatically. `--vad` configures the client;
|
||||
it does not load a VAD model or change the backend deployment by itself.
|
||||
|
||||
VAD mode keeps uploading audio during replies and supports speech interruption.
|
||||
The engine-owned Qwen duplex plugin maintains conversation history and tracks
|
||||
played replies through playback acknowledgements. Each committed utterance starts
|
||||
a new generation request; the model does not use native streaming-input KV decoding.
|
||||
STT mode pauses microphone upload while the reply is generated and played.
|
||||
In VAD mode, click **Camera** to upload sampled frames with your spoken question;
|
||||
see [camera setup and limits](../realtime_web/README.md#qwen-vad-camera-input).
|
||||
STT remains audio-only. To switch back to manual turns, use the default turn
|
||||
deployment, restart the UI with `--stt`, refresh the page, and reconnect. Stop the existing UI before reusing port 7863.
|
||||
See the [shared UI guide](../realtime_web/README.md) for HTTPS access and testing.
|
||||
@@ -6,14 +6,14 @@
|
||||
This client:
|
||||
1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
|
||||
2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
|
||||
3) Receives response.audio.*, response.output_text.*, and transcript events,
|
||||
3) Receives response.output_audio.*, response.output_text.*, and transcript events,
|
||||
4) Saves synthesized audio to an output WAV file and optional text file.
|
||||
|
||||
By default each ``response.audio.delta`` is treated as an **incremental PCM**
|
||||
By default each ``response.output_audio.delta`` is treated as an **incremental PCM**
|
||||
chunk and all chunks are concatenated into the final ``--output-wav``.
|
||||
|
||||
Optional debugging: pass ``--delta-dump-dir DIR`` to write every
|
||||
``response.audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, …
|
||||
``response.output_audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, …
|
||||
|
||||
Usage:
|
||||
python openai_realtime_client.py \
|
||||
@@ -130,7 +130,7 @@ async def run_client(
|
||||
"prefix_padding_ms": 300,
|
||||
"silence_duration_ms": 500,
|
||||
"create_response": True,
|
||||
"interrupt_response": False,
|
||||
"interrupt_response": True,
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -187,7 +187,7 @@ async def run_client(
|
||||
if event_type == "session.created":
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.delta":
|
||||
if event_type == "response.output_audio.delta":
|
||||
sr = event.get("sample_rate_hz")
|
||||
if isinstance(sr, int) and sr > 0:
|
||||
output_sample_rate = sr
|
||||
@@ -223,16 +223,17 @@ async def run_client(
|
||||
print(f"{log_prefix}text usage: {usage}")
|
||||
continue
|
||||
|
||||
if event_type == "response.audio_transcript.done":
|
||||
if event_type == "response.output_audio_transcript.done":
|
||||
final_audio_transcript = event.get("transcript", "")
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.done":
|
||||
if event_type == "response.output_audio.done":
|
||||
if not server_vad:
|
||||
break
|
||||
continue
|
||||
|
||||
if event_type == "response.done" and server_vad:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
break
|
||||
|
||||
if event_type == "error":
|
||||
@@ -339,7 +340,7 @@ def main() -> None:
|
||||
"--delta-dump-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory",
|
||||
help="If set, each response.output_audio.delta is saved as delta_NNNNNN.wav under this directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server-vad",
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Qwen3-Omni turn-based browser call UI."""
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from examples.online_serving.realtime_web.server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(default_profile="qwen3-turn")
|
||||
@@ -0,0 +1,221 @@
|
||||
# Shared Realtime Web UI
|
||||
|
||||
One browser shell provides microphone capture, speaker playback, a level meter,
|
||||
conversation text, and an event log. Model profiles own the wire protocol.
|
||||
This implements [RFC #7222](https://github.com/vllm-project/vllm-omni/issues/7222).
|
||||
|
||||
| Profile | Turn control | Camera | Playback acknowledgement |
|
||||
| --- | --- | --- | --- |
|
||||
| `minicpm-native` | Model-controlled listen/speak, continuous audio input | Frames accompany audio | Yes |
|
||||
| `qwen3-turn --stt` (default) | User presses **Send turn** | No | No |
|
||||
| `qwen3-turn --vad` | Server detects trailing silence; speech interrupts replies | Sampled frames with each spoken turn | Yes |
|
||||
|
||||
Qwen3 VAD uses an engine-owned duplex plugin: microphone upload continues
|
||||
while replies stream, and speech can interrupt generation and playback. The
|
||||
checkpoint still generates committed turns; this is not native streaming-input
|
||||
KV decoding. STT remains explicit-turn and audio-only.
|
||||
|
||||
## MiniCPM: existing command stays valid
|
||||
|
||||
Start your MiniCPM backend using the [existing deployment instructions](../minicpmo/README.md), then:
|
||||
|
||||
```bash
|
||||
python -m examples.online_serving.minicpmo.realtime_web \
|
||||
--ws-backend ws://127.0.0.1:8099 \
|
||||
--ref-audio /path/to/ref_minicpm_signature.wav
|
||||
```
|
||||
|
||||
The compatibility wrapper serves the shared assets with `minicpm-native`.
|
||||
The current native query, `extra_body`, reference voice, continuous microphone
|
||||
upload, camera frames and playback acknowledgements are retained.
|
||||
|
||||
## Qwen3: explicit-turn STT adapter
|
||||
|
||||
Start the backend, then the UI in a second terminal:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
|
||||
python -m examples.online_serving.qwen3_omni.realtime_web \
|
||||
--backend ws://127.0.0.1:8091 --stt --port 7863
|
||||
```
|
||||
|
||||
Open `http://localhost:7863`, start a session, speak, and press **Send turn**.
|
||||
Wait for the answer to finish playing before speaking again. The UI opens a
|
||||
fresh STT connection for each turn; the displayed conversation is a local log,
|
||||
and **previous turns are not provided as model history**. The STT endpoint
|
||||
accepts only model selection, so system-prompt controls are hidden in this mode.
|
||||
|
||||
The adapter follows the shipped `qwen3_omni/openai_realtime_client.py` and
|
||||
`vllm_omni/entrypoints/openai/realtime_connection.py`:
|
||||
|
||||
1. Explicit `duplex=0` selects the legacy STT handler on a turn deployment.
|
||||
It does not enable STT on a duplex deployment.
|
||||
2. Send `{type: "session.update", model: ...}` and `commit(final=false)`.
|
||||
3. Stream mono PCM16 at 16 kHz; **Send turn** flushes buffered audio, then sends `commit(final=true)`.
|
||||
4. `response.output_audio.delta.audio` contains PCM; `response.output_audio.done`
|
||||
terminates this adapter, including a response with no audio.
|
||||
5. The local implementation emits **model output** on `transcription.*`, so the
|
||||
UI displays it as assistant text. This differs from the early RFC's suggested
|
||||
user-transcript mapping. The VAD adapter also supports `response.output_text.*`
|
||||
and audio-transcript events without duplicating the answer.
|
||||
|
||||
## Qwen3 with Server VAD (automatic turns)
|
||||
|
||||
Run commands from the repository root with the vLLM-Omni environment activated.
|
||||
Prepare the pinned Silero v6.2 ONNX artifact as described in the
|
||||
[Qwen3 Server VAD instructions](../qwen3_omni/README.md#realtime-websocket-client-openai_realtime_clientpy).
|
||||
The backend requires ONNX Runtime and a compatible local artifact; setting
|
||||
`--vad` on the UI alone does not enable VAD on the backend.
|
||||
|
||||
Create an overlay named `qwen3_vad.yaml`, replacing both paths with absolute paths
|
||||
on the backend host:
|
||||
|
||||
```yaml
|
||||
base_config: /path/to/vllm-omni/vllm_omni/deploy/qwen3_omni_duplex.yaml
|
||||
duplex_session:
|
||||
server_vad_model_path: /path/to/silero_vad.onnx
|
||||
```
|
||||
|
||||
The base configuration enables duplex mode and sets the per-prompt audio and image limits.
|
||||
|
||||
Start the backend:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config qwen3_vad.yaml
|
||||
```
|
||||
|
||||
In a second terminal, wait until the health check succeeds, then start the UI:
|
||||
|
||||
```bash
|
||||
curl --fail http://127.0.0.1:8091/health
|
||||
python -m examples.online_serving.qwen3_omni.realtime_web \
|
||||
--backend ws://127.0.0.1:8091 --vad --port 7863
|
||||
```
|
||||
|
||||
Open `http://localhost:7863` and start a session. Speak and pause: the server
|
||||
commits the turn automatically after 500 ms of silence. No **Send turn** action
|
||||
is needed. Defaults are a speech threshold of 0.5 and 300 ms of prefix padding.
|
||||
Speak again to interrupt an answer. The engine owns input buffering, VAD,
|
||||
request cancellation, playback acknowledgements, and conversation history.
|
||||
Each committed utterance starts a normal Thinker → Talker → Code2Wav request;
|
||||
it does not append input to a live Qwen KV cache. The capability response reports
|
||||
`implementation_level: turn_based_duplex` and `supports_core_resumable_request: false`.
|
||||
|
||||
The plugin retains up to four audio inputs (including the current turn), pruning
|
||||
older audio at an 8 MiB base64 budget; one current utterance is bounded by the
|
||||
engine's pending-input limit. The UI reports playback progress both on normal
|
||||
drain and on interruption. Qwen does not provide text/audio alignment, so assistant
|
||||
text enters history only after the complete audio response has been generated and
|
||||
acknowledged as played. A partially played answer contributes no guessed text
|
||||
prefix. Image conversation items remain available until deleted. Model context limits still apply.
|
||||
The bundled duplex deploy allows four audio inputs and eight images per prompt.
|
||||
Use headphones to avoid speaker audio triggering VAD interruption. Reference
|
||||
voices and tool calls are not supported by this plugin.
|
||||
|
||||
### Switching between VAD and manual turns
|
||||
|
||||
| Mode | Backend | UI flag | Submit a turn |
|
||||
| --- | --- | --- | --- |
|
||||
| Without VAD | Default Qwen turn deployment | `--stt` (default) | Press **Send turn** |
|
||||
| With VAD | Server VAD configuration and Silero artifact | `--vad` | Pause after speaking |
|
||||
|
||||
Stop the existing UI process before starting another on the same port, then
|
||||
refresh the browser and reconnect. To use STT, start a backend with the default
|
||||
turn deployment and restart the UI with `--stt`. The duplex backend exposes
|
||||
Realtime duplex and ChatCompletion; it does not expose the legacy STT handler.
|
||||
To enable VAD on a default backend, first restart that backend with the overlay.
|
||||
Make `--backend` match the actual backend port; the UI and backend use separate
|
||||
ports. `Connection refused` means the target backend is unavailable: check its
|
||||
startup log and `/health` before connecting.
|
||||
|
||||
VAD connects with `duplex=1` and sends nested `session.audio.input` configuration
|
||||
with `create_response: true` and `interrupt_response: true`. It sends playback
|
||||
ACKs when speaker audio drains. `response.audio.done` / `response.output_audio.done`
|
||||
only drain playback; `response.done` terminates a turn. Backpressure displays a
|
||||
notice without pausing the microphone. Failed handshakes time out after 15 seconds;
|
||||
Silero initialization failures display a specific error.
|
||||
|
||||
After updating the backend Python code, restart the backend, refresh the UI, and
|
||||
start a new session so the plugin and browser assets use the updated code. A
|
||||
successful `/health` check confirms readiness, not which deployment mode is active.
|
||||
|
||||
## Qwen VAD camera input
|
||||
|
||||
After starting a session with `--vad`, click **Camera** and allow access. The UI
|
||||
sends one JPEG per second, resized to a longest side of 448 pixels, independently
|
||||
of microphone uploads (including while muted). Each frame is a user conversation
|
||||
item using the [OpenAI Realtime image input](https://developers.openai.com/api/docs/guides/realtime-conversations#image-inputs) shape:
|
||||
|
||||
```json
|
||||
{"type":"conversation.item.create","item":{"id":"camera_1","type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/jpeg;base64,..."}]}}
|
||||
```
|
||||
|
||||
Images become conversation context immediately; they do not start a response.
|
||||
Speak and let VAD commit your audio, or send `response.create` after adding an
|
||||
image/text item. A manual `input_audio_buffer.commit` only commits audio; follow
|
||||
it with `response.create` to generate. Images persist across audio commits,
|
||||
interruptions, and audio-buffer clears. Remove them with `conversation.item.delete`.
|
||||
The browser retains eight camera items and deletes the oldest before adding a
|
||||
ninth. Camera-off stops capture but leaves those items in the conversation.
|
||||
|
||||
This deployment accepts JPEG/PNG Base64 data URLs with a total image-context
|
||||
budget of eight images and 4 MiB; external image URLs are not supported. Clients
|
||||
must delete old image items when reaching the limit. There is no precise
|
||||
video/audio timestamp alignment or native streaming-video KV update.
|
||||
|
||||
### Protocol compatibility boundary
|
||||
|
||||
Qwen VAD uses `session.update`, 24 kHz PCM16 `input_audio_buffer.append`,
|
||||
`conversation.item.create` with `input_image`/`input_text`, and `response.create`.
|
||||
The engine resamples input to Qwen's 16 kHz processing rate. Qwen rejects the
|
||||
custom `video_frames` field on an append: it answers whole turns, so it has no
|
||||
audio unit for a frame track to align to. MiniCPM keeps that field for its
|
||||
native camera protocol, where Stage 0 interleaves frames at unit boundaries.
|
||||
|
||||
This is a supported subset, not full OpenAI Realtime API compatibility.
|
||||
`duplex=1`, `overlap_policy`, `playback.ack`, and `session.close` remain vLLM-Omni
|
||||
extensions. The legacy `--stt` mode is unchanged and is not the standard Realtime
|
||||
conversation API. This demo does not add WebRTC, semantic VAD, or tool calling.
|
||||
|
||||
## Shared host options
|
||||
|
||||
The shared entry point can select either profile explicitly:
|
||||
|
||||
```bash
|
||||
python -m examples.online_serving.realtime_web --profile qwen3-turn \
|
||||
--backend ws://127.0.0.1:8091 --stt --port 7863
|
||||
```
|
||||
|
||||
- `--backend` / `--ws-backend`: backend WebSocket origin (not the full `/v1/realtime` URL).
|
||||
- `--model`: override the wrapper's model name.
|
||||
- `--public-realtime-url`: optional browser-visible WebSocket URL; otherwise the
|
||||
static host proxies `/v1/realtime` on the same origin.
|
||||
- `--host`, `--port`: UI bind address and port (default port 7862).
|
||||
- `--ref-audio`: required for MiniCPM; rejected for Qwen3.
|
||||
|
||||
Microphone and camera access require `localhost` or HTTPS. For a remote backend,
|
||||
use an SSH tunnel to the UI host or an HTTPS reverse proxy with WebSocket support.
|
||||
An explicit `wss://` public URL retains its scheme.
|
||||
|
||||
## Validation
|
||||
|
||||
```bash
|
||||
node --test tests/examples/test_realtime_web_profiles.cjs
|
||||
pytest tests/examples/test_minicpmo_realtime_web_server.py \
|
||||
tests/examples/test_minicpmo_realtime_web_static.py \
|
||||
tests/examples/test_realtime_web_profiles.py
|
||||
pytest -m cpu tests/model_executor/models/qwen3_omni/test_duplex_plugin.py \
|
||||
tests/engine/duplex/test_session_runner.py \
|
||||
tests/engine/test_duplex_orchestrator.py
|
||||
```
|
||||
|
||||
These CPU tests cover profile messages, two STT turns, VAD terminal/drain ordering,
|
||||
backpressure, native capture during playback, server config injection, and shared
|
||||
worklets. Backend regressions cover current audio inclusion in history-based
|
||||
prompts, delayed playback ACK ordering, and no extra generation during silence
|
||||
after a completed VAD turn. Hardware acceptance still requires a MiniCPM call
|
||||
with camera/barge-in and two Qwen turns against each enabled backend, including
|
||||
speech interruption in VAD mode; simulated tests do not
|
||||
establish model quality or hardware latency.
|
||||
@@ -0,0 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared browser audio UI and model-specific realtime profiles."""
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from .server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+12
-11
@@ -4,37 +4,35 @@
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="light">
|
||||
<title>MiniCPM-o 4.5 Voice</title>
|
||||
<title>Omni Realtime Voice</title>
|
||||
<link rel="stylesheet" href="static/styles.css">
|
||||
</head>
|
||||
<body>
|
||||
<header class="app-header">
|
||||
<div>
|
||||
<p class="eyebrow">Experimental full-duplex runtime</p>
|
||||
<h1>MiniCPM-o 4.5 Voice</h1>
|
||||
<p id="pageEyebrow" class="eyebrow"></p>
|
||||
<h1 id="pageTitle">Omni Realtime Voice</h1>
|
||||
</div>
|
||||
<span id="connectionState" class="status status-offline">Offline</span>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<p id="profileDescription" role="note"></p>
|
||||
<section class="call-bar" aria-label="Session controls">
|
||||
<div class="call-actions">
|
||||
<button id="callButton" class="primary" type="button">Start session</button>
|
||||
<button id="sendTurnButton" class="secondary" type="button" hidden disabled>Send turn</button>
|
||||
<button id="muteButton" class="secondary" type="button" disabled>Mute</button>
|
||||
<button id="cameraButton" class="secondary" type="button" disabled>Camera</button>
|
||||
<video id="cameraPreview" muted playsinline style="display:none;width:96px;height:72px;border-radius:8px;object-fit:cover;background:#000"></video>
|
||||
</div>
|
||||
<div style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
|
||||
<div id="promptControls" style="display:flex;gap:8px;align-items:flex-start;margin-top:8px;width:100%">
|
||||
<label style="display:inline-flex;flex-direction:column;gap:4px;font-size:13px;opacity:.85;white-space:nowrap">
|
||||
Prompt preset
|
||||
<select id="promptPreset" style="font-size:13px;padding:4px 6px;border-radius:6px">
|
||||
<option value="omni" selected>Omni (official omni preset)</option>
|
||||
<option value="chinese_call">中文通话 (official audio preset)</option>
|
||||
<option value="english_call">English Call (official audio preset)</option>
|
||||
<option value="custom">Custom</option>
|
||||
</select>
|
||||
</label>
|
||||
<textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical">Streaming Omni Conversation.</textarea>
|
||||
<textarea id="systemPrompt" rows="2" placeholder="System prompt" style="flex:1;font-size:13px;padding:6px 8px;border-radius:6px;resize:vertical"></textarea>
|
||||
</div>
|
||||
<div class="meter" aria-label="Microphone level">
|
||||
<span>Mic</span>
|
||||
@@ -53,7 +51,7 @@
|
||||
<p class="eyebrow">Live transcript</p>
|
||||
<h2 id="conversationTitle">Conversation</h2>
|
||||
</div>
|
||||
<span class="policy-label">Model-controlled listen / speak</span>
|
||||
<span id="policyLabel" class="policy-label"></span>
|
||||
</div>
|
||||
<div id="conversation" class="conversation" aria-live="polite">
|
||||
<p id="emptyConversation" class="empty-state">Start a session and speak naturally.</p>
|
||||
@@ -73,7 +71,10 @@
|
||||
</details>
|
||||
</main>
|
||||
|
||||
<script>window.FULL_DUPLEX_CONFIG = __FULL_DUPLEX_CONFIG__;</script>
|
||||
<script>window.OMNI_REALTIME_CONFIG = __FULL_DUPLEX_CONFIG__; window.FULL_DUPLEX_CONFIG = window.OMNI_REALTIME_CONFIG;</script>
|
||||
<script src="static/profiles/common.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
|
||||
<script src="static/profiles/minicpm_native.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
|
||||
<script src="static/profiles/qwen3_turn.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
|
||||
<script src="static/app.js?v=__FULL_DUPLEX_APP_VERSION__"></script>
|
||||
</body>
|
||||
</html>
|
||||
+335
-142
@@ -1,8 +1,10 @@
|
||||
(() => {
|
||||
'use strict';
|
||||
|
||||
const config = window.FULL_DUPLEX_CONFIG || {};
|
||||
const config = window.OMNI_REALTIME_CONFIG || window.FULL_DUPLEX_CONFIG || {};
|
||||
const profile = window.OmniRealtimeProfiles[config.profile || 'minicpm-native'](config);
|
||||
const callButton = document.getElementById('callButton');
|
||||
const sendTurnButton = document.getElementById('sendTurnButton');
|
||||
const muteButton = document.getElementById('muteButton');
|
||||
const cameraButton = document.getElementById('cameraButton');
|
||||
const cameraPreview = document.getElementById('cameraPreview');
|
||||
@@ -27,20 +29,26 @@
|
||||
const INITIAL_PLAYBACK_BUFFER_MS = 400;
|
||||
const SESSION_CLOSE_TIMEOUT_MS = 1000;
|
||||
|
||||
// Default prompts mirroring the official MiniCPM-o-Demo presets
|
||||
// (assets/presets/{omni,audio_duplex}/*.yaml).
|
||||
const PROMPT_PRESETS = {
|
||||
omni: 'Streaming Omni Conversation.',
|
||||
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
|
||||
+ '请用高自然度的方式和用户聊天。你处于双工模式,可以一边听、一边说。'
|
||||
+ '你是由面壁智能开发的人工智能助手:面壁小钢炮。',
|
||||
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
|
||||
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
|
||||
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
|
||||
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
|
||||
};
|
||||
const PROMPT_PRESETS = profile.presets;
|
||||
document.title = profile.title;
|
||||
document.getElementById('pageTitle').textContent = profile.title;
|
||||
document.getElementById('pageEyebrow').textContent = profile.eyebrow;
|
||||
document.getElementById('profileDescription').textContent = profile.description;
|
||||
document.getElementById('policyLabel').textContent = profile.policy;
|
||||
cameraButton.hidden = !profile.camera;
|
||||
sendTurnButton.hidden = !profile.clientCommit;
|
||||
promptPreset.replaceChildren();
|
||||
for (const name of [...Object.keys(PROMPT_PRESETS), 'custom']) {
|
||||
const option = document.createElement('option');
|
||||
option.value = name;
|
||||
option.textContent = name.replaceAll('_', ' ');
|
||||
promptPreset.appendChild(option);
|
||||
}
|
||||
systemPromptInput.value = Object.values(PROMPT_PRESETS)[0];
|
||||
document.getElementById('promptControls').hidden = !profile.instructions;
|
||||
|
||||
let socket = null;
|
||||
let closingSocket = null;
|
||||
let mediaStream = null;
|
||||
let captureContext = null;
|
||||
let captureNode = null;
|
||||
@@ -65,6 +73,70 @@
|
||||
let liveUserTurn = null;
|
||||
let liveAssistantTurn = null;
|
||||
let sessionCloseResolver = null;
|
||||
let responseComplete = false;
|
||||
let playbackComplete = true;
|
||||
let turnSubmitted = false;
|
||||
let echoTimer = null;
|
||||
let sessionGeneration = 0;
|
||||
let audioChain = Promise.resolve();
|
||||
// Retain interrupted IDs until startSession() resets the session. Queued
|
||||
// events and pending audio decodes can outlive response.done, so removing
|
||||
// IDs there could resume interrupted playback. This set grows with the
|
||||
// interruptions in a call; sessionGeneration rejects old-session decodes
|
||||
// after the set is cleared for a new session.
|
||||
const interruptedResponses = new Set();
|
||||
const pendingEvents = new Set();
|
||||
let assistantTextChannel = null;
|
||||
let connectionReady = false;
|
||||
let turnCounter = 0;
|
||||
let stopping = null;
|
||||
let turnTimeout = null;
|
||||
|
||||
function markBusy() {
|
||||
assistantActive = true;
|
||||
if (profile.halfDuplex) pendingCapture = [];
|
||||
}
|
||||
|
||||
function armTurnTimeout() {
|
||||
clearTimeout(turnTimeout);
|
||||
turnTimeout = window.setTimeout(() => {
|
||||
failSession('Timed out waiting for the model response. Check the backend event log and restart the session.');
|
||||
}, 120000);
|
||||
}
|
||||
|
||||
async function failSession(message) {
|
||||
appendLog(message, true);
|
||||
await stopSession({ terminal: false });
|
||||
setConnection('Error', 'error');
|
||||
runtimeDetail.textContent = message;
|
||||
}
|
||||
|
||||
function finishResponseIfReady() {
|
||||
if (!responseComplete || !playbackComplete || echoTimer !== null) return;
|
||||
const generation = sessionGeneration;
|
||||
echoTimer = window.setTimeout(async () => {
|
||||
echoTimer = null;
|
||||
if (!running || generation !== sessionGeneration) return;
|
||||
if (profile.reconnectEachTurn) {
|
||||
connectionReady = false;
|
||||
const previous = socket;
|
||||
socket = null;
|
||||
if (previous) { previous.onclose = null; previous.onmessage = null; previous.close(); }
|
||||
try { await openSocket(); }
|
||||
catch (error) { await failSession(error.message); return; }
|
||||
if (!running || generation !== sessionGeneration) return;
|
||||
}
|
||||
assistantActive = false;
|
||||
responseComplete = false;
|
||||
turnSubmitted = false;
|
||||
currentResponseId = null;
|
||||
responseHasAudio = false;
|
||||
assistantTextChannel = null;
|
||||
if (profile.halfDuplex) pendingCapture = [];
|
||||
sendTurnButton.disabled = !profile.clientCommit;
|
||||
setModel(profile.waiting);
|
||||
}, ECHO_GUARD_MS);
|
||||
}
|
||||
|
||||
function staticAssetUrl(path) {
|
||||
const version = String(config.appVersion || '').trim();
|
||||
@@ -82,13 +154,7 @@
|
||||
}
|
||||
|
||||
function realtimeUrl() {
|
||||
const url = new URL(config.realtimePath, window.location.href);
|
||||
url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
url.searchParams.set('duplex', '1');
|
||||
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
|
||||
url.searchParams.set('native_duplex', '1');
|
||||
url.searchParams.set('autostart', '0');
|
||||
return url.toString();
|
||||
return profile.url(config, window.location.href);
|
||||
}
|
||||
|
||||
function setConnection(label, kind) {
|
||||
@@ -251,7 +317,7 @@
|
||||
}
|
||||
|
||||
function microphoneUploadEnabled() {
|
||||
return running && !muted;
|
||||
return running && connectionReady && !muted && (!profile.halfDuplex || !assistantActive);
|
||||
}
|
||||
|
||||
function flushCapture() {
|
||||
@@ -268,19 +334,9 @@
|
||||
offset += chunk.length;
|
||||
}
|
||||
pendingCapture = [];
|
||||
const pcm = resampleInt16(merged, captureRate, INPUT_RATE);
|
||||
const appendEvent = {
|
||||
type: 'input_audio_buffer.append',
|
||||
audio: int16ToBase64(pcm),
|
||||
format: 'pcm16',
|
||||
sample_rate_hz: INPUT_RATE,
|
||||
};
|
||||
// Omni duplex: ~1 fps camera frame rides the audio append (official
|
||||
// MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk).
|
||||
if (cameraPendingFrame) {
|
||||
appendEvent.video_frames = [cameraPendingFrame];
|
||||
cameraPendingFrame = null;
|
||||
}
|
||||
const pcm = resampleInt16(merged, captureRate, profile.inputSampleRate || INPUT_RATE);
|
||||
const appendEvent = profile.append(int16ToBase64(pcm), cameraPendingFrame);
|
||||
cameraPendingFrame = null;
|
||||
socket.send(JSON.stringify(appendEvent));
|
||||
}
|
||||
|
||||
@@ -288,13 +344,17 @@
|
||||
currentResponseId = responseId || currentResponseId;
|
||||
responseHasAudio = false;
|
||||
assistantActive = true;
|
||||
setModel('Speaking');
|
||||
responseComplete = false;
|
||||
playbackComplete = true;
|
||||
if (profile.halfDuplex) pendingCapture = [];
|
||||
setModel(profile.halfDuplex ? 'Thinking' : 'Speaking');
|
||||
}
|
||||
|
||||
function feedPlayback(decoded, responseId) {
|
||||
if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
|
||||
const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
|
||||
responseHasAudio = true;
|
||||
playbackComplete = false;
|
||||
assistantActive = true;
|
||||
setPlayback('Buffering');
|
||||
playbackNode.port.postMessage({
|
||||
@@ -306,6 +366,7 @@
|
||||
}
|
||||
|
||||
function requestPlaybackDrain(responseId) {
|
||||
if (!responseHasAudio) { playbackComplete = true; finishResponseIfReady(); return; }
|
||||
if (!playbackNode) return;
|
||||
playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId });
|
||||
}
|
||||
@@ -315,90 +376,133 @@
|
||||
if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true);
|
||||
return;
|
||||
}
|
||||
socket.send(JSON.stringify({
|
||||
type: 'playback.ack',
|
||||
response_id: responseId,
|
||||
item_id: `item_${responseId}`,
|
||||
played_ms: playedMs,
|
||||
committed_ms: playedMs,
|
||||
}));
|
||||
const ack = profile.ack(responseId, playedMs);
|
||||
if (ack) socket.send(JSON.stringify(ack));
|
||||
}
|
||||
|
||||
function playbackDrained(message) {
|
||||
const responseId = message.responseId || currentResponseId;
|
||||
sendPlaybackAck(responseId, Number(message.playedMs) || 0);
|
||||
if (currentResponseId && responseId !== currentResponseId) return;
|
||||
if (profile.playbackAck) sendPlaybackAck(responseId, Number(message.playedMs) || 0);
|
||||
setPlayback('Idle');
|
||||
if (message.underrunMs > 0) {
|
||||
appendLog(`playback underrun ${message.underrunMs} ms`);
|
||||
}
|
||||
window.setTimeout(() => {
|
||||
assistantActive = false;
|
||||
currentResponseId = null;
|
||||
responseHasAudio = false;
|
||||
if (running) setModel('Listening');
|
||||
}, ECHO_GUARD_MS);
|
||||
playbackComplete = true;
|
||||
if (message.underrunMs > 0) appendLog(`playback underrun ${message.underrunMs} ms`);
|
||||
if (!profile.halfDuplex && !profile.waitForResponseDone) responseComplete = true;
|
||||
finishResponseIfReady();
|
||||
}
|
||||
|
||||
function handleEvent(event) {
|
||||
async function handleAudioEvent(action) {
|
||||
markBusy();
|
||||
currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
|
||||
setModel('Speaking');
|
||||
const generation = sessionGeneration;
|
||||
const responseId = currentResponseId;
|
||||
const decoded = await decodeAudioDelta(action.event);
|
||||
if (generation === sessionGeneration && !interruptedResponses.has(responseId)) feedPlayback(decoded, responseId);
|
||||
}
|
||||
|
||||
function handleTranscriptEvent(action) {
|
||||
if (action.role === 'assistant') {
|
||||
// Qwen may emit both text and audio-transcript representations.
|
||||
// Display one channel per response instead of duplicating the answer.
|
||||
if (profile.deduplicateTranscript && assistantTextChannel && action.channel !== assistantTextChannel) return;
|
||||
if (action.text) assistantTextChannel = action.channel || assistantTextChannel;
|
||||
}
|
||||
if (action.kind === 'text') addTranscript(action.role, action.text);
|
||||
else if (profile.deduplicateTranscript && action.role === 'assistant') {
|
||||
if (action.text) {
|
||||
const turn = ensureTurn('assistant');
|
||||
turn.value = action.text;
|
||||
turn.text.textContent = action.text;
|
||||
}
|
||||
} else finishTranscript(action.role, action.text);
|
||||
}
|
||||
|
||||
async function handleEvent(event, action = profile.mapEvent(event)) {
|
||||
appendEventLog(event);
|
||||
const responseId = responseIdOf(event);
|
||||
switch (event.type) {
|
||||
case 'session.created':
|
||||
case 'session.updated':
|
||||
const responseId = action.responseId || responseIdOf(event);
|
||||
if (responseId && interruptedResponses.has(responseId) && action.kind !== 'interrupt') return;
|
||||
switch (action.kind) {
|
||||
case 'connected':
|
||||
setConnection('Connected', 'online');
|
||||
setModel('Listening');
|
||||
if (!assistantActive) setModel(profile.waiting);
|
||||
break;
|
||||
case 'response.listen':
|
||||
case 'interrupt': {
|
||||
if (responseId) interruptedResponses.add(responseId);
|
||||
// A late cancellation belongs to its own response, not the newer one
|
||||
// currently playing. Speech-start events interrupt the local playback.
|
||||
if (responseId && responseId !== currentResponseId) break;
|
||||
if (currentResponseId) interruptedResponses.add(currentResponseId);
|
||||
sessionGeneration += 1;
|
||||
audioChain = Promise.resolve();
|
||||
// Move queued events off the interrupted decode, preserving any newer
|
||||
// response already received. Its chunks must not be lost with the old one.
|
||||
const queued = [...pendingEvents];
|
||||
pendingEvents.clear();
|
||||
for (const pending of queued) enqueueEvent(pending.event, pending.action, pending.source);
|
||||
clearTimeout(echoTimer);
|
||||
echoTimer = null;
|
||||
// Local playback completes immediately on interruption, including
|
||||
// when the server already completed generation.
|
||||
clearTimeout(turnTimeout);
|
||||
if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
|
||||
responseComplete = true;
|
||||
playbackComplete = true;
|
||||
responseHasAudio = false;
|
||||
assistantActive = false;
|
||||
setModel('Listening');
|
||||
currentResponseId = null;
|
||||
assistantTextChannel = null;
|
||||
finishTranscript('assistant');
|
||||
setPlayback('Idle');
|
||||
setModel(profile.waiting);
|
||||
break;
|
||||
case 'response.created':
|
||||
case 'response.speak':
|
||||
}
|
||||
case 'listen':
|
||||
assistantActive = false;
|
||||
setModel(profile.waiting);
|
||||
break;
|
||||
case 'begin':
|
||||
if (profile.halfDuplex) armTurnTimeout();
|
||||
if (echoTimer !== null) { clearTimeout(echoTimer); echoTimer = null; }
|
||||
beginAssistant(responseId);
|
||||
break;
|
||||
case 'response.audio.delta':
|
||||
currentResponseId = responseId || currentResponseId;
|
||||
assistantActive = true;
|
||||
setModel('Speaking');
|
||||
decodeAudioDelta(event)
|
||||
.then((decoded) => feedPlayback(decoded, responseId))
|
||||
.catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true));
|
||||
case 'audio':
|
||||
await handleAudioEvent(action);
|
||||
break;
|
||||
case 'response.audio.done':
|
||||
case 'drain':
|
||||
requestPlaybackDrain(responseId);
|
||||
break;
|
||||
case 'response.audio_transcript.delta':
|
||||
addTranscript('assistant', event.delta || '');
|
||||
case 'text': case 'text-final':
|
||||
handleTranscriptEvent(action);
|
||||
break;
|
||||
case 'response.audio_transcript.done':
|
||||
finishTranscript('assistant', event.transcript || '');
|
||||
break;
|
||||
case 'conversation.item.input_audio_transcription.delta':
|
||||
addTranscript('user', event.delta || '');
|
||||
break;
|
||||
case 'conversation.item.input_audio_transcription.completed':
|
||||
finishTranscript('user', event.transcript || '');
|
||||
break;
|
||||
case 'response.done':
|
||||
case 'done':
|
||||
clearTimeout(turnTimeout);
|
||||
responseComplete = true;
|
||||
finishTranscript('assistant');
|
||||
if (!responseHasAudio) requestPlaybackDrain(responseId);
|
||||
requestPlaybackDrain(responseId);
|
||||
finishResponseIfReady();
|
||||
break;
|
||||
case 'playback.acknowledged':
|
||||
{
|
||||
const acknowledgement = event.event || event;
|
||||
runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`;
|
||||
}
|
||||
case 'backpressure':
|
||||
armTurnTimeout();
|
||||
markBusy();
|
||||
setModel('Thinking / Speaking');
|
||||
runtimeDetail.textContent = action.message;
|
||||
break;
|
||||
case 'session.closed':
|
||||
case 'ack':
|
||||
runtimeDetail.textContent = `Playback committed ${action.committedMs} ms`;
|
||||
break;
|
||||
case 'closed':
|
||||
if (sessionCloseResolver) sessionCloseResolver();
|
||||
sessionCloseResolver = null;
|
||||
break;
|
||||
case 'error':
|
||||
setConnection('Error', 'error');
|
||||
runtimeDetail.textContent = String(event.error || event.code || 'Server error');
|
||||
break;
|
||||
default:
|
||||
if (action.fatal) { await failSession(action.message); break; }
|
||||
// A rejected frame or a refused update costs one operation, not the
|
||||
// call: report it and keep listening.
|
||||
appendLog(action.message, true);
|
||||
runtimeDetail.textContent = action.message;
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,8 +511,15 @@
|
||||
playbackRate = playbackContext.sampleRate;
|
||||
await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js'));
|
||||
playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback');
|
||||
const currentPlaybackNode = playbackNode;
|
||||
playbackNode.port.onmessage = (message) => {
|
||||
if (playbackNode !== currentPlaybackNode) return;
|
||||
if (message.data.type === 'playback-started') setPlayback('Playing');
|
||||
else if (message.data.type === 'playback-stopped') {
|
||||
// The old response may report after the next one has started. ACK its
|
||||
// own cursor without changing the current response's UI state.
|
||||
if (profile.playbackAck) sendPlaybackAck(message.data.responseId, Number(message.data.playedMs) || 0);
|
||||
}
|
||||
else if (message.data.type === 'playback-drained') playbackDrained(message.data);
|
||||
else if (message.data.type === 'playback-underrun') {
|
||||
runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`;
|
||||
@@ -449,52 +560,86 @@
|
||||
await captureContext.resume();
|
||||
}
|
||||
|
||||
function enqueueEvent(event, action, source) {
|
||||
const pending = { event, action, source };
|
||||
pendingEvents.add(pending);
|
||||
const generation = sessionGeneration;
|
||||
audioChain = audioChain.then(() => {
|
||||
if (!pendingEvents.delete(pending)) return;
|
||||
if (socket === source || action.kind === 'closed') return handleEvent(event, action);
|
||||
}).catch((error) => {
|
||||
if (generation === sessionGeneration) return failSession(`Server event failed: ${error.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
function openSocket() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = realtimeUrl();
|
||||
socket = new WebSocket(url);
|
||||
const current = new WebSocket(url);
|
||||
socket = current;
|
||||
connectionReady = false;
|
||||
turnCounter += 1;
|
||||
let settled = false;
|
||||
socket.onopen = () => {
|
||||
const timer = window.setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
const extraBody = {
|
||||
auto_response: true,
|
||||
native_duplex: true,
|
||||
};
|
||||
const session = {
|
||||
modalities: ['audio', 'text'],
|
||||
voice: 'default',
|
||||
extra_body: extraBody,
|
||||
};
|
||||
// Reference voice for TTS cloning, provided by the server via
|
||||
// --ref-audio (mirrors the official demo's default ref audio).
|
||||
if (config.refAudio) session.ref_audio = config.refAudio;
|
||||
const instructions = systemPromptInput ? systemPromptInput.value.trim() : '';
|
||||
if (instructions) session.instructions = instructions;
|
||||
socket.send(JSON.stringify({ type: 'session.update', session }));
|
||||
runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
|
||||
current.onclose = null;
|
||||
current.close();
|
||||
reject(new Error(`Session handshake timed out. ${profile.connectionHint}`));
|
||||
}, 15000);
|
||||
const rejectOnce = (message) => {
|
||||
clearTimeout(timer);
|
||||
if (!settled) { settled = true; reject(new Error(message)); }
|
||||
};
|
||||
current.onopen = () => {
|
||||
const instructions = systemPromptInput.value.trim();
|
||||
for (const event of profile.initialMessages(config, instructions)) current.send(JSON.stringify(event));
|
||||
appendLog(`websocket open ${url}`);
|
||||
resolve();
|
||||
};
|
||||
socket.onmessage = (message) => {
|
||||
if (typeof message.data !== 'string') return;
|
||||
try {
|
||||
handleEvent(JSON.parse(message.data));
|
||||
} catch (error) {
|
||||
appendLog(`invalid server event: ${error.message || error}`, true);
|
||||
current.onmessage = (message) => {
|
||||
if (typeof message.data !== 'string' || (socket !== current && closingSocket !== current)) return;
|
||||
let event;
|
||||
try { event = JSON.parse(message.data); }
|
||||
catch (error) {
|
||||
if (socket !== current) return;
|
||||
const detail = `Invalid server event: ${error.message}`;
|
||||
if (!settled) rejectOnce(detail);
|
||||
else void failSession(detail);
|
||||
return;
|
||||
}
|
||||
};
|
||||
socket.onerror = () => {
|
||||
if (!settled) {
|
||||
// Shutdown has detached the active socket. Only its close acknowledgement
|
||||
// may pass, directly: the event queue can itself be awaiting cleanup.
|
||||
if (closingSocket === current) {
|
||||
if (event?.type === 'session.closed' && sessionCloseResolver) sessionCloseResolver();
|
||||
return;
|
||||
}
|
||||
const action = profile.mapEvent(event);
|
||||
if (action.kind === 'error' && !settled) {
|
||||
rejectOnce(action.message);
|
||||
return;
|
||||
}
|
||||
if (event.type === profile.readyEvent && !settled) {
|
||||
clearTimeout(timer);
|
||||
settled = true;
|
||||
reject(new Error(`WebSocket connection failed: ${url}`));
|
||||
connectionReady = true;
|
||||
runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
|
||||
resolve();
|
||||
}
|
||||
// Stop the speaker immediately, even while an old WAV is decoding.
|
||||
// Ordinary completion still waits for its preceding audio chunks.
|
||||
if (action.kind === 'interrupt') {
|
||||
void handleEvent(event, action);
|
||||
return;
|
||||
}
|
||||
enqueueEvent(event, action, current);
|
||||
};
|
||||
socket.onclose = (event) => {
|
||||
current.onerror = () => rejectOnce(`WebSocket connection failed. ${profile.connectionHint}`);
|
||||
current.onclose = (event) => {
|
||||
clearTimeout(timer);
|
||||
connectionReady = false;
|
||||
appendLog(`websocket closed code=${event.code}`);
|
||||
if (running) {
|
||||
setConnection('Disconnected', 'error');
|
||||
stopSession();
|
||||
}
|
||||
if (!settled) rejectOnce(`Connection closed before session ready. ${profile.connectionHint}`);
|
||||
else if (running && socket === current) failSession(`Backend disconnected (${event.code}). ${profile.connectionHint}`);
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -517,46 +662,61 @@
|
||||
setConnection('Connecting', 'connecting');
|
||||
runtimeDetail.textContent = 'Requesting microphone access';
|
||||
try {
|
||||
sessionGeneration += 1;
|
||||
audioChain = Promise.resolve();
|
||||
interruptedResponses.clear();
|
||||
pendingEvents.clear();
|
||||
await openPlayback();
|
||||
await openCapture();
|
||||
await openSocket();
|
||||
running = true;
|
||||
muted = false;
|
||||
assistantActive = false;
|
||||
sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS);
|
||||
sendTimer = window.setInterval(flushCapture, profile.sendIntervalMs || SEND_INTERVAL_MS);
|
||||
startClock();
|
||||
callButton.textContent = 'End session';
|
||||
callButton.classList.add('is-active');
|
||||
muteButton.disabled = false;
|
||||
cameraButton.disabled = false;
|
||||
cameraButton.disabled = !profile.camera;
|
||||
sendTurnButton.disabled = !profile.clientCommit;
|
||||
setConnection('Connected', 'online');
|
||||
setModel('Listening');
|
||||
setModel(profile.waiting);
|
||||
appendLog('session started');
|
||||
} catch (error) {
|
||||
appendLog(`start failed: ${error.message || error}`, true);
|
||||
await stopSession({ terminal: false });
|
||||
setConnection('Error', 'error');
|
||||
runtimeDetail.textContent = String(error.message || error);
|
||||
await stopSession();
|
||||
} finally {
|
||||
callButton.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function startCamera() {
|
||||
if (cameraStream) return;
|
||||
if (!profile.camera || cameraStream) return;
|
||||
cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
|
||||
cameraPreview.srcObject = cameraStream;
|
||||
cameraPreview.style.display = '';
|
||||
await cameraPreview.play().catch(() => {});
|
||||
// Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
|
||||
// no client-side resize (the server normalizes at scale_resolution=448).
|
||||
cameraTimer = window.setInterval(() => {
|
||||
const captureCameraFrame = () => {
|
||||
if (!cameraStream || cameraPreview.videoWidth === 0) return;
|
||||
cameraCanvas.width = cameraPreview.videoWidth;
|
||||
cameraCanvas.height = cameraPreview.videoHeight;
|
||||
cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0);
|
||||
cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
|
||||
}, 1000);
|
||||
const scale = profile.cameraMaxDimension
|
||||
? Math.min(1, profile.cameraMaxDimension / Math.max(cameraPreview.videoWidth, cameraPreview.videoHeight)) : 1;
|
||||
cameraCanvas.width = Math.max(1, Math.round(cameraPreview.videoWidth * scale));
|
||||
cameraCanvas.height = Math.max(1, Math.round(cameraPreview.videoHeight * scale));
|
||||
cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0, cameraCanvas.width, cameraCanvas.height);
|
||||
const frame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
|
||||
if (profile.imageMessages) {
|
||||
if (connectionReady && socket?.readyState === WebSocket.OPEN) {
|
||||
for (const event of profile.imageMessages(frame)) socket.send(JSON.stringify(event));
|
||||
}
|
||||
} else cameraPendingFrame = frame;
|
||||
};
|
||||
// Do not make the first spoken turn race a one-second timer.
|
||||
captureCameraFrame();
|
||||
cameraTimer = window.setInterval(captureCameraFrame, 1000);
|
||||
cameraButton.textContent = 'Camera off';
|
||||
cameraButton.classList.add('is-active');
|
||||
appendLog('camera on (1 fps omni frames)');
|
||||
@@ -592,15 +752,33 @@
|
||||
const finish = () => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
if (sessionCloseResolver === finish) sessionCloseResolver = null;
|
||||
resolve();
|
||||
};
|
||||
sessionCloseResolver = finish;
|
||||
window.setTimeout(finish, timeoutMs);
|
||||
const timer = window.setTimeout(finish, timeoutMs);
|
||||
});
|
||||
}
|
||||
|
||||
async function stopSession({ terminal = true } = {}) {
|
||||
function stopSession(options = {}) {
|
||||
if (stopping) return stopping;
|
||||
stopping = cleanupSession(options).finally(() => { stopping = null; });
|
||||
return stopping;
|
||||
}
|
||||
|
||||
async function cleanupSession({ terminal = true } = {}) {
|
||||
sessionGeneration += 1;
|
||||
pendingEvents.clear();
|
||||
connectionReady = false;
|
||||
clearTimeout(echoTimer);
|
||||
clearTimeout(turnTimeout);
|
||||
echoTimer = null;
|
||||
responseComplete = false;
|
||||
playbackComplete = true;
|
||||
turnSubmitted = false;
|
||||
assistantTextChannel = null;
|
||||
sendTurnButton.disabled = true;
|
||||
running = false;
|
||||
assistantActive = false;
|
||||
pendingCapture = [];
|
||||
@@ -609,15 +787,16 @@
|
||||
sendTimer = null;
|
||||
clockTimer = null;
|
||||
if (socket) {
|
||||
const closingSocket = socket;
|
||||
closingSocket = socket;
|
||||
socket = null;
|
||||
closingSocket.onclose = null;
|
||||
if (terminal && closingSocket.readyState === WebSocket.OPEN) {
|
||||
if (terminal && profile.closeSession && closingSocket.readyState === WebSocket.OPEN) {
|
||||
const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS);
|
||||
closingSocket.send(JSON.stringify({ type: 'session.close' }));
|
||||
await closed;
|
||||
}
|
||||
closingSocket.close(1000, 'client stop');
|
||||
closingSocket = null;
|
||||
}
|
||||
if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
|
||||
if (mediaStream) {
|
||||
@@ -634,6 +813,8 @@
|
||||
playbackNode = null;
|
||||
currentResponseId = null;
|
||||
responseHasAudio = false;
|
||||
finishTranscript('user');
|
||||
finishTranscript('assistant');
|
||||
meterFill.style.width = '0%';
|
||||
sessionTimer.textContent = '00:00';
|
||||
callButton.textContent = 'Start session';
|
||||
@@ -661,6 +842,18 @@
|
||||
if (running) stopSession();
|
||||
else startSession();
|
||||
});
|
||||
sendTurnButton.addEventListener('click', () => {
|
||||
if (!running || !connectionReady || !profile.clientCommit || turnSubmitted) return;
|
||||
flushCapture();
|
||||
for (const event of profile.commitMessages()) socket.send(JSON.stringify(event));
|
||||
turnSubmitted = true;
|
||||
responseComplete = false;
|
||||
markBusy();
|
||||
armTurnTimeout();
|
||||
sendTurnButton.disabled = true;
|
||||
setModel('Thinking');
|
||||
appendLog('turn submitted');
|
||||
});
|
||||
muteButton.addEventListener('click', toggleMute);
|
||||
clearLogButton.addEventListener('click', () => {
|
||||
eventLog.textContent = '';
|
||||
File renamed without changes.
+7
@@ -39,6 +39,13 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
}
|
||||
this.notifyIfDrained();
|
||||
} else if (message.type === 'clear') {
|
||||
if (this.activeResponseId) {
|
||||
this.port.postMessage({
|
||||
type: 'playback-stopped',
|
||||
responseId: this.activeResponseId,
|
||||
playedMs: Math.round((this.playedFrames * 1000) / sampleRate),
|
||||
});
|
||||
}
|
||||
this.queue = [];
|
||||
this.offset = 0;
|
||||
this.playedFrames = 0;
|
||||
@@ -0,0 +1,56 @@
|
||||
/* Shared wire helpers. Profiles translate server events into shell actions. */
|
||||
(function (root) {
|
||||
'use strict';
|
||||
const profiles = root.OmniRealtimeProfiles = root.OmniRealtimeProfiles || {};
|
||||
profiles.url = (config, location) => {
|
||||
const url = new URL(config.realtimePath || 'v1/realtime', location);
|
||||
if (url.protocol === 'http:') url.protocol = 'ws:';
|
||||
if (url.protocol === 'https:') url.protocol = 'wss:';
|
||||
return url;
|
||||
};
|
||||
/* Errors that leave the session unusable, so the shell has to tear it down.
|
||||
Everything else costs one operation --- a rejected image, a refused
|
||||
session.update, a late playback ack --- and the call carries on.
|
||||
'runtime_append_failed' is deliberately absent: the engine reports it and
|
||||
keeps the session, unlike 'runtime_append_task_failed' which kills it. */
|
||||
profiles.fatalErrorCodes = new Set([
|
||||
'unknown_session',
|
||||
'session_closed',
|
||||
'resource_exhausted',
|
||||
'unsupported',
|
||||
'unsupported_turn_detection',
|
||||
'server_vad_unavailable',
|
||||
'runtime_append_task_failed',
|
||||
'internal_error',
|
||||
]);
|
||||
profiles.event = (event) => {
|
||||
const responseId = event.response_id || event.response?.id || null;
|
||||
const action = (kind, extra = {}) => ({ kind, responseId, ...extra });
|
||||
switch (event.type) {
|
||||
case 'session.created': case 'session.updated': return action('connected');
|
||||
case 'response.created': return action('begin');
|
||||
case 'response.audio.delta': case 'response.output_audio.delta':
|
||||
return action('audio', { event: { ...event, delta: event.delta || event.audio || event.response?.audio } });
|
||||
case 'response.audio.done': case 'response.output_audio.done': return action('drain');
|
||||
case 'response.audio_transcript.delta': case 'response.output_audio_transcript.delta':
|
||||
return action('text', { role: 'assistant', channel: 'audio', text: event.delta || '' });
|
||||
case 'response.audio_transcript.done': case 'response.output_audio_transcript.done':
|
||||
return action('text-final', { role: 'assistant', channel: 'audio', text: event.transcript || '' });
|
||||
case 'conversation.item.input_audio_transcription.delta':
|
||||
return action('text', { role: 'user', text: event.delta || '' });
|
||||
case 'conversation.item.input_audio_transcription.completed':
|
||||
return action('text-final', { role: 'user', text: event.transcript || '' });
|
||||
case 'response.done': return action('done');
|
||||
case 'session.closed': return action('closed');
|
||||
case 'error': {
|
||||
const code = event.code || event.error?.code;
|
||||
return action('error', {
|
||||
code,
|
||||
fatal: profiles.fatalErrorCodes.has(code),
|
||||
message: typeof event.error === 'string' ? event.error : event.error?.message || event.message || code || 'Server error',
|
||||
});
|
||||
}
|
||||
default: return action('ignore');
|
||||
}
|
||||
};
|
||||
})(globalThis);
|
||||
@@ -0,0 +1,55 @@
|
||||
(function (root) {
|
||||
'use strict';
|
||||
const profiles = root.OmniRealtimeProfiles;
|
||||
profiles['minicpm-native'] = () => ({
|
||||
title: 'MiniCPM-o 4.5 Voice', eyebrow: 'Experimental full-duplex runtime',
|
||||
policy: 'Model-controlled listen / speak',
|
||||
description: 'Speak naturally. The model can listen and speak at the same time. Camera frames are sent with your audio.',
|
||||
waiting: 'Listening', camera: true, playbackAck: true, clientCommit: false,
|
||||
halfDuplex: false, closeSession: true, reconnectEachTurn: false,
|
||||
readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
|
||||
presets: {
|
||||
omni: 'Streaming Omni Conversation.',
|
||||
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
|
||||
+ '请用高自然度的方式和用户聊天。你处于双工模式,可以一边听、一边说。'
|
||||
+ '你是由面壁智能开发的人工智能助手:面壁小钢炮。',
|
||||
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
|
||||
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
|
||||
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
|
||||
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
|
||||
},
|
||||
url(config, location) {
|
||||
const url = profiles.url(config, location);
|
||||
url.searchParams.set('duplex', '1');
|
||||
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
|
||||
url.searchParams.set('autostart', '0');
|
||||
return url.toString();
|
||||
},
|
||||
initialMessages(config, instructions) {
|
||||
const session = { modalities: ['audio', 'text'], voice: 'default',
|
||||
extra_body: { auto_response: true } };
|
||||
if (config.refAudio) session.ref_audio = config.refAudio;
|
||||
if (instructions) session.instructions = instructions;
|
||||
return [{ type: 'session.update', session }];
|
||||
},
|
||||
append(audio, frame) {
|
||||
const event = { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
|
||||
if (frame) event.video_frames = [frame];
|
||||
return event;
|
||||
},
|
||||
commitMessages: () => [],
|
||||
ack(responseId, playedMs) {
|
||||
return { type: 'playback.ack', response_id: responseId, item_id: `item_${responseId}`,
|
||||
played_ms: playedMs, committed_ms: playedMs };
|
||||
},
|
||||
mapEvent(event) {
|
||||
if (event.type === 'response.listen') return { kind: 'listen' };
|
||||
if (event.type === 'response.speak') return { kind: 'begin', responseId: event.response_id || event.response?.id };
|
||||
if (event.type === 'playback.acknowledged') {
|
||||
return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
|
||||
}
|
||||
return profiles.event(event);
|
||||
},
|
||||
connectionHint: 'Check the MiniCPM duplex deployment and reference voice configuration.',
|
||||
});
|
||||
})(globalThis);
|
||||
@@ -0,0 +1,116 @@
|
||||
(function (root) {
|
||||
'use strict';
|
||||
const profiles = root.OmniRealtimeProfiles;
|
||||
profiles['qwen3-turn'] = (config) => {
|
||||
const vad = config.adapter === 'vad';
|
||||
let interruptOnSpeech = vad;
|
||||
let imageSeq = 0;
|
||||
let imageItems = [];
|
||||
// The engine refuses a ninth image (or 4 MiB of them) for the whole
|
||||
// session, so the camera track retires its own oldest frame instead.
|
||||
const MAX_IMAGE_ITEMS = 8;
|
||||
const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
|
||||
return {
|
||||
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
|
||||
policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
|
||||
description: vad
|
||||
? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
|
||||
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
|
||||
waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
|
||||
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
|
||||
inputSampleRate: vad ? 24000 : 16000,
|
||||
readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
|
||||
presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
|
||||
url(config, location) {
|
||||
const url = profiles.url(config, location);
|
||||
for (const key of ['native_duplex', 'minicpmo45_native_duplex', 'autostart']) url.searchParams.delete(key);
|
||||
// Explicit opt-out matches the local file client, including servers
|
||||
// that auto-select the duplex endpoint when the query is omitted.
|
||||
url.searchParams.set('duplex', vad ? '1' : '0');
|
||||
url.searchParams.set('model', config.model || 'Qwen/Qwen3-Omni-30B-A3B-Instruct');
|
||||
return url.toString();
|
||||
},
|
||||
initialMessages(config, instructions) {
|
||||
imageItems = [];
|
||||
interruptOnSpeech = vad;
|
||||
if (!vad) return [
|
||||
{ type: 'session.update', model: config.model },
|
||||
{ type: 'input_audio_buffer.commit', final: false },
|
||||
];
|
||||
const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
|
||||
format: { type: 'audio/pcm', rate: 24000 },
|
||||
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
|
||||
silence_duration_ms: 500, create_response: true, interrupt_response: true },
|
||||
} } };
|
||||
if (instructions) session.instructions = instructions;
|
||||
return [{ type: 'session.update', session }];
|
||||
},
|
||||
append(audio) {
|
||||
return vad ? { type: 'input_audio_buffer.append', audio }
|
||||
: { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
|
||||
},
|
||||
// Qwen is a turn model: a frame is a picture added to the conversation,
|
||||
// not a track interleaved into the audio. That is exactly the OpenAI
|
||||
// Realtime image interface, so the camera sends conversation items
|
||||
// rather than an extension field on the append.
|
||||
imageMessages(frame) {
|
||||
if (!vad) return [];
|
||||
const messages = [];
|
||||
if (imageItems.length >= MAX_IMAGE_ITEMS) {
|
||||
// Retire before creating: the engine counts what is already stored.
|
||||
messages.push({ type: 'conversation.item.delete', item_id: imageItems.shift() });
|
||||
}
|
||||
const id = `camera_${++imageSeq}`;
|
||||
imageItems.push(id);
|
||||
messages.push({ type: 'conversation.item.create', item: {
|
||||
id, type: 'message', role: 'user',
|
||||
content: [{ type: 'input_image', image_url: `data:image/jpeg;base64,${frame}` }],
|
||||
} });
|
||||
return messages;
|
||||
},
|
||||
commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
|
||||
ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
|
||||
item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
|
||||
mapEvent(event) {
|
||||
if (event.type === 'session.created' || event.type === 'session.updated') {
|
||||
const input = event.session?.audio?.input;
|
||||
const turnDetection = input && 'turn_detection' in input
|
||||
? input.turn_detection : event.session?.turn_detection;
|
||||
if (turnDetection !== undefined) {
|
||||
interruptOnSpeech = vad && turnDetection?.type === 'server_vad'
|
||||
&& turnDetection.interrupt_response !== false;
|
||||
}
|
||||
}
|
||||
// Generation can finish long before the speaker drains its queue. In
|
||||
// that case the server has no active response left to cancel for us.
|
||||
if (interruptOnSpeech && event.type === 'input_audio_buffer.speech_started') {
|
||||
return { kind: 'interrupt' };
|
||||
}
|
||||
if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
|
||||
if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
|
||||
// On the shipped STT route transcription.* is model-generated text,
|
||||
// not a separate ASR transcript of the user. See realtime_connection.py.
|
||||
return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
|
||||
}
|
||||
if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
|
||||
return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
|
||||
}
|
||||
if (vad && (event.type === 'output_audio_buffer.cleared' ||
|
||||
(event.type === 'response.done' && event.response?.status === 'cancelled'))) {
|
||||
return { kind: 'interrupt', responseId: event.response_id || event.response?.id || null };
|
||||
}
|
||||
if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
|
||||
if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
|
||||
if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
|
||||
const action = profiles.event(event);
|
||||
if (action.kind === 'error') {
|
||||
if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
|
||||
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
|
||||
else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
|
||||
}
|
||||
return action;
|
||||
},
|
||||
connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
|
||||
};
|
||||
};
|
||||
})(globalThis);
|
||||
+3
@@ -125,3 +125,6 @@ button:disabled { opacity: .46; cursor: default; }
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*, *::before, *::after { animation-duration: .01ms !important; animation-iteration-count: 1 !important; }
|
||||
}
|
||||
|
||||
/* Profile-specific controls may have inline flex styles. */
|
||||
[hidden] { display: none !important; }
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Static host and same-origin proxy for the shared realtime browser UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import uvicorn
|
||||
import websockets
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import HTMLResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
APP_DIR = Path(__file__).parent / "app"
|
||||
STATIC_DIR = APP_DIR / "static"
|
||||
|
||||
|
||||
def _join_ws_url(base: str, path: str, query: str) -> str:
|
||||
return base.rstrip("/") + path + (("?" + query) if query else "")
|
||||
|
||||
|
||||
async def _pump_client_to_backend(client: WebSocket, backend) -> None:
|
||||
try:
|
||||
while True:
|
||||
message = await client.receive()
|
||||
if message["type"] == "websocket.disconnect":
|
||||
await backend.close()
|
||||
return
|
||||
if message.get("text") is not None:
|
||||
await backend.send(message["text"])
|
||||
elif message.get("bytes") is not None:
|
||||
await backend.send(message["bytes"])
|
||||
except (WebSocketDisconnect, websockets.ConnectionClosed):
|
||||
with contextlib.suppress(Exception):
|
||||
await backend.close()
|
||||
|
||||
|
||||
async def _pump_backend_to_client(client: WebSocket, backend) -> None:
|
||||
try:
|
||||
async for message in backend:
|
||||
if isinstance(message, bytes):
|
||||
await client.send_bytes(message)
|
||||
else:
|
||||
await client.send_text(message)
|
||||
except (WebSocketDisconnect, websockets.ConnectionClosed):
|
||||
return
|
||||
except RuntimeError as exc:
|
||||
if "websocket.send" in str(exc) and "websocket.close" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def _expected_proxy_close(exc: BaseException) -> bool:
|
||||
if isinstance(exc, (WebSocketDisconnect, websockets.ConnectionClosed, asyncio.CancelledError)):
|
||||
return True
|
||||
return isinstance(exc, RuntimeError) and "websocket.send" in str(exc) and "websocket.close" in str(exc)
|
||||
|
||||
|
||||
def build_app(
|
||||
*,
|
||||
ws_backend: str | None = None,
|
||||
model: str | None = None,
|
||||
public_realtime_url: str | None = None,
|
||||
ref_audio: str | None = None,
|
||||
profile: str = "minicpm-native",
|
||||
adapter: str = "stt",
|
||||
) -> FastAPI:
|
||||
if profile not in {"minicpm-native", "qwen3-turn"}:
|
||||
raise ValueError(f"Unknown profile: {profile}")
|
||||
if adapter not in {"stt", "vad"}:
|
||||
raise ValueError(f"Unknown adapter: {adapter}")
|
||||
if profile == "qwen3-turn" and ref_audio:
|
||||
raise ValueError("Qwen3 profile does not accept --ref-audio")
|
||||
native = profile == "minicpm-native"
|
||||
ws_backend = ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
|
||||
model = model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
||||
app = FastAPI(title="Omni Realtime Web UI")
|
||||
index_path = APP_DIR / "index.html"
|
||||
app_version_hash = hashlib.sha256()
|
||||
for asset_path in (
|
||||
STATIC_DIR / "app.js",
|
||||
STATIC_DIR / "pcm_worklet.js",
|
||||
STATIC_DIR / "playback_worklet.js",
|
||||
*sorted((STATIC_DIR / "profiles").glob("*.js")),
|
||||
):
|
||||
app_version_hash.update(asset_path.read_bytes())
|
||||
app_version = app_version_hash.hexdigest()[:12]
|
||||
|
||||
ref_audio_uri: str | None = None
|
||||
if ref_audio:
|
||||
ref_path = Path(ref_audio)
|
||||
if not ref_path.is_file():
|
||||
raise SystemExit(f"--ref-audio not found: {ref_audio}")
|
||||
encoded = base64.b64encode(ref_path.read_bytes()).decode("ascii")
|
||||
ref_audio_uri = f"data:audio/wav;base64,{encoded}"
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index() -> HTMLResponse:
|
||||
config = json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"profile": profile,
|
||||
"adapter": adapter,
|
||||
"realtimePath": public_realtime_url or "v1/realtime",
|
||||
"refAudio": ref_audio_uri,
|
||||
"appVersion": app_version,
|
||||
},
|
||||
ensure_ascii=True,
|
||||
).replace("<", "\\u003c")
|
||||
html = (
|
||||
index_path.read_text(encoding="utf-8")
|
||||
.replace("__FULL_DUPLEX_CONFIG__", config)
|
||||
.replace("__FULL_DUPLEX_APP_VERSION__", app_version)
|
||||
)
|
||||
return HTMLResponse(html, headers={"Cache-Control": "no-store"})
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> Response:
|
||||
return Response(content="ok", media_type="text/plain")
|
||||
|
||||
@app.websocket("/v1/realtime")
|
||||
async def realtime_proxy(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
query = urlencode(websocket.query_params.multi_items())
|
||||
backend_url = _join_ws_url(ws_backend, "/v1/realtime", query)
|
||||
logger.info("Proxying Realtime WebSocket to %s", backend_url)
|
||||
try:
|
||||
async with websockets.connect(
|
||||
backend_url,
|
||||
max_size=64 * 1024 * 1024,
|
||||
) as backend:
|
||||
tasks = {
|
||||
asyncio.create_task(_pump_client_to_backend(websocket, backend)),
|
||||
asyncio.create_task(_pump_backend_to_client(websocket, backend)),
|
||||
}
|
||||
done, pending = await asyncio.wait(
|
||||
tasks,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
await asyncio.gather(*pending, return_exceptions=True)
|
||||
for result in await asyncio.gather(*done, return_exceptions=True):
|
||||
if isinstance(result, BaseException) and not _expected_proxy_close(result):
|
||||
raise result
|
||||
except (WebSocketDisconnect, websockets.ConnectionClosed):
|
||||
return
|
||||
except (OSError, TimeoutError) as exc:
|
||||
message = (
|
||||
f"Cannot connect to realtime backend {ws_backend}. "
|
||||
"Check that vllm serve is running and --backend matches its --port. "
|
||||
"For containers or remote hosts, use the backend address reachable from the UI server."
|
||||
)
|
||||
logger.warning("%s (%s)", message, exc)
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.send_json({"type": "error", "code": "backend_unavailable", "error": message})
|
||||
await websocket.close(code=1011)
|
||||
except Exception:
|
||||
logger.exception("Realtime WebSocket proxy failed")
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.close(code=1011)
|
||||
finally:
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.close()
|
||||
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
return app
|
||||
|
||||
|
||||
def main(default_profile: str = "minicpm-native") -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=7862)
|
||||
parser.add_argument("--profile", choices=("minicpm-native", "qwen3-turn"), default=default_profile)
|
||||
parser.add_argument("--ws-backend", "--backend", dest="ws_backend")
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--stt", dest="adapter", action="store_const", const="stt")
|
||||
mode.add_argument("--vad", dest="adapter", action="store_const", const="vad")
|
||||
parser.set_defaults(adapter="stt")
|
||||
parser.add_argument(
|
||||
"--public-realtime-url",
|
||||
help="Browser-visible ws:// or wss:// Realtime URL; defaults to the same-origin proxy.",
|
||||
)
|
||||
parser.add_argument("--model")
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
help=(
|
||||
"Reference voice wav for TTS voice cloning, e.g. the "
|
||||
"official MiniCPM-o-Demo assets/ref_audio/ref_minicpm_signature.wav"
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
native = args.profile == "minicpm-native"
|
||||
if native and not args.ref_audio:
|
||||
parser.error("--ref-audio is required for the minicpm-native profile")
|
||||
if not native and args.ref_audio:
|
||||
parser.error("--ref-audio is only supported by minicpm-native")
|
||||
args.model = args.model or ("openbmb/MiniCPM-o-4_5" if native else "Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
||||
args.ws_backend = args.ws_backend or ("ws://127.0.0.1:8099" if native else "ws://127.0.0.1:8091")
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
uvicorn.run(
|
||||
build_app(
|
||||
ws_backend=args.ws_backend,
|
||||
model=args.model,
|
||||
public_realtime_url=args.public_realtime_url,
|
||||
ref_audio=args.ref_audio,
|
||||
profile=args.profile,
|
||||
adapter=args.adapter,
|
||||
),
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level="info",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -21,7 +21,7 @@ The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=Tr
|
||||
| Direction | Message | Format | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. Image-conditioned models take a first frame as `image_reference`: `{"image_url":"https://..."}` or a `data:` URL. |
|
||||
| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt midway through generation. `event_id` is optional and `transition_chunks` defaults to the model setting. |
|
||||
| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt and/or camera midway through generation. Camera at `event.multi_modal_data.camera` is structural SE3 (`mode` + `translation`/`rotation`). `event_id` is optional and `transition_chunks` defaults to the model setting. |
|
||||
| Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
|
||||
| Server to client | `video.chunk_metadata` | JSON text: `{"type":"video.chunk_metadata","request_id":"...","kind":"media","transport_chunk_index":0,"generation_chunk_index":0,"num_frames":9,"byte_length":1234,"started_event_ids":[],"active_event_ids":[],"completed_event_ids":[]}` | Precedes each binary frame and describes the immediately following payload. |
|
||||
| Server to client | Video chunk | Binary WebSocket frame | Fragmented MP4 (`m4s`) video bytes. |
|
||||
@@ -59,10 +59,10 @@ python streaming_video_client.py \
|
||||
--output helios_stream.mp4
|
||||
```
|
||||
|
||||
The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
|
||||
The example client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
|
||||
The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.
|
||||
|
||||
Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them (LingBot-World, for one, does not, and rejects the whole session). Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
|
||||
Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them. Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
@@ -73,6 +73,21 @@ python streaming_video_client.py \
|
||||
]'
|
||||
```
|
||||
|
||||
Schedule midway camera updates with `--camera-updates` when the served model
|
||||
registers the camera modality (e.g., LingBot World 2).
|
||||
It accepts `["camera"]["data"]["translation"/"rotation"]` in the endpoint's required SE3 format.
|
||||
It also accepts `["actions"]` at the update level and converts keystrokes to the SE3 format.
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
--model robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
|
||||
--image-reference /path/to/first_frame.png \
|
||||
--camera-updates '[
|
||||
{"at": 1.0, "actions": ["w"]},
|
||||
{"at": 3.0, "camera": {"mode": "target", "data": {"translation": [0.0, 0.0, 1.0], "rotation": [0.0, 0.0, 0.0, 1.0]}}, "transition_chunks": 2}
|
||||
]'
|
||||
```
|
||||
|
||||
The Helios demo continues the default prompt into an underwater storm:
|
||||
|
||||
```bash
|
||||
@@ -144,6 +159,7 @@ python streaming_video_client.py \
|
||||
--output lingbot_world_v2_stream.mp4
|
||||
```
|
||||
|
||||
`camera_action_script` carries one three-latent-frame WASD action list per generated
|
||||
chunk. Mid-session `session.interaction` only updates the prompt, so a LingBot rollout
|
||||
follows the camera script it started with.
|
||||
`camera_action_script` is an optional request-scoped WASD script (one
|
||||
three-latent-frame action list per generated chunk). Prefer mid-session
|
||||
`session.interaction` camera control via `--camera-updates` when you need
|
||||
live motion; the engine wire format is structural SE3 only.
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Client-side helpers that map WASD/IJKL keys to structural SE3 camera payloads.
|
||||
Convenience for demos.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
# Unity-frame (+X right, +Y up, +Z forward) step sizes used by the optional
|
||||
# WASD helper. Magnitudes mirror LingBot's script integrator; the engine never
|
||||
# sees key tokens — only structural SE3 payloads.
|
||||
_WASD_STEP = 0.05
|
||||
_WASD_PITCH_DEG = 4.0
|
||||
_WASD_YAW_DEG = 6.0
|
||||
# Match LingBot's per-frame pitch clamp; without absolute pose this only limits
|
||||
# a single helper call, not cumulative held velocity.
|
||||
_WASD_PITCH_LIMIT_DEG = 85.0
|
||||
|
||||
|
||||
def _quat_from_axis_angle(axis: tuple[float, float, float], degrees: float) -> list[float]:
|
||||
"""Return a unit quaternion ``[x, y, z, w]`` for a right-handed axis-angle."""
|
||||
radians = math.radians(degrees)
|
||||
half = 0.5 * radians
|
||||
sine = math.sin(half)
|
||||
return [axis[0] * sine, axis[1] * sine, axis[2] * sine, math.cos(half)]
|
||||
|
||||
|
||||
def wasd_to_camera_payload(actions: list[str], *, mode: str = "velocity") -> dict[str, Any]:
|
||||
"""Map WASD/IJKL key tokens to a structural camera interaction payload.
|
||||
|
||||
This is a client convenience only. Servers reject ``data.actions``.
|
||||
|
||||
The delta is chosen so that at the identity pose, ``T_base @ T_delta``
|
||||
matches LingBot's script integrator rotation
|
||||
``R_y(yaw) @ R_x(pitch)``. Without the current absolute pose this remains
|
||||
an approximation once the camera has already yawed/pitched.
|
||||
"""
|
||||
keys = {str(action).lower() for action in actions}
|
||||
dx = _WASD_STEP * (("d" in keys) - ("a" in keys))
|
||||
dy = 0.0
|
||||
dz = _WASD_STEP * (("w" in keys) - ("s" in keys))
|
||||
pitch = _WASD_PITCH_DEG * (("i" in keys) - ("k" in keys))
|
||||
yaw = _WASD_YAW_DEG * (("l" in keys) - ("j" in keys))
|
||||
pitch = max(-_WASD_PITCH_LIMIT_DEG, min(_WASD_PITCH_LIMIT_DEG, pitch))
|
||||
|
||||
# R_delta = R_y(yaw) @ R_x(pitch): apply pitch first, then yaw.
|
||||
qx, qy, qz, qw = 0.0, 0.0, 0.0, 1.0
|
||||
if abs(pitch) > 0.0:
|
||||
px, py, pz, pw = _quat_from_axis_angle((1.0, 0.0, 0.0), pitch)
|
||||
qx, qy, qz, qw = px, py, pz, pw
|
||||
if abs(yaw) > 0.0:
|
||||
yx, yy, yz, yw = _quat_from_axis_angle((0.0, 1.0, 0.0), yaw)
|
||||
# q = yaw ⊗ pitch
|
||||
qx, qy, qz, qw = (
|
||||
yw * qx + yx * qw + yy * qz - yz * qy,
|
||||
yw * qy - yx * qz + yy * qw + yz * qx,
|
||||
yw * qz + yx * qy - yy * qx + yz * qw,
|
||||
yw * qw - yx * qx - yy * qy - yz * qz,
|
||||
)
|
||||
|
||||
return {
|
||||
"mode": mode,
|
||||
"data": {
|
||||
"translation": [dx, dy, dz],
|
||||
"rotation": [qx, qy, qz, qw],
|
||||
},
|
||||
}
|
||||
@@ -26,6 +26,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import av
|
||||
from camera_utils import wasd_to_camera_payload
|
||||
|
||||
try:
|
||||
from websockets.asyncio.client import connect # pyright: ignore[reportMissingImports]
|
||||
@@ -56,6 +57,15 @@ class ScheduledPromptUpdate:
|
||||
transition_chunks: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScheduledCameraUpdate:
|
||||
"""Client-side schedule entry for a midway camera ``session.interaction``."""
|
||||
|
||||
at: float
|
||||
camera: dict[str, Any]
|
||||
transition_chunks: int | None
|
||||
|
||||
|
||||
def _image_reference(value: str) -> dict[str, Any]:
|
||||
"""Accept an http(s) URL as-is, or inline a local image as a data URL."""
|
||||
if value.startswith(("http://", "https://", "data:")):
|
||||
@@ -119,6 +129,61 @@ def _parse_prompt_updates(value: str) -> list[ScheduledPromptUpdate]:
|
||||
return sorted(updates, key=lambda update: update.at)
|
||||
|
||||
|
||||
def _parse_camera_updates(value: str) -> list[ScheduledCameraUpdate]:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates must be valid JSON: {exc}") from exc
|
||||
if not isinstance(parsed, list):
|
||||
raise argparse.ArgumentTypeError("--camera-updates must be a JSON array")
|
||||
|
||||
updates: list[ScheduledCameraUpdate] = []
|
||||
for index, item in enumerate(parsed):
|
||||
if not isinstance(item, dict):
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}] must be a JSON object")
|
||||
try:
|
||||
at = float(item["at"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}] must have a valid 'at' field") from exc
|
||||
if at < 0:
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].at must be >= 0")
|
||||
|
||||
if "camera" in item:
|
||||
camera = item["camera"]
|
||||
if not isinstance(camera, dict):
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].camera must be an object")
|
||||
elif "actions" in item:
|
||||
actions = item["actions"]
|
||||
if not isinstance(actions, list):
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].actions must be a list of WASD/IJKL keys")
|
||||
mode = item.get("mode", "velocity")
|
||||
if mode not in ("target", "velocity"):
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].mode must be target or velocity")
|
||||
camera = wasd_to_camera_payload(actions, mode=mode)
|
||||
else:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"--camera-updates[{index}] must provide 'camera' (structural SE3) "
|
||||
"or 'actions' (client-side WASD helper)"
|
||||
)
|
||||
|
||||
transition_chunks: int | None
|
||||
if "transition_chunks" not in item:
|
||||
transition_chunks = None if camera.get("mode") == "velocity" else DEFAULT_TRANSITION_CHUNKS
|
||||
else:
|
||||
try:
|
||||
transition_chunks = int(item["transition_chunks"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"--camera-updates[{index}].transition_chunks must be an integer"
|
||||
) from exc
|
||||
if transition_chunks < 0:
|
||||
raise argparse.ArgumentTypeError(f"--camera-updates[{index}].transition_chunks must be >= 0")
|
||||
|
||||
updates.append(ScheduledCameraUpdate(at=at, camera=camera, transition_chunks=transition_chunks))
|
||||
|
||||
return sorted(updates, key=lambda update: update.at)
|
||||
|
||||
|
||||
def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
@@ -173,7 +238,7 @@ async def _run_prompt_update_scheduler(
|
||||
payload = {
|
||||
"type": "session.interaction",
|
||||
"interaction": {
|
||||
"event_id": f"scheduled-{update.at:.3f}",
|
||||
"event_id": f"scheduled-prompt-{update.at:.3f}",
|
||||
"event": {"prompt": update.prompt},
|
||||
"transition_chunks": update.transition_chunks,
|
||||
},
|
||||
@@ -183,6 +248,30 @@ async def _run_prompt_update_scheduler(
|
||||
print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
|
||||
|
||||
|
||||
async def _run_camera_update_scheduler(
|
||||
websocket: Any,
|
||||
updates: list[ScheduledCameraUpdate],
|
||||
*,
|
||||
video_started_at: float,
|
||||
send_lock: asyncio.Lock,
|
||||
) -> None:
|
||||
for update in updates:
|
||||
delay = (video_started_at + update.at) - time.perf_counter()
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
interaction: dict[str, Any] = {
|
||||
"event_id": f"scheduled-camera-{update.at:.3f}",
|
||||
"event": {"multi_modal_data": {"camera": update.camera}},
|
||||
}
|
||||
if update.transition_chunks is not None:
|
||||
interaction["transition_chunks"] = update.transition_chunks
|
||||
payload = {"type": "session.interaction", "interaction": interaction}
|
||||
async with send_lock:
|
||||
await websocket.send(json.dumps(payload, ensure_ascii=False))
|
||||
print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
|
||||
|
||||
|
||||
def _count_decoded_frames(video_bytes: bytes, *, stream_format: str) -> int | None:
|
||||
"""Return the number of decodable video frames in a streamed byte sequence, if known."""
|
||||
if not video_bytes:
|
||||
@@ -241,6 +330,7 @@ async def stream_video(args: argparse.Namespace) -> None:
|
||||
ws = None
|
||||
send_lock = asyncio.Lock()
|
||||
prompt_update_task: asyncio.Task[None] | None = None
|
||||
camera_update_task: asyncio.Task[None] | None = None
|
||||
pending_chunk_metadata: dict[str, Any] | None = None
|
||||
|
||||
try:
|
||||
@@ -295,8 +385,8 @@ async def stream_video(args: argparse.Namespace) -> None:
|
||||
if msg_type == "video.start":
|
||||
stream_format = msg.get("format") or stream_format
|
||||
print(f"Video session started: request_id={msg.get('request_id')} format={msg.get('format')}")
|
||||
video_started_at = time.perf_counter()
|
||||
if args.prompt_updates and prompt_update_task is None:
|
||||
video_started_at = time.perf_counter()
|
||||
prompt_update_task = asyncio.create_task(
|
||||
_run_prompt_update_scheduler(
|
||||
websocket,
|
||||
@@ -305,6 +395,15 @@ async def stream_video(args: argparse.Namespace) -> None:
|
||||
send_lock=send_lock,
|
||||
)
|
||||
)
|
||||
if args.camera_updates and camera_update_task is None:
|
||||
camera_update_task = asyncio.create_task(
|
||||
_run_camera_update_scheduler(
|
||||
websocket,
|
||||
args.camera_updates,
|
||||
video_started_at=video_started_at,
|
||||
send_lock=send_lock,
|
||||
)
|
||||
)
|
||||
elif msg_type == "video.chunk_metadata":
|
||||
# Save the metadata for the incoming video chunk
|
||||
pending_chunk_metadata = msg
|
||||
@@ -315,6 +414,8 @@ async def stream_video(args: argparse.Namespace) -> None:
|
||||
done = True
|
||||
if prompt_update_task is not None:
|
||||
prompt_update_task.cancel()
|
||||
if camera_update_task is not None:
|
||||
camera_update_task.cancel()
|
||||
break
|
||||
elif msg_type == "error":
|
||||
print(f"ERROR: {msg.get('message', msg)}")
|
||||
@@ -335,6 +436,10 @@ async def stream_video(args: argparse.Namespace) -> None:
|
||||
prompt_update_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await prompt_update_task
|
||||
if camera_update_task is not None:
|
||||
camera_update_task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await camera_update_task
|
||||
print(
|
||||
"Saving video... (May take a while. Remuxing concatenated chunks into one progressive MP4 file with total duration metadata in file header)"
|
||||
)
|
||||
@@ -405,9 +510,9 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help="JSON object merged into extra_params; overrides preset keys on conflict.",
|
||||
)
|
||||
# Midway prompt updates are pipeline-specific: a model that does not implement
|
||||
# them (LingBot-World, for one) rejects the whole session on the first one, so
|
||||
# nothing is scheduled unless the caller asks for it.
|
||||
# Midway interactions are pipeline-specific: unsupported modalities reject
|
||||
# the update (and may leave the session running). Nothing is scheduled unless
|
||||
# the caller asks for it.
|
||||
parser.add_argument(
|
||||
"--prompt-updates",
|
||||
type=_parse_prompt_updates,
|
||||
@@ -419,6 +524,19 @@ def parse_args() -> argparse.Namespace:
|
||||
f"'transition_chunks' defaults to {DEFAULT_TRANSITION_CHUNKS}."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--camera-updates",
|
||||
type=_parse_camera_updates,
|
||||
default=[],
|
||||
help=(
|
||||
"JSON array of scheduled camera updates for models that register the "
|
||||
"camera modality (e.g. LingBot-World). Each object needs 'at' and either "
|
||||
"'camera' (structural SE3: mode + translation/rotation) or "
|
||||
"'actions' (client-side WASD/IJKL helper converted before send). "
|
||||
"Optional 'transition_chunks' defaults to None for velocity and "
|
||||
f"{DEFAULT_TRANSITION_CHUNKS} for target."
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -14,8 +14,10 @@ For the full list of supported architectures across all modalities, see
|
||||
|
||||
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Breeze-TTS-2 | `BreezeBlue/Breeze-TTS-2` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | speaker tags (`S0`..`S9`, default `S0`) | — |
|
||||
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | uploaded audio voice only; no presets | ✓ |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
|
||||
| Gepard-1.0 | `nineninesix/gepard-1.0` | — (zero-shot default voice) | ✓ (PCM / WAV stream) | `default` only | — |
|
||||
| GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
|
||||
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — |
|
||||
| IndexTTS-2.5 | native `checkpoints/` bundle | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — |
|
||||
@@ -25,7 +27,6 @@ For the full list of supported architectures across all modalities, see
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | ✓ | — | — | — |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
|
||||
| dots.tts | `dots-studio/dots.tts-soar` | — (text-only) | ✓ (PCM stream) | — (default placeholder only) | — |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
|
||||
|
||||
CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](../../offline_inference/text_to_speech/README.md#cosyvoice3) instead.
|
||||
@@ -95,48 +96,52 @@ curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
}' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
|
||||
```
|
||||
|
||||
Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for IndexTTS-2, and 24 kHz for many others).
|
||||
Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 22.05 kHz for Gepard and IndexTTS-2, and 24 kHz for many others).
|
||||
|
||||
For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](../../../docs/serving/speech_api.md).
|
||||
|
||||
---
|
||||
|
||||
## dots.tts
|
||||
## Gepard-1.0
|
||||
|
||||
Single-stage, text-only TTS at 48 kHz. The current integration supports
|
||||
no-reference synthesis; voice cloning, reference audio, named voices, and
|
||||
precomputed speaker embeddings are not supported yet.
|
||||
Single-stage native AR TTS at 22.05 kHz mono. Zero-shot only: omit `voice` or pass `"default"`. Voice cloning from reference audio is not available yet.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Same NeMo NanoCodec install as the [offline Gepard section](../../offline_inference/text_to_speech/README.md#gepard-10). On a host whose CUDA toolkit cannot build kernels — no `nvcc`/`ninja`, or a consumer Blackwell (`sm_120`) card — also `export VLLM_USE_FLASHINFER_SAMPLER=0` before launch (same note as offline). That env is not a deploy-YAML field.
|
||||
|
||||
### Launch
|
||||
|
||||
```bash
|
||||
vllm serve dots-studio/dots.tts-soar --omni --trust-remote-code --port 8091
|
||||
vllm-omni serve nineninesix/gepard-1.0 --omni --port 8091 --trust-remote-code \
|
||||
--stage-init-timeout 900 \
|
||||
--deploy-config vllm_omni/deploy/gepard.yaml
|
||||
# or:
|
||||
./gepard/run_server.sh
|
||||
```
|
||||
|
||||
`--stage-init-timeout 900` matches the online e2e fixture and the offline example. Serve defaults to 300s, which is often too short for a cold download of the talker plus NanoCodec.
|
||||
|
||||
The packaged `vllm_omni/deploy/gepard.yaml` must be passed with `--deploy-config`. The checkpoint self-identifies as `qwen3_5_text`, so omitting the YAML launches a diffusion fallback instead of the Gepard pipeline. The YAML sets `async_chunk: false`, `max_num_seqs: 4`, and currently pins `seed: 42`, so serving is deterministic by default until that YAML seed is removed. Pass an explicit per-request `seed` in tests and clients rather than depending on either default.
|
||||
|
||||
### Sending requests
|
||||
|
||||
```bash
|
||||
# Non-streaming WAV
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, this is a dots TTS online-serving test.",
|
||||
"voice": "default",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
python examples/online_serving/text_to_speech/gepard/speech_client.py \
|
||||
--text "Hello, this is Gepard speaking."
|
||||
|
||||
# Streaming 48 kHz mono PCM
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, this is a streaming dots TTS test.",
|
||||
"voice": "default",
|
||||
"stream": true,
|
||||
"stream_format": "audio",
|
||||
"response_format": "pcm"
|
||||
}' --no-buffer | play -t raw -r 48000 -e signed -b 16 -c 1 -
|
||||
python examples/online_serving/text_to_speech/gepard/speech_client.py \
|
||||
--text "Hello, this is Gepard speaking." --seed 7 --stream --output output.pcm
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 22.05 kHz mono. `max_new_tokens` is a **frame** budget (1 token = 1 frame = 1024 samples ≈ 46.4 ms at 21.5 fps; adapter bounds 1..4096).
|
||||
- Supported request fields: `input` (required), `voice` (`default` only), `response_format` (`wav` default; `wav/pcm/flac/mp3` non-streaming; `opus` 400 because 22.05 kHz is not an Opus sample rate; streaming `pcm/wav` only), `stream` / `stream_format`, `max_new_tokens`, `seed`.
|
||||
- Unsupported: `speed`, `extra_params` (including `temperature`/`top_p`/`top_k`), `ref_audio`, `ref_text`, `speaker_embedding`, `task_type`, `instructions`, `language`, and `word_timestamps`.
|
||||
- Concurrent requests at `max_num_seqs: 4` are supported. Native-AR recompute preemption is a known limitation of this architecture (a request that is preempted mid-generation can resume incorrectly); keep concurrency at or below `max_num_seqs` and treat preemption as out of scope until the platform fix lands.
|
||||
- Optional comparison against the upstream Gepard reference server needs Blackwell/Hopper + CUDA 13 + Postgres and is not part of CI.
|
||||
|
||||
---
|
||||
|
||||
## GLM-TTS
|
||||
@@ -739,6 +744,20 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
|
||||
./qwen3_tts/run_server.sh Base
|
||||
```
|
||||
|
||||
For a local deployment with multiple API frontend processes sharing one set
|
||||
of TTS stage engines, add `--api-server-count`:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||
--omni --api-server-count 2 --port 8091
|
||||
```
|
||||
|
||||
This mode supports local EngineCore stages only; headless or remote stage
|
||||
deployments are not supported. Runtime voice upload and deletion are disabled
|
||||
with multiple API frontends because their registries are process-local.
|
||||
Built-in voices, inline `ref_audio`, and voices restored from `custom_voice_dir`
|
||||
at startup are supported.
|
||||
|
||||
### Executor backend
|
||||
|
||||
Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
|
||||
@@ -793,6 +812,10 @@ checkpoint. When a request names an uploaded voice, the server infers
|
||||
`task_type="Base"`. Built-in presets such as `vivian` and `ryan` remain
|
||||
CustomVoice speakers and require a CustomVoice checkpoint.
|
||||
|
||||
The runtime upload and delete routes require a single API frontend; with
|
||||
`--api-server-count > 1`, use inline `ref_audio` or restore precomputed voices
|
||||
from `custom_voice_dir` at startup.
|
||||
|
||||
### Precomputed custom voices
|
||||
|
||||
For reused Base voice-cloning speakers, precompute the reference artifacts once
|
||||
@@ -1025,3 +1048,63 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
|
||||
- Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
|
||||
- Voice cloning is gated upstream and may require a recent `mistral_common`.
|
||||
- A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.
|
||||
|
||||
## Breeze-TTS-2
|
||||
|
||||
Two-stage AR TTS (T5Gemma2 + Qwen3 talker with a depth decoder → bundled Qwen3-TTS codec) at 24 kHz mono, from [BreezeBlue](https://huggingface.co/BreezeBlue/Breeze-TTS-2). Four modes are selected automatically from the request fields: plain (`input`), voice design (`instructions`), clone (`ref_audio`+`ref_text`), and voice direction (reference trio + `instructions`).
|
||||
|
||||
### Launch
|
||||
|
||||
```bash
|
||||
vllm serve BreezeBlue/Breeze-TTS-2 --omni --port 8091
|
||||
# or:
|
||||
./breeze_tts_2/run_server.sh
|
||||
```
|
||||
|
||||
The deploy config at `vllm_omni/deploy/breeze_tts_2.yaml` auto-loads (async-chunk streaming, 8-frame inter-stage chunks).
|
||||
|
||||
### Sending requests
|
||||
|
||||
```bash
|
||||
# Plain synthesis (speaker tags S0..S9, default S0)
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, this is Breeze TTS 2 on vLLM-Omni.",
|
||||
"voice": "S0",
|
||||
"response_format": "wav",
|
||||
"sample_rate": 24000
|
||||
}' --output breeze_plain.wav
|
||||
|
||||
# Voice direction (clone a reference, then steer the delivery)
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "We need to discuss what happened last night.",
|
||||
"ref_audio": "file:///path/to/reference.wav",
|
||||
"ref_text": "The exact transcript of the reference audio.",
|
||||
"instructions": "Speak slowly with a restrained, serious tone.",
|
||||
"response_format": "wav"
|
||||
}' --output breeze_direction.wav
|
||||
```
|
||||
|
||||
### Streaming PCM
|
||||
|
||||
```bash
|
||||
curl -N -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Streaming output from Breeze TTS 2.",
|
||||
"stream": true,
|
||||
"stream_format": "audio",
|
||||
"response_format": "pcm",
|
||||
"sample_rate": 24000
|
||||
}' --output breeze_stream.pcm
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Only `sample_rate=24000` is accepted; `speed` must be `1.0`.
|
||||
- `guidance_scale`/`cfg_scale` must be `1.0`; `negative_prompt` is rejected (CFG companion support is a follow-up).
|
||||
- `instructions` without `ref_audio` is voice design; with `ref_audio`+`ref_text` it is voice direction.
|
||||
- See [`recipes/BreezeBlue/Breeze-TTS-2.md`](../../../recipes/BreezeBlue/Breeze-TTS-2.md) for the full recipe and the offline example under [`examples/offline_inference/text_to_speech/breeze_tts_2/`](../../offline_inference/text_to_speech/breeze_tts_2/).
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# Launch Breeze-TTS-2 online serving with the async-chunk deploy config.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${1:-BreezeBlue/Breeze-TTS-2}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
exec vllm-omni serve "${MODEL}" \
|
||||
--deploy-config vllm_omni/deploy/breeze_tts_2.yaml \
|
||||
--omni --port "${PORT}"
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Launch vLLM-Omni server for Gepard-1.0 TTS (zero-shot default voice).
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 ./run_server.sh
|
||||
#
|
||||
# Packaged deploy config: vllm_omni/deploy/gepard.yaml (async_chunk=false,
|
||||
# 22.05 kHz mono, default seed 42 until the YAML seed is removed).
|
||||
# Cold start downloads the talker and NeMo NanoCodec; the serve default of
|
||||
# --stage-init-timeout 300 is too tight (offline end2end.py uses 900).
|
||||
#
|
||||
# On a host whose CUDA toolkit cannot JIT FlashInfer (no nvcc/ninja, or a
|
||||
# consumer Blackwell sm_120 card), also:
|
||||
# export VLLM_USE_FLASHINFER_SAMPLER=0
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-nineninesix/gepard-1.0}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
echo "Starting Gepard-1.0 server with model: $MODEL"
|
||||
|
||||
vllm-omni serve "$MODEL" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
--omni \
|
||||
--stage-init-timeout 900 \
|
||||
--deploy-config vllm_omni/deploy/gepard.yaml
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Client for Gepard-1.0 TTS via /v1/audio/speech.
|
||||
|
||||
Gepard is zero-shot: omit ``voice`` or pass ``"default"``. Output is 22.05 kHz
|
||||
mono. ``seed`` is optional; the packaged deploy YAML currently pins ``seed: 42``
|
||||
until that default is removed, so requests without an explicit seed are still
|
||||
deterministic.
|
||||
|
||||
Examples:
|
||||
python speech_client.py --text "Hello, this is Gepard speaking."
|
||||
python speech_client.py --text "Hello, this is Gepard speaking." --seed 7
|
||||
python speech_client.py --text "Hello, this is Gepard speaking." --stream --output output.pcm
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
DEFAULT_MODEL = "nineninesix/gepard-1.0"
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"voice": args.voice,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
if args.seed is not None:
|
||||
payload["seed"] = args.seed
|
||||
if args.max_new_tokens is not None:
|
||||
payload["max_new_tokens"] = args.max_new_tokens
|
||||
if args.stream:
|
||||
payload["stream"] = True
|
||||
payload["stream_format"] = "audio"
|
||||
if args.response_format not in ("pcm", "wav"):
|
||||
print(f"Note: streaming requires pcm/wav; overriding --response-format {args.response_format} to 'pcm'")
|
||||
payload["response_format"] = "pcm"
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
print(f"Voice: {args.voice}")
|
||||
if args.seed is not None:
|
||||
print(f"Seed: {args.seed}")
|
||||
print("Generating audio...")
|
||||
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or ("output.wav" if payload["response_format"] == "wav" else "output.pcm")
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream("POST", api_url, json=payload, headers=headers) as resp:
|
||||
if resp.status_code != 200:
|
||||
print(f"Error: {resp.status_code}")
|
||||
print(resp.read().decode())
|
||||
return
|
||||
total_bytes = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in resp.iter_bytes():
|
||||
f.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
print(f"Streamed {total_bytes} bytes to: {output_path}")
|
||||
return
|
||||
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
output_path = args.output or "output.wav"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Gepard-1.0 TTS client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--voice", default="default", help="Voice name (only 'default' is supported)")
|
||||
parser.add_argument("--seed", type=int, default=None, help="Sampling seed")
|
||||
parser.add_argument(
|
||||
"--max-new-tokens",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Frame budget (1 token = 1 frame ≈ 46.4 ms at 21.5 fps)",
|
||||
)
|
||||
parser.add_argument("--stream", action="store_true", help="Enable streaming (PCM output)")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm"],
|
||||
help="Audio format (default: wav). Streaming is pcm/wav only. Opus is not supported at 22.05 kHz.",
|
||||
)
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Batch speech client for Qwen3-TTS via /v1/audio/speech/batch endpoint.
|
||||
|
||||
This script demonstrates how to synthesize multiple texts in a single request.
|
||||
@@ -7,7 +10,7 @@ the reference for each item.
|
||||
|
||||
Start the server (with batch-optimized stage settings for best throughput):
|
||||
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--omni \
|
||||
--trust-remote-code \
|
||||
--stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Gradio demo for Qwen3-TTS with gapless streaming audio playback.
|
||||
|
||||
Uses a custom AudioWorklet-based player for gap-free streaming,
|
||||
@@ -351,7 +354,7 @@ def generate_speech(
|
||||
raise gr.Error(f"Failed to decode audio: {e}")
|
||||
|
||||
|
||||
def create_app(api_base: str):
|
||||
def create_app(api_base: str, task_type: str = "CustomVoice"):
|
||||
"""Create the FastAPI app with streaming proxy + Gradio UI."""
|
||||
fastapi_app = FastAPI()
|
||||
|
||||
@@ -463,10 +466,11 @@ def create_app(api_base: str):
|
||||
lines=4,
|
||||
)
|
||||
with gr.Row():
|
||||
task_type = gr.Radio(
|
||||
task_type_input = gr.Radio(
|
||||
choices=TASK_TYPES,
|
||||
value="CustomVoice",
|
||||
value=task_type,
|
||||
label="Task Type",
|
||||
interactive=False,
|
||||
scale=2,
|
||||
)
|
||||
language = gr.Dropdown(
|
||||
@@ -479,17 +483,21 @@ def create_app(api_base: str):
|
||||
choices=voices,
|
||||
value=voices[0] if voices else None,
|
||||
label="speaker",
|
||||
visible=True,
|
||||
visible=task_type == "CustomVoice",
|
||||
allow_custom_value=True,
|
||||
)
|
||||
instructions = gr.Textbox(
|
||||
label="Instructions",
|
||||
placeholder="e.g., Speak with excitement / A warm, friendly female voice",
|
||||
lines=2,
|
||||
visible=True,
|
||||
info="Optional style/emotion instructions",
|
||||
visible=task_type != "Base",
|
||||
info=(
|
||||
"Required: describe the voice style"
|
||||
if task_type == "VoiceDesign"
|
||||
else "Optional style/emotion instructions"
|
||||
),
|
||||
)
|
||||
with gr.Column(visible=False) as ref_group:
|
||||
with gr.Column(visible=task_type == "Base") as ref_group:
|
||||
ref_audio = gr.Audio(
|
||||
label="Reference Audio (upload for voice cloning)",
|
||||
type="numpy",
|
||||
@@ -563,7 +571,7 @@ def create_app(api_base: str):
|
||||
autoplay=True,
|
||||
visible=False,
|
||||
)
|
||||
with gr.Column(visible=True) as examples_cv:
|
||||
with gr.Column(visible=task_type == "CustomVoice") as examples_cv:
|
||||
gr.Examples(
|
||||
examples=[
|
||||
[
|
||||
@@ -588,7 +596,7 @@ def create_app(api_base: str):
|
||||
inputs=[text_input, voice, language, instructions],
|
||||
label="examples",
|
||||
)
|
||||
with gr.Column(visible=False) as examples_vd:
|
||||
with gr.Column(visible=task_type == "VoiceDesign") as examples_vd:
|
||||
gr.Examples(
|
||||
examples=[
|
||||
[
|
||||
@@ -605,7 +613,7 @@ def create_app(api_base: str):
|
||||
inputs=[text_input, language, instructions],
|
||||
label="examples",
|
||||
)
|
||||
with gr.Column(visible=False) as examples_base:
|
||||
with gr.Column(visible=task_type == "Base") as examples_base:
|
||||
gr.Examples(
|
||||
examples=[
|
||||
[
|
||||
@@ -653,9 +661,9 @@ def create_app(api_base: str):
|
||||
gr.update(visible=is_base), # examples_base
|
||||
)
|
||||
|
||||
task_type.change(
|
||||
task_type_input.change(
|
||||
fn=on_task_change,
|
||||
inputs=[task_type],
|
||||
inputs=[task_type_input],
|
||||
outputs=[
|
||||
voice,
|
||||
instructions,
|
||||
@@ -703,7 +711,7 @@ def create_app(api_base: str):
|
||||
|
||||
all_inputs = [
|
||||
text_input,
|
||||
task_type,
|
||||
task_type_input,
|
||||
voice,
|
||||
language,
|
||||
instructions,
|
||||
@@ -788,6 +796,9 @@ def parse_args():
|
||||
description="Gradio demo for Qwen3-TTS with gapless AudioWorklet streaming.",
|
||||
)
|
||||
add_common_args(parser)
|
||||
parser.add_argument(
|
||||
"--task-type", choices=TASK_TYPES, default="CustomVoice", help="Task matching the served checkpoint"
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -796,7 +807,7 @@ def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
print(f"Connecting to vLLM server at: {args.api_base}")
|
||||
|
||||
app = create_app(args.api_base)
|
||||
app = create_app(args.api_base, task_type=args.task_type)
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
@@ -8,25 +8,28 @@ to generate audio from text using Qwen3-TTS models.
|
||||
|
||||
Examples:
|
||||
# CustomVoice task (predefined speaker)
|
||||
python openai_speech_client.py --text "Hello, how are you?" --voice vivian
|
||||
python openai_speech_client.py --text "Hello, how are you?" --speaker vivian
|
||||
|
||||
# CustomVoice with emotion instruction
|
||||
python openai_speech_client.py --text "I'm so happy!" --voice vivian \
|
||||
python openai_speech_client.py --text "I'm so happy!" --speaker vivian \
|
||||
--instructions "Speak with excitement"
|
||||
|
||||
# VoiceDesign task (voice from description)
|
||||
python openai_speech_client.py --text "Hello world" \
|
||||
--model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
|
||||
--task-type VoiceDesign \
|
||||
--instructions "A warm, friendly female voice"
|
||||
|
||||
# Base task (voice cloning)
|
||||
python openai_speech_client.py --text "Hello world" \
|
||||
--model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--task-type Base \
|
||||
--ref-audio "https://example.com/reference.wav" \
|
||||
--ref-text "This is the reference transcript"
|
||||
|
||||
# Base task with pre-computed speaker embedding
|
||||
python openai_speech_client.py --text "Hello world" \
|
||||
--model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--task-type Base \
|
||||
--speaker-embedding embedding.json
|
||||
"""
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Launch both vLLM server and Gradio demo for Qwen3-TTS
|
||||
#
|
||||
# Usage:
|
||||
@@ -179,7 +182,7 @@ fi
|
||||
echo ""
|
||||
echo "Starting Gradio demo..."
|
||||
cd "$SCRIPT_DIR"
|
||||
GRADIO_CMD=("python" "gradio_demo.py" "--api-base" "$API_BASE" "--host" "$GRADIO_IP" "--port" "$GRADIO_PORT")
|
||||
GRADIO_CMD=("python" "gradio_demo.py" "--api-base" "$API_BASE" "--host" "$GRADIO_IP" "--port" "$GRADIO_PORT" "--task-type" "$TASK_TYPE")
|
||||
if [ "$GRADIO_SHARE" = true ]; then
|
||||
GRADIO_CMD+=("--share")
|
||||
fi
|
||||
|
||||
Reference in new issue
Block a user