[Doc][TTS] CosyVoice3 online docs + residual TTS yaml cleanup + remove VoxCPM v1 (#3748)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed
1 parent
1c530d984d
commit
fc8486cf20
35 files changed
+248
-4147
No files matched your search
@@ -233,7 +233,6 @@ sentinel for regressions in this area.
|
||||
benchmarks/tts/
|
||||
├── README.md (this file)
|
||||
├── bench_tts.py CLI — serve-mode benchmark driver
|
||||
├── bench_voxcpm_offline.py CLI — offline VoxCPM benchmark (sync + streaming)
|
||||
├── plot_results.py Generate per-task / per-concurrency curves
|
||||
└── model_configs.yaml Model registry (supported tasks + extra body)
|
||||
```
|
||||
|
||||
@@ -1,922 +0,0 @@
|
||||
"""Offline VoxCPM benchmark for vLLM Omni.
|
||||
|
||||
Supports both:
|
||||
- sync one-shot (Omni.generate)
|
||||
- streaming (AsyncOmni.generate with async_chunk config)
|
||||
- text-only synthesis
|
||||
- voice cloning
|
||||
- text/clone batch inputs from txt or jsonl
|
||||
|
||||
Usage::
|
||||
|
||||
# Sync (default voice)
|
||||
python benchmarks/tts/bench_voxcpm_offline.py \\
|
||||
--model /path/to/VoxCPM \\
|
||||
--text "Hello world" \\
|
||||
--output-dir results/audio/
|
||||
|
||||
# Streaming (async_chunk)
|
||||
python benchmarks/tts/bench_voxcpm_offline.py \\
|
||||
--model /path/to/VoxCPM \\
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \\
|
||||
--txt-prompts prompts.txt \\
|
||||
--output-dir results/audio/
|
||||
|
||||
# Voice cloning batch via JSONL
|
||||
python benchmarks/tts/bench_voxcpm_offline.py \\
|
||||
--model /path/to/VoxCPM \\
|
||||
--jsonl-prompts prompts.jsonl \\
|
||||
--output-dir results/audio/
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import AsyncOmni, Omni
|
||||
|
||||
|
||||
def _find_repo_root(start: Path) -> Path:
|
||||
"""Walk up from ``start`` until a repo marker is found.
|
||||
|
||||
Falls back to ``parents[2]`` for backwards compatibility if no marker hits
|
||||
(which can only happen in unusual checkouts — the tree should always have
|
||||
pyproject.toml + vllm_omni/ at the top level).
|
||||
"""
|
||||
for candidate in [start, *start.parents]:
|
||||
if (candidate / "pyproject.toml").is_file() and (candidate / "vllm_omni").is_dir():
|
||||
return candidate
|
||||
return start.parents[2]
|
||||
|
||||
|
||||
REPO_ROOT = _find_repo_root(Path(__file__).resolve())
|
||||
DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
|
||||
DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "deploy" / "voxcpm.yaml"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PromptSpec:
|
||||
text: str
|
||||
label: str
|
||||
ref_audio: str | None = None
|
||||
ref_text: str | None = None
|
||||
|
||||
|
||||
def _require_soundfile():
|
||||
try:
|
||||
import soundfile as sf # type: ignore
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError(
|
||||
"soundfile is required to write VoxCPM benchmark WAV outputs. Install it with: pip install soundfile"
|
||||
) from exc
|
||||
return sf
|
||||
|
||||
|
||||
def _build_prompt(
|
||||
args,
|
||||
*,
|
||||
text: str,
|
||||
ref_audio: str | None = None,
|
||||
ref_text: str | None = None,
|
||||
global_request_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
additional_information: dict[str, list[Any]] = {
|
||||
"text": [text],
|
||||
"cfg_value": [args.cfg_value],
|
||||
"inference_timesteps": [args.inference_timesteps],
|
||||
"min_len": [args.min_len],
|
||||
"max_new_tokens": [args.max_new_tokens],
|
||||
}
|
||||
if args.streaming_prefix_len is not None:
|
||||
additional_information["streaming_prefix_len"] = [args.streaming_prefix_len]
|
||||
|
||||
if ref_audio:
|
||||
additional_information["ref_audio"] = [ref_audio]
|
||||
if ref_text:
|
||||
additional_information["ref_text"] = [ref_text]
|
||||
if global_request_id is not None:
|
||||
additional_information["global_request_id"] = [global_request_id]
|
||||
|
||||
return {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": additional_information,
|
||||
}
|
||||
|
||||
|
||||
def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor:
|
||||
audio = mm.get("audio", mm.get("model_outputs"))
|
||||
if audio is None:
|
||||
raise ValueError("No audio output found in multimodal output.")
|
||||
if isinstance(audio, list):
|
||||
parts = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio]
|
||||
audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0)
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio)
|
||||
return audio.float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def _extract_sample_rate(mm: dict[str, Any]) -> int:
|
||||
sr_raw = mm.get("sr", 24000)
|
||||
if isinstance(sr_raw, list) and sr_raw:
|
||||
sr_raw = sr_raw[-1]
|
||||
if hasattr(sr_raw, "item"):
|
||||
return int(sr_raw.item())
|
||||
return int(sr_raw)
|
||||
|
||||
|
||||
def _emit_offline_metrics(
|
||||
*,
|
||||
request_id: str,
|
||||
elapsed_s: float,
|
||||
first_audio_elapsed: float | None,
|
||||
audio_duration_s: float,
|
||||
) -> None:
|
||||
metrics = {
|
||||
"request_id": request_id,
|
||||
"ttfp_ms": round(first_audio_elapsed * 1000.0, 3) if first_audio_elapsed is not None else None,
|
||||
"audio_duration_s": round(audio_duration_s, 6),
|
||||
"rtf": round(elapsed_s / audio_duration_s, 6) if audio_duration_s > 0 else None,
|
||||
}
|
||||
print(f"[OfflineMetrics] {metrics}")
|
||||
|
||||
|
||||
def _write_audio_tensor(output_path: Path, audio_tensor: Any, sample_rate: int) -> None:
|
||||
sf = _require_soundfile()
|
||||
if isinstance(audio_tensor, torch.Tensor):
|
||||
audio_np = audio_tensor.float().cpu().clamp(-1.0, 1.0).numpy()
|
||||
else:
|
||||
audio_np = torch.as_tensor(audio_tensor).float().cpu().clamp(-1.0, 1.0).numpy()
|
||||
sf.write(
|
||||
output_path,
|
||||
audio_np,
|
||||
sample_rate,
|
||||
format="WAV",
|
||||
subtype="PCM_16",
|
||||
)
|
||||
|
||||
|
||||
def _save_wav(mm: dict[str, Any], output_dir: Path, request_id: str) -> Path:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_dir / f"output_{request_id}.wav"
|
||||
_write_audio_tensor(output_path, _extract_audio_tensor(mm), _extract_sample_rate(mm))
|
||||
return output_path
|
||||
|
||||
|
||||
def _iter_request_multimodal_outputs(request_output: Any):
|
||||
outputs = getattr(request_output, "outputs", None)
|
||||
if outputs:
|
||||
for output in outputs:
|
||||
mm = getattr(output, "multimodal_output", None)
|
||||
if isinstance(mm, dict):
|
||||
yield mm
|
||||
|
||||
mm = getattr(request_output, "multimodal_output", None)
|
||||
if isinstance(mm, dict):
|
||||
yield mm
|
||||
|
||||
|
||||
def _read_non_empty_lines(path: str) -> list[str]:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return [line.strip() for line in f if line.strip()]
|
||||
|
||||
|
||||
def _load_prompt_specs(args) -> list[PromptSpec]:
|
||||
specs: list[PromptSpec] = []
|
||||
|
||||
if args.txt_prompts is not None:
|
||||
texts = _read_non_empty_lines(args.txt_prompts)
|
||||
if not texts:
|
||||
raise ValueError(f"No prompts found in {args.txt_prompts}")
|
||||
for idx, text in enumerate(texts, start=1):
|
||||
specs.append(
|
||||
PromptSpec(
|
||||
text=text,
|
||||
label=f"item{idx:03d}",
|
||||
ref_audio=args.ref_audio,
|
||||
ref_text=args.ref_text,
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
if args.jsonl_prompts is not None:
|
||||
with open(args.jsonl_prompts, encoding="utf-8") as f:
|
||||
for line_no, raw_line in enumerate(f, start=1):
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"{args.jsonl_prompts}:{line_no} is not valid JSON: {exc}") from exc
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError(f"{args.jsonl_prompts}:{line_no} must be a JSON object")
|
||||
|
||||
text = item.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError(f"{args.jsonl_prompts}:{line_no} requires non-empty string field 'text'")
|
||||
|
||||
ref_audio = item.get("ref_audio", args.ref_audio)
|
||||
ref_text = item.get("ref_text", args.ref_text)
|
||||
if (ref_audio is None) != (ref_text is None):
|
||||
raise ValueError(
|
||||
f"{args.jsonl_prompts}:{line_no} must provide both 'ref_audio' and 'ref_text' together"
|
||||
)
|
||||
|
||||
specs.append(
|
||||
PromptSpec(
|
||||
text=text.strip(),
|
||||
label=f"item{len(specs) + 1:03d}",
|
||||
ref_audio=ref_audio,
|
||||
ref_text=ref_text,
|
||||
)
|
||||
)
|
||||
|
||||
if not specs:
|
||||
raise ValueError(f"No prompts found in {args.jsonl_prompts}")
|
||||
return specs
|
||||
|
||||
specs.append(
|
||||
PromptSpec(
|
||||
text=args.text,
|
||||
label="item001",
|
||||
ref_audio=args.ref_audio,
|
||||
ref_text=args.ref_text,
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
def _build_prompt_for_spec(args, spec: PromptSpec, *, global_request_id: str | None = None) -> dict[str, Any]:
|
||||
return _build_prompt(
|
||||
args,
|
||||
text=spec.text,
|
||||
ref_audio=spec.ref_audio,
|
||||
ref_text=spec.ref_text,
|
||||
global_request_id=global_request_id,
|
||||
)
|
||||
|
||||
|
||||
def _count_voice_clone_prompts(prompt_specs: list[PromptSpec]) -> int:
|
||||
return sum(1 for spec in prompt_specs if spec.ref_audio is not None)
|
||||
|
||||
|
||||
def _get_warmup_specs(prompt_specs: list[PromptSpec]) -> list[PromptSpec]:
|
||||
return prompt_specs[:1]
|
||||
|
||||
|
||||
def _extract_stream_finished(stage_output: Any) -> bool:
|
||||
request_output = getattr(stage_output, "request_output", None)
|
||||
request_finished = getattr(request_output, "finished", None)
|
||||
if request_finished is not None:
|
||||
return bool(request_finished)
|
||||
return bool(getattr(stage_output, "finished", False))
|
||||
|
||||
|
||||
def _build_profiled_stage_config(
|
||||
stage_configs_path: str,
|
||||
profiler_dir: str,
|
||||
) -> str:
|
||||
stage_config_path = Path(stage_configs_path)
|
||||
yaml_text = stage_config_path.read_text(encoding="utf-8")
|
||||
injected_lines: list[str] = []
|
||||
injected_count = 0
|
||||
|
||||
for line in yaml_text.splitlines():
|
||||
injected_lines.append(line)
|
||||
if line.strip() != "engine_args:":
|
||||
continue
|
||||
indent = line[: len(line) - len(line.lstrip())]
|
||||
child_indent = indent + " "
|
||||
grandchild_indent = child_indent + " "
|
||||
injected_lines.extend(
|
||||
[
|
||||
f"{child_indent}profiler_config:",
|
||||
f'{grandchild_indent}profiler: "torch"',
|
||||
f'{grandchild_indent}torch_profiler_dir: "{profiler_dir}"',
|
||||
f"{grandchild_indent}torch_profiler_with_stack: true",
|
||||
]
|
||||
)
|
||||
injected_count += 1
|
||||
|
||||
if injected_count == 0:
|
||||
raise ValueError(f"No engine_args block found in stage config: {stage_configs_path}")
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
encoding="utf-8",
|
||||
delete=False,
|
||||
suffix=".yaml",
|
||||
prefix=f"{stage_config_path.stem}_profile_",
|
||||
)
|
||||
tmp.write("\n".join(injected_lines) + "\n")
|
||||
tmp.close()
|
||||
return tmp.name
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Offline split-stage VoxCPM inference with vLLM Omni (auto sync/streaming by stage config)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default=os.environ.get("VOXCPM_MODEL"),
|
||||
help="Local VoxCPM model directory. Defaults to $VOXCPM_MODEL.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="This is a split-stage VoxCPM synthesis example running on vLLM Omni.",
|
||||
help="Text to synthesize. Ignored when --txt-prompts or --jsonl-prompts is used.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--txt-prompts",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to a .txt file with one synthesis text per line.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--jsonl-prompts",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to a .jsonl file. Each line must contain at least {'text': ...}; "
|
||||
"clone rows can also set ref_audio/ref_text, and ref_text must be the "
|
||||
"real transcript of ref_audio."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Optional reference audio path for voice cloning. With --txt-prompts, "
|
||||
"the same reference is applied to every line."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Real transcript of the reference audio. Placeholder text or mismatched "
|
||||
"text will usually produce noisy/electronic clone audio."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=str(DEFAULT_STAGE_SYNC),
|
||||
help="Stage config YAML path. Routing is selected only from this path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cfg-value",
|
||||
type=float,
|
||||
default=2.0,
|
||||
help="Classifier-free guidance value for VoxCPM.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--inference-timesteps",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Number of inference timesteps.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-len",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Minimum generated token length.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-new-tokens",
|
||||
type=int,
|
||||
default=4096,
|
||||
help="Maximum generated token length.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--streaming-prefix-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="VoxCPM streaming window (optional, streaming mode only).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Directory for output WAV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-init-timeout",
|
||||
type=int,
|
||||
default=600,
|
||||
help="Stage initialization timeout in seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
dest="log_stats",
|
||||
action="store_true",
|
||||
help="Enable vLLM Omni stats logging.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-log-stats",
|
||||
dest="log_stats",
|
||||
action="store_false",
|
||||
help="Disable vLLM Omni stats logging.",
|
||||
)
|
||||
parser.set_defaults(log_stats=True)
|
||||
parser.add_argument(
|
||||
"--num-runs",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of full inference runs (same prompt each time). Default 1.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup-runs",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Optional number of warmup passes before measured runs. Warmup uses only "
|
||||
"the first prompt and does not save outputs."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-profiler",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Enable torch profiler for the configured stages. A temporary profiled "
|
||||
"stage config is generated automatically."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profiler-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Directory for profiler traces. Defaults to <output-dir>/profiler when profiling is enabled.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profiler-stages",
|
||||
type=int,
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="Optional stage ids to profile. Defaults to all stages that have profiler_config.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profiler-wait-seconds",
|
||||
type=float,
|
||||
default=30.0,
|
||||
help="Seconds to wait after stop_profile for trace files to flush.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.model:
|
||||
parser.error("--model is required unless $VOXCPM_MODEL is set")
|
||||
if args.txt_prompts is not None and args.jsonl_prompts is not None:
|
||||
parser.error("--txt-prompts and --jsonl-prompts are mutually exclusive")
|
||||
if (args.ref_audio is None) != (args.ref_text is None):
|
||||
parser.error("--ref-audio and --ref-text must be provided together")
|
||||
if args.num_runs < 1:
|
||||
parser.error("--num-runs must be >= 1")
|
||||
if args.warmup_runs < 0:
|
||||
parser.error("--warmup-runs must be >= 0")
|
||||
if args.output_dir is None:
|
||||
args.output_dir = (
|
||||
"output_audio_streaming" if _is_streaming_stage_config(args.stage_configs_path) else "output_audio"
|
||||
)
|
||||
if args.enable_profiler and args.profiler_dir is None:
|
||||
args.profiler_dir = str(Path(args.output_dir) / "profiler")
|
||||
try:
|
||||
args.prompt_specs = _load_prompt_specs(args)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def _is_streaming_stage_config(stage_configs_path: str) -> bool:
|
||||
cfg_name = Path(stage_configs_path).name.lower()
|
||||
return "async_chunk" in cfg_name
|
||||
|
||||
|
||||
async def _collect_streaming_audio(
|
||||
omni: AsyncOmni,
|
||||
args: Any,
|
||||
spec: PromptSpec,
|
||||
request_id: str,
|
||||
*,
|
||||
phase_label: str,
|
||||
prompt_index: int,
|
||||
prompt_count: int,
|
||||
print_prompt: bool = False,
|
||||
) -> tuple[torch.Tensor, int, float, float | None]:
|
||||
prompt = _build_prompt_for_spec(args, spec, global_request_id=request_id)
|
||||
delta_chunks: list[torch.Tensor] = []
|
||||
sample_rate = 24000
|
||||
chunk_i = 0
|
||||
prev_total_samples = 0
|
||||
t_start = time.perf_counter()
|
||||
first_audio_elapsed: float | None = None
|
||||
|
||||
if print_prompt:
|
||||
print(f"---prompt---:{prompt}")
|
||||
|
||||
async for stage_output in omni.generate(prompt, request_id=request_id):
|
||||
mm = getattr(stage_output, "multimodal_output", None)
|
||||
if not isinstance(mm, dict):
|
||||
ro = getattr(stage_output, "request_output", None)
|
||||
if ro is None:
|
||||
continue
|
||||
mm = getattr(ro, "multimodal_output", None)
|
||||
if not isinstance(mm, dict) and getattr(ro, "outputs", None):
|
||||
seq = ro.outputs[0]
|
||||
mm = getattr(seq, "multimodal_output", None)
|
||||
if not isinstance(mm, dict):
|
||||
continue
|
||||
sample_rate = _extract_sample_rate(mm)
|
||||
try:
|
||||
w = _extract_audio_tensor(mm)
|
||||
n = int(w.numel())
|
||||
if n == 0:
|
||||
continue
|
||||
finished = _extract_stream_finished(stage_output)
|
||||
if n > prev_total_samples:
|
||||
delta = w.reshape(-1)[prev_total_samples:]
|
||||
prev_total_samples = n
|
||||
elif finished and n == prev_total_samples:
|
||||
delta = w.reshape(-1)[:0]
|
||||
else:
|
||||
delta = w.reshape(-1)
|
||||
prev_total_samples += int(delta.numel())
|
||||
if int(delta.numel()) > 0:
|
||||
delta_chunks.append(delta)
|
||||
if first_audio_elapsed is None and int(delta.numel()) > 0:
|
||||
first_audio_elapsed = time.perf_counter() - t_start
|
||||
logger.info(
|
||||
"%s prompt=%d/%d chunk=%d delta_samples=%d buf_len=%d finished=%s",
|
||||
phase_label,
|
||||
prompt_index + 1,
|
||||
prompt_count,
|
||||
chunk_i,
|
||||
int(delta.numel()),
|
||||
n,
|
||||
finished,
|
||||
)
|
||||
chunk_i += 1
|
||||
except ValueError:
|
||||
if not _extract_stream_finished(stage_output):
|
||||
logger.debug("skip non-audio partial output chunk=%d", chunk_i)
|
||||
|
||||
if not delta_chunks:
|
||||
raise RuntimeError("No audio chunks received; check stage config and logs.")
|
||||
|
||||
audio_cat = torch.cat([c.reshape(-1) for c in delta_chunks], dim=0)
|
||||
elapsed = time.perf_counter() - t_start
|
||||
return audio_cat, sample_rate, elapsed, first_audio_elapsed
|
||||
|
||||
|
||||
async def _abort_streaming_residual_work(
|
||||
omni: AsyncOmni,
|
||||
request_id: str,
|
||||
*,
|
||||
settle_seconds: float = 0.1,
|
||||
) -> None:
|
||||
"""Stop any late stage-0 work once the final audio has been collected."""
|
||||
await omni.engine.abort_async([request_id])
|
||||
if settle_seconds > 0:
|
||||
await asyncio.sleep(settle_seconds)
|
||||
|
||||
|
||||
async def _run_streaming_single(
|
||||
omni: AsyncOmni,
|
||||
args: Any,
|
||||
spec: PromptSpec,
|
||||
output_dir: Path,
|
||||
request_id: str,
|
||||
*,
|
||||
run_index: int,
|
||||
num_runs: int,
|
||||
prompt_index: int,
|
||||
prompt_count: int,
|
||||
) -> Path:
|
||||
audio_cat, sample_rate, elapsed, first_audio_elapsed = await _collect_streaming_audio(
|
||||
omni,
|
||||
args,
|
||||
spec,
|
||||
request_id,
|
||||
phase_label=f"run={run_index + 1}/{num_runs}",
|
||||
prompt_index=prompt_index,
|
||||
prompt_count=prompt_count,
|
||||
print_prompt=(run_index == 0 and prompt_index == 0),
|
||||
)
|
||||
await _abort_streaming_residual_work(omni, request_id)
|
||||
output_path = output_dir / f"output_run{run_index + 1}_{spec.label}.wav"
|
||||
_write_audio_tensor(output_path, audio_cat, sample_rate)
|
||||
audio_duration_s = float(audio_cat.numel()) / float(sample_rate) if sample_rate > 0 else 0.0
|
||||
ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else ""
|
||||
rtf_text = f", rtf={elapsed / audio_duration_s:.3f}" if audio_duration_s > 0 else ""
|
||||
print(
|
||||
f"Saved (streaming) run {run_index + 1}/{num_runs}, "
|
||||
f"prompt {prompt_index + 1}/{prompt_count}: {output_path} ({elapsed:.2f}s{ttfp_text}{rtf_text})"
|
||||
)
|
||||
_emit_offline_metrics(
|
||||
request_id=request_id,
|
||||
elapsed_s=elapsed,
|
||||
first_audio_elapsed=first_audio_elapsed,
|
||||
audio_duration_s=audio_duration_s,
|
||||
)
|
||||
return output_path
|
||||
|
||||
|
||||
async def _run_streaming_warmup(args, omni: AsyncOmni) -> None:
|
||||
if args.warmup_runs == 0:
|
||||
return
|
||||
|
||||
warmup_specs = _get_warmup_specs(args.prompt_specs)
|
||||
print(
|
||||
f"Warmup: {args.warmup_runs} run(s) using the first prompt "
|
||||
f"({len(warmup_specs)} prompt(s)); outputs will be discarded."
|
||||
)
|
||||
for warmup_index in range(args.warmup_runs):
|
||||
t_warmup = time.perf_counter()
|
||||
tasks = []
|
||||
request_ids: list[str] = []
|
||||
for prompt_index, spec in enumerate(warmup_specs):
|
||||
request_id = f"warmup_stream_{warmup_index + 1}_{spec.label}_{uuid.uuid4().hex[:8]}"
|
||||
request_ids.append(request_id)
|
||||
tasks.append(
|
||||
_collect_streaming_audio(
|
||||
omni,
|
||||
args,
|
||||
spec,
|
||||
request_id,
|
||||
phase_label=f"warmup={warmup_index + 1}/{args.warmup_runs}",
|
||||
prompt_index=prompt_index,
|
||||
prompt_count=len(warmup_specs),
|
||||
)
|
||||
)
|
||||
results = await asyncio.gather(*tasks)
|
||||
for request_id in request_ids:
|
||||
await _abort_streaming_residual_work(omni, request_id)
|
||||
total_samples = sum(int(audio.numel()) for audio, _, _, _ in results)
|
||||
warmup_ttfps = [ttfp for _, _, _, ttfp in results if ttfp is not None]
|
||||
ttfp_text = f", ttfp={min(warmup_ttfps):.2f}s" if warmup_ttfps else ""
|
||||
print(
|
||||
f"Warmup (streaming) {warmup_index + 1}/{args.warmup_runs} finished: "
|
||||
f"{len(results)} prompt(s), {total_samples} sample(s) "
|
||||
f"({time.perf_counter() - t_warmup:.2f}s{ttfp_text})"
|
||||
)
|
||||
|
||||
|
||||
async def _run_streaming(args) -> list[Path]:
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
await _run_streaming_warmup(args, omni)
|
||||
profiler_started = False
|
||||
if args.enable_profiler:
|
||||
profile_prefix = f"voxcpm_streaming_{int(time.time())}"
|
||||
stages_text = args.profiler_stages if args.profiler_stages is not None else "all-configured"
|
||||
print(f"Starting profiler (streaming): stages={stages_text}, dir={args.profiler_dir}")
|
||||
await omni.start_profile(profile_prefix=profile_prefix, stages=args.profiler_stages)
|
||||
profiler_started = True
|
||||
t_total = time.perf_counter()
|
||||
total_elapsed = 0.0
|
||||
paths: list[Path] = []
|
||||
prompt_specs: list[PromptSpec] = args.prompt_specs
|
||||
try:
|
||||
for run in range(args.num_runs):
|
||||
for prompt_index, spec in enumerate(prompt_specs):
|
||||
request_id = f"stream_{run + 1}_{spec.label}_{uuid.uuid4().hex[:8]}"
|
||||
paths.append(
|
||||
await _run_streaming_single(
|
||||
omni,
|
||||
args,
|
||||
spec,
|
||||
output_dir,
|
||||
request_id,
|
||||
run_index=run,
|
||||
num_runs=args.num_runs,
|
||||
prompt_index=prompt_index,
|
||||
prompt_count=len(prompt_specs),
|
||||
)
|
||||
)
|
||||
total_elapsed = time.perf_counter() - t_total
|
||||
finally:
|
||||
if profiler_started:
|
||||
print("Stopping profiler (streaming)...")
|
||||
await omni.stop_profile(stages=args.profiler_stages)
|
||||
if args.profiler_wait_seconds > 0:
|
||||
print(f"Waiting {args.profiler_wait_seconds:.1f}s for profiler traces to flush...")
|
||||
await asyncio.sleep(args.profiler_wait_seconds)
|
||||
|
||||
print(
|
||||
f"All streaming runs finished: {args.num_runs} run(s), "
|
||||
f"{len(prompt_specs)} prompt(s), {len(paths)} file(s) in {total_elapsed:.2f}s total"
|
||||
)
|
||||
return paths
|
||||
|
||||
|
||||
def _run_sync(args) -> list[Path]:
|
||||
output_dir = Path(args.output_dir)
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
def _run_sync_single(
|
||||
spec: PromptSpec,
|
||||
*,
|
||||
request_prefix: str,
|
||||
save_outputs: bool,
|
||||
run_index: int | None = None,
|
||||
) -> tuple[list[Path], int, float | None, float, float, str]:
|
||||
global_request_id = f"{request_prefix}_{spec.label}"
|
||||
prompt = _build_prompt_for_spec(args, spec, global_request_id=global_request_id)
|
||||
if save_outputs and run_index == 0 and spec.label == "item001":
|
||||
print(f"---prompt---:{prompt}")
|
||||
|
||||
saved_paths: list[Path] = []
|
||||
output_count = 0
|
||||
first_audio_elapsed: float | None = None
|
||||
total_audio_duration_s = 0.0
|
||||
metrics_request_id = global_request_id
|
||||
t_start = time.perf_counter()
|
||||
for stage_outputs in omni.generate(prompt):
|
||||
request_output = stage_outputs.request_output
|
||||
if request_output is None:
|
||||
continue
|
||||
request_output_id = getattr(request_output, "request_id", None)
|
||||
if isinstance(request_output_id, str) and request_output_id:
|
||||
metrics_request_id = request_output_id
|
||||
for j, mm in enumerate(_iter_request_multimodal_outputs(request_output)):
|
||||
output_count += 1
|
||||
if first_audio_elapsed is None:
|
||||
try:
|
||||
audio_tensor = _extract_audio_tensor(mm)
|
||||
if int(audio_tensor.numel()) > 0:
|
||||
first_audio_elapsed = time.perf_counter() - t_start
|
||||
total_audio_duration_s += float(audio_tensor.numel()) / float(_extract_sample_rate(mm))
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
audio_tensor = _extract_audio_tensor(mm)
|
||||
total_audio_duration_s += float(audio_tensor.numel()) / float(_extract_sample_rate(mm))
|
||||
except ValueError:
|
||||
pass
|
||||
if not save_outputs:
|
||||
continue
|
||||
save_stem = f"run{run_index + 1}_{spec.label}" if j == 0 else f"run{run_index + 1}_{spec.label}_{j}"
|
||||
saved_paths.append(_save_wav(mm, output_dir, save_stem))
|
||||
|
||||
if output_count == 0:
|
||||
raise RuntimeError("No output from Omni.generate")
|
||||
elapsed_s = time.perf_counter() - t_start
|
||||
return saved_paths, output_count, first_audio_elapsed, elapsed_s, total_audio_duration_s, metrics_request_id
|
||||
|
||||
if args.warmup_runs:
|
||||
warmup_specs = _get_warmup_specs(args.prompt_specs)
|
||||
print(
|
||||
f"Warmup: {args.warmup_runs} run(s) using the first prompt "
|
||||
f"({len(warmup_specs)} prompt(s)); outputs will be discarded."
|
||||
)
|
||||
for warmup_index in range(args.warmup_runs):
|
||||
t_warmup = time.perf_counter()
|
||||
_, output_count, first_audio_elapsed, elapsed_s, audio_duration_s, _ = _run_sync_single(
|
||||
warmup_specs[0],
|
||||
request_prefix=f"warmup_sync{warmup_index + 1}",
|
||||
save_outputs=False,
|
||||
)
|
||||
ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else ""
|
||||
rtf_text = f", rtf={elapsed_s / audio_duration_s:.3f}" if audio_duration_s > 0 else ""
|
||||
print(
|
||||
f"Warmup (sync) {warmup_index + 1}/{args.warmup_runs} finished: "
|
||||
f"{output_count} output(s) ({time.perf_counter() - t_warmup:.2f}s{ttfp_text}{rtf_text})"
|
||||
)
|
||||
|
||||
profiler_started = False
|
||||
if args.enable_profiler:
|
||||
profile_prefix = f"voxcpm_sync_{int(time.time())}"
|
||||
stages_text = args.profiler_stages if args.profiler_stages is not None else "all-configured"
|
||||
print(f"Starting profiler (sync): stages={stages_text}, dir={args.profiler_dir}")
|
||||
omni.start_profile(profile_prefix=profile_prefix, stages=args.profiler_stages)
|
||||
profiler_started = True
|
||||
|
||||
t_total = time.perf_counter()
|
||||
total_elapsed = 0.0
|
||||
saved_paths: list[Path] = []
|
||||
prompt_specs: list[PromptSpec] = args.prompt_specs
|
||||
try:
|
||||
for run in range(args.num_runs):
|
||||
t_run = time.perf_counter()
|
||||
run_paths: list[Path] = []
|
||||
for prompt_index, spec in enumerate(prompt_specs):
|
||||
prompt_paths, _, first_audio_elapsed, elapsed_s, audio_duration_s, metrics_request_id = (
|
||||
_run_sync_single(
|
||||
spec,
|
||||
request_prefix=f"sync_run{run + 1}_{prompt_index + 1:03d}",
|
||||
save_outputs=True,
|
||||
run_index=run,
|
||||
)
|
||||
)
|
||||
run_paths.extend(prompt_paths)
|
||||
ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else ""
|
||||
rtf_text = f", rtf={elapsed_s / audio_duration_s:.3f}" if audio_duration_s > 0 else ""
|
||||
print(
|
||||
f"Saved (sync) run {run + 1}/{args.num_runs}, "
|
||||
f"prompt {prompt_index + 1}/{len(prompt_specs)}: {len(prompt_paths)} file(s){ttfp_text}{rtf_text}"
|
||||
)
|
||||
_emit_offline_metrics(
|
||||
request_id=metrics_request_id,
|
||||
elapsed_s=elapsed_s,
|
||||
first_audio_elapsed=first_audio_elapsed,
|
||||
audio_duration_s=audio_duration_s,
|
||||
)
|
||||
|
||||
saved_paths.extend(run_paths)
|
||||
print(
|
||||
f"Run {run + 1}/{args.num_runs} finished: {len(run_paths)} file(s) ({time.perf_counter() - t_run:.2f}s)"
|
||||
)
|
||||
for path in run_paths:
|
||||
print(f" {path}")
|
||||
|
||||
total_elapsed = time.perf_counter() - t_total
|
||||
finally:
|
||||
if profiler_started:
|
||||
print("Stopping profiler (sync)...")
|
||||
omni.stop_profile(stages=args.profiler_stages)
|
||||
if args.profiler_wait_seconds > 0:
|
||||
print(f"Waiting {args.profiler_wait_seconds:.1f}s for profiler traces to flush...")
|
||||
time.sleep(args.profiler_wait_seconds)
|
||||
|
||||
print(
|
||||
f"All sync runs finished: {args.num_runs} run(s), "
|
||||
f"{len(prompt_specs)} prompt(s), {len(saved_paths)} file(s) in {total_elapsed:.2f}s total"
|
||||
)
|
||||
return saved_paths
|
||||
|
||||
|
||||
def main(args) -> int:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
profiled_stage_config_path: str | None = None
|
||||
original_stage_config_path = args.stage_configs_path
|
||||
if args.enable_profiler:
|
||||
Path(args.profiler_dir).mkdir(parents=True, exist_ok=True)
|
||||
profiled_stage_config_path = _build_profiled_stage_config(
|
||||
args.stage_configs_path,
|
||||
str(Path(args.profiler_dir).resolve()),
|
||||
)
|
||||
args.stage_configs_path = profiled_stage_config_path
|
||||
|
||||
is_streaming = _is_streaming_stage_config(args.stage_configs_path)
|
||||
voice_clone_count = _count_voice_clone_prompts(args.prompt_specs)
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Stage config: {original_stage_config_path}")
|
||||
print(f"Route: {'streaming' if is_streaming else 'sync'} (from stage-configs-path)")
|
||||
print(f"Prompt count: {len(args.prompt_specs)}")
|
||||
print("Batch mode: sequential (aligned with native VoxCPM)")
|
||||
print(f"Warmup runs: {args.warmup_runs}")
|
||||
print(f"Voice cloning prompts: {voice_clone_count}/{len(args.prompt_specs)}")
|
||||
if args.enable_profiler:
|
||||
print(f"Profiler: enabled (dir={args.profiler_dir}, stages={args.profiler_stages or 'all-configured'})")
|
||||
print(f"Profiled stage config: {args.stage_configs_path}")
|
||||
if voice_clone_count:
|
||||
print("Voice cloning note: --ref-text/ref_text must match the spoken content of the reference audio.")
|
||||
print(f"Num runs: {args.num_runs}")
|
||||
try:
|
||||
if is_streaming:
|
||||
asyncio.run(_run_streaming(args))
|
||||
else:
|
||||
_run_sync(args)
|
||||
finally:
|
||||
if profiled_stage_config_path is not None and os.path.exists(profiled_stage_config_path):
|
||||
os.unlink(profiled_stage_config_path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
raise SystemExit(main(parse_args()))
|
||||
@@ -121,7 +121,6 @@ Configuration classes.
|
||||
- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
|
||||
|
||||
## Workers
|
||||
|
||||
@@ -5,6 +5,9 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation.
|
||||
- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz.
|
||||
- **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
|
||||
- **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
|
||||
- **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz.
|
||||
|
||||
See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, and MOSS-TTS-Nano.
|
||||
|
||||
Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
|
||||
|
||||
@@ -26,6 +29,10 @@ vllm serve fishaudio/s2-pro --omni --port 8091
|
||||
|
||||
# Voxtral TTS
|
||||
vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
|
||||
|
||||
# CosyVoice3 (voice cloning only — supply ref_audio + ref_text per request)
|
||||
vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 \
|
||||
--omni --port 8091 --trust-remote-code
|
||||
```
|
||||
|
||||
### Generate Speech
|
||||
|
||||
@@ -17,7 +17,6 @@ list of supported architectures across all modalities, see
|
||||
| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
|
||||
|---|---|---|---|---|---|---|
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz |
|
||||
| VoxCPM | local model dir | split | ✓ | ✓ (`voxcpm_async_chunk.yaml`) | — | 24 kHz |
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz |
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design (`--instruct`), language (`--lang`) | 24 kHz |
|
||||
@@ -72,61 +71,6 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM
|
||||
|
||||
Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
pip install voxcpm soundfile
|
||||
# or use a local source tree:
|
||||
export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
|
||||
```
|
||||
|
||||
If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
|
||||
|
||||
```bash
|
||||
export VOXCPM_MODEL=/path/to/voxcpm-model
|
||||
export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
|
||||
mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
|
||||
cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
|
||||
cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
|
||||
python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
|
||||
```
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This sentence is synthesized with a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in reference.wav."
|
||||
```
|
||||
|
||||
### Streaming
|
||||
Pass the async-chunk stage config:
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
|
||||
--text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
|
||||
```
|
||||
|
||||
### Notes
|
||||
- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
|
||||
- Streaming is currently single-request oriented.
|
||||
- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
|
||||
- For online serving, see [`examples/online_serving/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm/README.md). For benchmark reporting, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
|
||||
|
||||
---
|
||||
|
||||
## CosyVoice3
|
||||
|
||||
2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
|
||||
@@ -381,10 +325,6 @@ Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-T
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/text_to_speech/qwen3_tts/end2end.py"
|
||||
``````
|
||||
??? abstract "voxcpm/end2end.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/text_to_speech/voxcpm/end2end.py"
|
||||
``````
|
||||
??? abstract "voxcpm2/end2end.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/text_to_speech/voxcpm2/end2end.py"
|
||||
|
||||
@@ -17,14 +17,14 @@ For the full list of supported architectures across all modalities, see
|
||||
|
||||
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
|
||||
|---|---|---|---|---|---|
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
|
||||
| VoxCPM | local model dir | ✓ | ✓ (PCM stream) | — | — |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
|
||||
|
||||
CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3) instead.
|
||||
For offline inference of any of these models, see the [offline TTS hub](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md).
|
||||
|
||||
## Common Quick Start
|
||||
|
||||
@@ -96,6 +96,54 @@ For full request-shape documentation (all parameters, response formats, error co
|
||||
|
||||
---
|
||||
|
||||
## CosyVoice3
|
||||
|
||||
2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
huggingface-cli download FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
||||
```
|
||||
|
||||
If your downloaded checkpoint lacks `config.json`, add one with `{"model_type": "cosyvoice3", "architectures": ["CosyVoice3Model"]}` (the loader reads `model_type` to select the class).
|
||||
|
||||
### Launch
|
||||
```bash
|
||||
vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code
|
||||
# or:
|
||||
./cosyvoice3/run_server.sh
|
||||
```
|
||||
|
||||
Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path.
|
||||
|
||||
### CLI client
|
||||
The client defaults to the official upstream zero-shot prompt, so it runs without extra flags:
|
||||
```bash
|
||||
cd examples/online_serving/text_to_speech/cosyvoice3
|
||||
python speech_client.py --text "收到好友从远方寄来的生日礼物。"
|
||||
```
|
||||
|
||||
Pass your own reference clip and transcript for a different voice:
|
||||
```bash
|
||||
python speech_client.py --text "Hello, this is a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Transcript of the reference audio."
|
||||
```
|
||||
|
||||
Stream PCM instead of WAV:
|
||||
```bash
|
||||
python speech_client.py --text "Hello world" --stream --output output.pcm
|
||||
```
|
||||
|
||||
The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-text`, `--response-format`, `--stream`, `--output`.
|
||||
|
||||
### Notes
|
||||
- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
|
||||
- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
|
||||
- For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3).
|
||||
|
||||
---
|
||||
|
||||
## Fish Speech S2 Pro
|
||||
|
||||
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
|
||||
@@ -196,10 +244,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
|
||||
./qwen3_tts/run_server.sh Base
|
||||
```
|
||||
|
||||
### Choosing an executor backend (uniproc vs mp)
|
||||
Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
|
||||
### Executor backend
|
||||
Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
|
||||
|
||||
To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
|
||||
To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor.
|
||||
|
||||
### Sending requests
|
||||
```bash
|
||||
@@ -281,71 +329,11 @@ python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
|
||||
`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
|
||||
|
||||
### Notes
|
||||
- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
|
||||
- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background.
|
||||
- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM
|
||||
|
||||
Split-stage TTS at 24 kHz.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
pip install voxcpm
|
||||
# or use a local source tree:
|
||||
export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
|
||||
```
|
||||
|
||||
If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
|
||||
```bash
|
||||
export VOXCPM_MODEL=/path/to/voxcpm-model
|
||||
export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
|
||||
mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
|
||||
cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
|
||||
cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
|
||||
python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
|
||||
```
|
||||
|
||||
### Launch
|
||||
```bash
|
||||
export VOXCPM_MODEL=/path/to/voxcpm-model
|
||||
./voxcpm/run_server.sh # async-chunk streaming (default)
|
||||
./voxcpm/run_server.sh sync # non-streaming
|
||||
```
|
||||
Or directly:
|
||||
```bash
|
||||
vllm serve "$VOXCPM_MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
|
||||
--trust-remote-code --enforce-eager --omni --port 8091
|
||||
```
|
||||
|
||||
### Sending requests
|
||||
```bash
|
||||
# Basic TTS
|
||||
python voxcpm/openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a VoxCPM online text-to-speech example."
|
||||
|
||||
# Voice cloning
|
||||
python voxcpm/openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This sentence is synthesized with a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in reference.wav."
|
||||
|
||||
# Streaming PCM
|
||||
python voxcpm/openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a streaming VoxCPM request." \
|
||||
--stream --output voxcpm_stream.pcm
|
||||
```
|
||||
|
||||
### Notes
|
||||
- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
|
||||
- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
|
||||
- For benchmark measurement, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM2
|
||||
@@ -407,6 +395,14 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "cosyvoice3/run_server.sh"
|
||||
|
||||
--8<-- "examples/online_serving/text_to_speech/cosyvoice3/run_server.sh"
|
||||
|
||||
??? abstract "cosyvoice3/speech_client.py"
|
||||
|
||||
--8<-- "examples/online_serving/text_to_speech/cosyvoice3/speech_client.py"
|
||||
|
||||
??? abstract "fish_speech/gradio_demo.py"
|
||||
``````py
|
||||
--8<-- "examples/online_serving/text_to_speech/fish_speech/gradio_demo.py"
|
||||
@@ -463,14 +459,6 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
|
||||
``````py
|
||||
--8<-- "examples/online_serving/text_to_speech/qwen3_tts/tts_common.py"
|
||||
``````
|
||||
??? abstract "voxcpm/openai_speech_client.py"
|
||||
``````py
|
||||
--8<-- "examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py"
|
||||
``````
|
||||
??? abstract "voxcpm/run_server.sh"
|
||||
``````sh
|
||||
--8<-- "examples/online_serving/text_to_speech/voxcpm/run_server.sh"
|
||||
``````
|
||||
??? abstract "voxcpm2/gradio_demo.py"
|
||||
``````py
|
||||
--8<-- "examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py"
|
||||
|
||||
@@ -14,13 +14,12 @@ list of supported architectures across all modalities, see
|
||||
|
||||
| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
|
||||
|---|---|---|---|---|---|---|
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 22.05 kHz |
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
|
||||
| Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz |
|
||||
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz |
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
|
||||
| VoxCPM | `openbmb/VoxCPM-0.5B` | split | ✓ | ✓ | — | 24 kHz |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
|
||||
|
||||
@@ -41,7 +40,7 @@ python examples/offline_inference/text_to_speech/<model>/end2end.py \
|
||||
|
||||
## CosyVoice3
|
||||
|
||||
2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
|
||||
2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
@@ -308,56 +307,6 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM
|
||||
|
||||
Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
pip install voxcpm soundfile
|
||||
# or use a local source tree:
|
||||
export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
|
||||
```
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
|
||||
--model openbmb/VoxCPM-0.5B \
|
||||
--text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
|
||||
--model openbmb/VoxCPM-0.5B \
|
||||
--text "This sentence is synthesized with a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in reference.wav."
|
||||
```
|
||||
|
||||
### Streaming
|
||||
Pass `--streaming` together with the legacy async-chunk stage config:
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
|
||||
--model openbmb/VoxCPM-0.5B \
|
||||
--streaming \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
|
||||
--text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
|
||||
```
|
||||
|
||||
### Persistent HF config (optional)
|
||||
The engine auto-patches a missing `model_type` in the checkpoint's `config.json` in-memory at startup (see `vllm_omni/engine/arg_utils.py`), so no manual setup is required.
|
||||
|
||||
If you'd rather maintain a persistent patched config dir (e.g. to share across runs or to add fields the auto-patcher doesn't set), export `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` to its location — both the bundled deploy and async-chunk yamls read it via `${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}`.
|
||||
|
||||
### Notes
|
||||
- Non-streaming auto-loads `vllm_omni/deploy/voxcpm.yaml` from the model directory name / HF `model_type`. Streaming uses the legacy `voxcpm_async_chunk.yaml` passed via `--stage-configs-path`.
|
||||
- Streaming is currently single-request oriented.
|
||||
- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
|
||||
- For online serving, see the [VoxCPM section in the online hub](../../online_serving/text_to_speech/README.md#voxcpm). For benchmark reporting, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM2
|
||||
|
||||
Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
|
||||
|
||||
@@ -1,220 +0,0 @@
|
||||
"""Minimal offline VoxCPM example for vLLM Omni."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import AsyncOmni, Omni
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
|
||||
def _build_prompt(args) -> dict[str, Any]:
|
||||
additional_information: dict[str, list[Any]] = {
|
||||
"text": [args.text],
|
||||
"cfg_value": [args.cfg_value],
|
||||
"inference_timesteps": [args.inference_timesteps],
|
||||
"min_len": [args.min_len],
|
||||
"max_new_tokens": [args.max_new_tokens],
|
||||
}
|
||||
if args.streaming_prefix_len is not None:
|
||||
additional_information["streaming_prefix_len"] = [args.streaming_prefix_len]
|
||||
if args.ref_audio is not None:
|
||||
additional_information["ref_audio"] = [args.ref_audio]
|
||||
if args.ref_text is not None:
|
||||
additional_information["ref_text"] = [args.ref_text]
|
||||
return {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": additional_information,
|
||||
}
|
||||
|
||||
|
||||
def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor:
|
||||
audio = mm.get("audio", mm.get("model_outputs"))
|
||||
if audio is None:
|
||||
raise ValueError("No audio output found in multimodal output.")
|
||||
if isinstance(audio, list):
|
||||
parts = [torch.as_tensor(item).float().cpu().reshape(-1) for item in audio]
|
||||
audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0)
|
||||
if not isinstance(audio, torch.Tensor):
|
||||
audio = torch.as_tensor(audio)
|
||||
return audio.float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def _extract_sample_rate(mm: dict[str, Any]) -> int:
|
||||
sr_raw = mm.get("sr", 24000)
|
||||
if isinstance(sr_raw, list) and sr_raw:
|
||||
sr_raw = sr_raw[-1]
|
||||
if hasattr(sr_raw, "item"):
|
||||
return int(sr_raw.item())
|
||||
return int(sr_raw)
|
||||
|
||||
|
||||
def _save_audio(audio: torch.Tensor, sample_rate: int, output_dir: Path, request_id: str) -> Path:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_dir / f"output_{request_id}.wav"
|
||||
sf.write(
|
||||
output_path,
|
||||
audio.float().cpu().clamp(-1.0, 1.0).numpy(),
|
||||
sample_rate,
|
||||
format="WAV",
|
||||
subtype="PCM_16",
|
||||
)
|
||||
return output_path
|
||||
|
||||
|
||||
async def _run_streaming(args) -> Path:
|
||||
prompt = _build_prompt(args)
|
||||
output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio_streaming")
|
||||
request_id = "streaming_example"
|
||||
sample_rate = 24000
|
||||
buffered_samples = 0
|
||||
chunks: list[torch.Tensor] = []
|
||||
started = time.perf_counter()
|
||||
omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
deploy_config=args.deploy_config,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
try:
|
||||
async for stage_output in omni.generate(prompt, request_id=request_id):
|
||||
mm = getattr(stage_output, "multimodal_output", None)
|
||||
if not isinstance(mm, dict):
|
||||
request_output = getattr(stage_output, "request_output", None)
|
||||
if request_output is None:
|
||||
continue
|
||||
mm = getattr(request_output, "multimodal_output", None)
|
||||
if not isinstance(mm, dict) and getattr(request_output, "outputs", None):
|
||||
mm = getattr(request_output.outputs[0], "multimodal_output", None)
|
||||
if not isinstance(mm, dict):
|
||||
continue
|
||||
audio = _extract_audio_tensor(mm)
|
||||
if audio.numel() == 0:
|
||||
continue
|
||||
sample_rate = _extract_sample_rate(mm)
|
||||
if audio.numel() > buffered_samples:
|
||||
delta = audio[buffered_samples:]
|
||||
buffered_samples = int(audio.numel())
|
||||
else:
|
||||
delta = audio
|
||||
buffered_samples += int(delta.numel())
|
||||
if delta.numel() > 0:
|
||||
chunks.append(delta)
|
||||
if not chunks:
|
||||
raise RuntimeError("No streaming audio chunks received from VoxCPM.")
|
||||
output_audio = torch.cat(chunks, dim=0)
|
||||
output_path = _save_audio(output_audio, sample_rate, output_dir, request_id)
|
||||
print(f"Saved streaming audio to: {output_path} ({time.perf_counter() - started:.2f}s)")
|
||||
return output_path
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
def _run_sync(args) -> Path:
|
||||
prompt = _build_prompt(args)
|
||||
output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio")
|
||||
request_id = "sync_example"
|
||||
started = time.perf_counter()
|
||||
last_mm: dict[str, Any] | None = None
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
deploy_config=args.deploy_config,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
for stage_outputs in omni.generate(prompt):
|
||||
request_output = getattr(stage_outputs, "request_output", None)
|
||||
if request_output is None:
|
||||
continue
|
||||
outputs = getattr(request_output, "outputs", None)
|
||||
if outputs:
|
||||
for output in outputs:
|
||||
mm = getattr(output, "multimodal_output", None)
|
||||
if isinstance(mm, dict):
|
||||
last_mm = mm
|
||||
mm = getattr(request_output, "multimodal_output", None)
|
||||
if isinstance(mm, dict):
|
||||
last_mm = mm
|
||||
if last_mm is None:
|
||||
raise RuntimeError("No audio output received from VoxCPM.")
|
||||
output_path = _save_audio(
|
||||
_extract_audio_tensor(last_mm),
|
||||
_extract_sample_rate(last_mm),
|
||||
output_dir,
|
||||
request_id,
|
||||
)
|
||||
print(f"Saved audio to: {output_path} ({time.perf_counter() - started:.2f}s)")
|
||||
return output_path
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="Minimal offline VoxCPM example for vLLM Omni.")
|
||||
parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help=("Override the deploy config path."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Legacy stage_args yaml path. Required for streaming "
|
||||
"(vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml); "
|
||||
"leave unset for non-streaming to use the auto-resolved deploy config."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--streaming",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Stream audio chunks as they arrive via AsyncOmni.",
|
||||
)
|
||||
parser.add_argument("--text", type=str, required=True, help="Input text for synthesis.")
|
||||
parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
|
||||
parser.add_argument("--ref-text", type=str, default=None, help="Transcript of the reference audio.")
|
||||
parser.add_argument("--output-dir", type=str, default=None, help="Output directory for generated wav files.")
|
||||
parser.add_argument("--cfg-value", type=float, default=2.0, help="Guidance value passed to VoxCPM.")
|
||||
parser.add_argument("--inference-timesteps", type=int, default=10, help="Number of diffusion timesteps.")
|
||||
parser.add_argument("--min-len", type=int, default=2, help="Minimum latent length.")
|
||||
parser.add_argument("--max-new-tokens", type=int, default=4096, help="Maximum latent length.")
|
||||
parser.add_argument(
|
||||
"--streaming-prefix-len",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Streaming prefix length used by voxcpm_async_chunk.yaml.",
|
||||
)
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.")
|
||||
parser.add_argument("--log-stats", action="store_true", help="Enable vLLM Omni stats logging.")
|
||||
nullify_stage_engine_defaults(parser)
|
||||
args = parser.parse_args()
|
||||
if (args.ref_audio is None) != (args.ref_text is None):
|
||||
raise ValueError("Voice cloning requires --ref-audio and --ref-text together.")
|
||||
return args
|
||||
|
||||
|
||||
def main(args) -> None:
|
||||
route = "streaming" if args.streaming else "sync"
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Deploy config: {args.deploy_config or '<auto from HF model_type>'}")
|
||||
print(f"Stage configs path: {args.stage_configs_path or '<unused>'}")
|
||||
print(f"Route: {route}")
|
||||
if route == "streaming":
|
||||
asyncio.run(_run_streaming(args))
|
||||
else:
|
||||
_run_sync(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(parse_args())
|
||||
@@ -19,7 +19,6 @@ For the full list of supported architectures across all modalities, see
|
||||
| MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ |
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
|
||||
| VoxCPM | `openbmb/VoxCPM-0.5B` | ✓ | ✓ (PCM stream) | — | — |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
|
||||
|
||||
@@ -294,10 +293,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
|
||||
./qwen3_tts/run_server.sh Base
|
||||
```
|
||||
|
||||
### Choosing an executor backend (uniproc vs mp)
|
||||
Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
|
||||
### Executor backend
|
||||
Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config <path>` to override.
|
||||
|
||||
To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
|
||||
To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor.
|
||||
|
||||
### Sending requests
|
||||
```bash
|
||||
@@ -375,73 +374,11 @@ python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-de
|
||||
`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
|
||||
|
||||
### Notes
|
||||
- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
|
||||
- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background.
|
||||
- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM
|
||||
|
||||
Split-stage TTS at 24 kHz.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
pip install voxcpm
|
||||
# or use a local source tree:
|
||||
export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
|
||||
```
|
||||
|
||||
If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
|
||||
```bash
|
||||
export VOXCPM_MODEL=/path/to/voxcpm-model
|
||||
export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
|
||||
mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
|
||||
cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
|
||||
cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
|
||||
python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
|
||||
```
|
||||
|
||||
### Launch
|
||||
```bash
|
||||
export VOXCPM_MODEL=/path/to/voxcpm-model
|
||||
./voxcpm/run_server.sh # async-chunk streaming (default)
|
||||
./voxcpm/run_server.sh sync # non-streaming
|
||||
```
|
||||
Or directly:
|
||||
```bash
|
||||
vllm serve "$VOXCPM_MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
|
||||
--trust-remote-code --enforce-eager --omni --port 8091
|
||||
```
|
||||
|
||||
### Sending requests
|
||||
```bash
|
||||
# Basic TTS
|
||||
python voxcpm/openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a VoxCPM online text-to-speech example."
|
||||
|
||||
# Voice cloning
|
||||
python voxcpm/openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This sentence is synthesized with a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in reference.wav."
|
||||
|
||||
# Streaming PCM
|
||||
python voxcpm/openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a streaming VoxCPM request." \
|
||||
--stream --output voxcpm_stream.pcm
|
||||
```
|
||||
|
||||
### Notes
|
||||
- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
|
||||
- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
|
||||
- For benchmark measurement, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).
|
||||
|
||||
---
|
||||
|
||||
## VoxCPM2
|
||||
|
||||
Single-stage native AR TTS at 48 kHz.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for CosyVoice3 TTS
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 ./run_server.sh
|
||||
#
|
||||
# Streaming (async-chunk) is on by default via vllm_omni/deploy/cosyvoice3.yaml.
|
||||
# Set NO_ASYNC_CHUNK=1 to use the legacy synchronous path.
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-FunAudioLLM/Fun-CosyVoice3-0.5B-2512}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "${NO_ASYNC_CHUNK:-}" ]]; then
|
||||
EXTRA_ARGS+=(--no-async-chunk)
|
||||
fi
|
||||
|
||||
echo "Starting CosyVoice3 server with model: $MODEL"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
--omni \
|
||||
"${EXTRA_ARGS[@]}"
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Client for CosyVoice3 TTS via /v1/audio/speech endpoint.
|
||||
|
||||
CosyVoice3 has no built-in voice presets: every request is voice cloning
|
||||
driven by ``ref_audio`` + ``ref_text``. The defaults below point at the
|
||||
official upstream zero-shot prompt so the script runs out of the box.
|
||||
|
||||
Examples:
|
||||
# Voice cloning with the default upstream prompt
|
||||
python speech_client.py --text "收到好友从远方寄来的生日礼物。"
|
||||
|
||||
# Custom reference clip + transcript
|
||||
python speech_client.py --text "Hello, this is a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Transcript of the reference audio."
|
||||
|
||||
# Streaming PCM output
|
||||
python speech_client.py --text "Hello world" --stream --output output.pcm
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
|
||||
|
||||
# Official CosyVoice zero-shot prompt and its transcript.
|
||||
DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
|
||||
DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_path: str) -> str:
|
||||
"""Encode a local audio file to a base64 data URL."""
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
ext = audio_path.lower().rsplit(".", 1)[-1]
|
||||
mime_map = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"}
|
||||
mime_type = mime_map.get(ext, "audio/wav")
|
||||
with open(audio_path, "rb") as f:
|
||||
audio_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{audio_b64}"
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
"""Generate speech via the /v1/audio/speech API."""
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
|
||||
if args.ref_audio.startswith(("http://", "https://")):
|
||||
payload["ref_audio"] = args.ref_audio
|
||||
else:
|
||||
payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
|
||||
payload["ref_text"] = args.ref_text
|
||||
|
||||
if args.stream:
|
||||
payload["stream"] = True
|
||||
payload["response_format"] = "pcm"
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}")
|
||||
print("Generating audio...")
|
||||
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or "output.pcm"
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream("POST", api_url, json=payload, headers=headers) as resp:
|
||||
if resp.status_code != 200:
|
||||
print(f"Error: {resp.status_code}")
|
||||
print(resp.read().decode())
|
||||
return
|
||||
total_bytes = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in resp.iter_bytes():
|
||||
f.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
print(f"Streamed {total_bytes} bytes to: {output_path}")
|
||||
else:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
try:
|
||||
text = response.content.decode("utf-8")
|
||||
if text.startswith('{"error"'):
|
||||
print(f"Error: {text}")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
output_path = args.output or "output.wav"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="CosyVoice3 TTS client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
default=DEFAULT_REF_AUDIO,
|
||||
help="Reference audio for voice cloning (path or URL)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
default=DEFAULT_REF_TEXT,
|
||||
help="Transcript of the reference audio",
|
||||
)
|
||||
parser.add_argument("--stream", action="store_true", help="Enable streaming (PCM output)")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
|
||||
help="Audio format (default: wav)",
|
||||
)
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,155 +0,0 @@
|
||||
"""OpenAI-compatible client for VoxCPM via /v1/audio/speech.
|
||||
|
||||
Examples:
|
||||
# Basic text-to-speech
|
||||
python openai_speech_client.py --text "Hello from VoxCPM"
|
||||
|
||||
# Voice cloning
|
||||
python openai_speech_client.py \
|
||||
--text "This sentence uses the cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in the reference audio."
|
||||
|
||||
# Streaming PCM output
|
||||
python openai_speech_client.py \
|
||||
--text "This is a streaming VoxCPM request." \
|
||||
--stream \
|
||||
--output output.pcm
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
DEFAULT_MODEL = "OpenBMB/VoxCPM1.5"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_path: str) -> str:
|
||||
"""Encode a local audio file to base64 data URL."""
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
|
||||
ext = audio_path.lower().rsplit(".", 1)[-1]
|
||||
mime_map = {
|
||||
"wav": "audio/wav",
|
||||
"mp3": "audio/mpeg",
|
||||
"flac": "audio/flac",
|
||||
"ogg": "audio/ogg",
|
||||
}
|
||||
mime_type = mime_map.get(ext, "audio/wav")
|
||||
|
||||
with open(audio_path, "rb") as f:
|
||||
audio_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{audio_b64}"
|
||||
|
||||
|
||||
def build_payload(args) -> dict[str, object]:
|
||||
payload: dict[str, object] = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"response_format": "pcm" if args.stream else args.response_format,
|
||||
}
|
||||
|
||||
if args.ref_audio:
|
||||
if args.ref_audio.startswith(("http://", "https://", "data:")):
|
||||
payload["ref_audio"] = args.ref_audio
|
||||
else:
|
||||
payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
|
||||
if args.ref_text:
|
||||
payload["ref_text"] = args.ref_text
|
||||
if args.max_new_tokens is not None:
|
||||
payload["max_new_tokens"] = args.max_new_tokens
|
||||
if args.stream:
|
||||
payload["stream"] = True
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
payload = build_payload(args)
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
if args.ref_audio:
|
||||
print("Mode: voice cloning")
|
||||
print(f"Reference audio: {args.ref_audio}")
|
||||
else:
|
||||
print("Mode: text-to-speech")
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or "voxcpm_output.pcm"
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream("POST", api_url, json=payload, headers=headers) as response:
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.read().decode("utf-8", errors="ignore"))
|
||||
return
|
||||
|
||||
total_bytes = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in response.iter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
f.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
print(f"Streamed {total_bytes} bytes to: {output_path}")
|
||||
return
|
||||
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
try:
|
||||
text = response.content.decode("utf-8")
|
||||
if text.startswith('{"error"'):
|
||||
print(f"Error: {text}")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
output_path = args.output or "voxcpm_output.wav"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="VoxCPM OpenAI-compatible speech client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name or path")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--ref-audio", default=None, help="Reference audio path, URL, or data URL")
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
default=None,
|
||||
help="The exact transcript spoken in the reference audio",
|
||||
)
|
||||
parser.add_argument("--stream", action="store_true", help="Enable streaming PCM output")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "pcm", "flac", "mp3", "aac", "opus"],
|
||||
help="Audio format for non-streaming mode (default: wav)",
|
||||
)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate")
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for VoxCPM online speech serving.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh # default: async_chunk stage config
|
||||
# ./run_server.sh async # async_chunk stage config
|
||||
# ./run_server.sh sync # no-async-chunk stage config
|
||||
# VOXCPM_MODEL=/path/to/model ./run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODE="${1:-async}"
|
||||
MODEL="${VOXCPM_MODEL:-OpenBMB/VoxCPM1.5}"
|
||||
|
||||
case "$MODE" in
|
||||
async)
|
||||
STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml"
|
||||
;;
|
||||
sync)
|
||||
STAGE_CONFIG="vllm_omni/deploy/voxcpm.yaml"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown mode: $MODE"
|
||||
echo "Supported: async, sync"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Starting VoxCPM server with model: $MODEL"
|
||||
echo "Stage config: $STAGE_CONFIG"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--stage-configs-path "$STAGE_CONFIG" \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni
|
||||
+1
-1
@@ -63,7 +63,7 @@ dev = [
|
||||
"funasr>=1.0.0",
|
||||
"lpips>=0.1.4", # for quantization fp8
|
||||
"orjson>=3.11.7", # to make v1/embeddings API fast
|
||||
"voxcpm>=2.0.2", # for voxcpm model Test
|
||||
"voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests
|
||||
"torchdiffeq>=0.2.5" # for covo_audio Test
|
||||
]
|
||||
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E test for VoxCPM offline inference."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
VOXCPM_MODEL = "OpenBMB/VoxCPM1.5"
|
||||
STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml")
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
# (model, stage_config_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
|
||||
_OMNI_RUNNER_PARAM = (VOXCPM_MODEL, STAGE_CONFIG)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.full_model,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
|
||||
def _build_prompt(text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": {
|
||||
"text": [text],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [1024],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _extract_audio_tensor(multimodal_output: dict[str, Any]) -> torch.Tensor:
|
||||
audio = multimodal_output.get("audio", multimodal_output.get("model_outputs"))
|
||||
assert audio is not None, f"No audio output found, keys={list(multimodal_output.keys())}"
|
||||
|
||||
if isinstance(audio, list):
|
||||
parts: list[torch.Tensor] = []
|
||||
for item in audio:
|
||||
if item is None:
|
||||
continue
|
||||
tensor = torch.as_tensor(item)
|
||||
if tensor.numel() == 0:
|
||||
continue
|
||||
parts.append(tensor.float().cpu().reshape(-1))
|
||||
return torch.cat(parts, dim=-1) if parts else torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
|
||||
for item in reversed(outputs):
|
||||
request_output = getattr(item, "request_output", None)
|
||||
if request_output is not None:
|
||||
multimodal_output = getattr(request_output, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
completions = getattr(request_output, "outputs", None) or []
|
||||
for completion in completions:
|
||||
multimodal_output = getattr(completion, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
|
||||
multimodal_output = getattr(item, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
|
||||
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm_zero_shot_001(omni_runner: OmniRunner) -> None:
|
||||
outputs = list(omni_runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
|
||||
multimodal_output = _extract_final_multimodal_output(outputs)
|
||||
audio = _extract_audio_tensor(multimodal_output)
|
||||
assert audio.numel() > SAMPLE_RATE // 2, f"Audio too short: {audio.numel()} samples"
|
||||
|
||||
duration_s = audio.shape[0] / SAMPLE_RATE
|
||||
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
|
||||
|
||||
peak = float(torch.max(torch.abs(audio)).item()) if audio.numel() > 0 else 0.0
|
||||
assert peak > 0.01, "Generated audio appears to be silence"
|
||||
|
||||
audio_np = audio.numpy()
|
||||
rms = float(np.sqrt(np.mean(np.square(audio_np)))) if audio_np.size else 0.0
|
||||
assert rms > 1e-4, "Generated audio RMS too low"
|
||||
@@ -232,24 +232,6 @@ def test_stage_configs_path_field():
|
||||
assert args.stage_configs_path == "/some/path.yaml"
|
||||
|
||||
|
||||
def test_voxcpm_model_arch_injects_model_type_override(mocker):
|
||||
"""Ensure VoxCPM model_arch injects hf_overrides for config resolution."""
|
||||
mocker.patch.object(OmniEngineArgs, "_ensure_omni_models_registered", return_value=True)
|
||||
mocker.patch.object(OmniEngineArgs, "_patch_empty_hf_config")
|
||||
mocker.patch.object(EngineArgs, "create_model_config", return_value=Mock())
|
||||
mocker.patch.object(OmniModelConfig, "from_vllm_model_config", return_value=Mock())
|
||||
|
||||
args = OmniEngineArgs(
|
||||
model="OpenBMB/VoxCPM1.5",
|
||||
model_arch="VoxCPMForConditionalGeneration",
|
||||
)
|
||||
args.create_model_config()
|
||||
|
||||
assert args.hf_overrides["architectures"] == ["VoxCPMForConditionalGeneration"]
|
||||
assert args.hf_overrides["model_type"] == "voxcpm"
|
||||
args._patch_empty_hf_config.assert_called_once_with("voxcpm")
|
||||
|
||||
|
||||
def test_strip_single_engine_args():
|
||||
"""_strip_single_engine_args should remove EngineArgs fields but keep omni fields."""
|
||||
kwargs = {
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""UTs for VoxCPM OpenAI speech serving behavior."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def voxcpm_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM1.5")
|
||||
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
mock_engine_client.tts_batch_max_items = 32
|
||||
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(
|
||||
model_stage="latent_generator",
|
||||
model_arch="VoxCPMForConditionalGeneration",
|
||||
),
|
||||
tts_args={},
|
||||
),
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(model_stage="vae"),
|
||||
tts_args={},
|
||||
),
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
return OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def voxcpm2_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM2")
|
||||
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
mock_engine_client.tts_batch_max_items = 32
|
||||
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(
|
||||
model_stage="latent_generator",
|
||||
model_arch="VoxCPM2TalkerForConditionalGeneration",
|
||||
),
|
||||
tts_args={},
|
||||
),
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(model_stage="vae"),
|
||||
tts_args={},
|
||||
),
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
return OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
class TestVoxCPMServing:
|
||||
def test_voxcpm_model_type_detection(self, voxcpm_server):
|
||||
assert voxcpm_server._tts_model_type == "voxcpm"
|
||||
assert voxcpm_server._is_tts is True
|
||||
assert voxcpm_server.supported_speakers == set()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_kwargs", "expected_substring"),
|
||||
[
|
||||
({"voice": "alice"}, "voice"),
|
||||
({"instructions": "whisper"}, "instructions"),
|
||||
({"language": "en"}, "language"),
|
||||
({"task_type": "CustomVoice"}, "plain tts"),
|
||||
({"x_vector_only_mode": True}, "x_vector_only_mode"),
|
||||
({"speaker_embedding": [0.1, 0.2]}, "speaker_embedding"),
|
||||
({"initial_codec_chunk_frames": 4}, "initial_codec_chunk_frames"),
|
||||
({"ref_text": "reference"}, "ref_audio"),
|
||||
],
|
||||
)
|
||||
def test_validate_voxcpm_rejects_unsupported_fields(self, voxcpm_server, request_kwargs, expected_substring):
|
||||
request = OpenAICreateSpeechRequest(input="hello voxcpm", **request_kwargs)
|
||||
error = voxcpm_server._validate_voxcpm_request(request)
|
||||
assert error is not None
|
||||
assert expected_substring in error.lower()
|
||||
|
||||
def test_validate_voxcpm_accepts_plain_tts_request(self, voxcpm_server):
|
||||
request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=256)
|
||||
assert voxcpm_server._validate_voxcpm_request(request) is None
|
||||
|
||||
def test_validate_voxcpm_accepts_voice_clone_request(self, voxcpm_server):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="clone this voice",
|
||||
ref_audio="data:audio/wav;base64,QUJD",
|
||||
ref_text="reference transcript",
|
||||
max_new_tokens=256,
|
||||
)
|
||||
assert voxcpm_server._validate_voxcpm_request(request) is None
|
||||
|
||||
def test_prepare_speech_generation_voxcpm_text_only(self, voxcpm_server):
|
||||
request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=321)
|
||||
|
||||
request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request))
|
||||
|
||||
assert request_id.startswith("speech-")
|
||||
assert generator == "generator"
|
||||
assert tts_params == {
|
||||
"text": ["hello voxcpm"],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [321],
|
||||
}
|
||||
|
||||
voxcpm_server.engine_client.generate.assert_called_once()
|
||||
call = voxcpm_server.engine_client.generate.call_args
|
||||
assert call.kwargs["prompt"] == {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": tts_params,
|
||||
"type": "token",
|
||||
}
|
||||
assert call.kwargs["output_modalities"] == ["audio"]
|
||||
|
||||
def test_prepare_speech_generation_voxcpm_voice_clone_resolves_ref_audio(self, voxcpm_server):
|
||||
voxcpm_server._resolve_ref_audio = AsyncMock(return_value=([0.1, -0.1, 0.2], 16000))
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="clone this voice",
|
||||
ref_audio="data:audio/wav;base64,QUJD",
|
||||
ref_text="reference transcript",
|
||||
max_new_tokens=512,
|
||||
)
|
||||
|
||||
request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request))
|
||||
|
||||
assert request_id.startswith("speech-")
|
||||
assert generator == "generator"
|
||||
assert tts_params == {
|
||||
"text": ["clone this voice"],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [512],
|
||||
"ref_text": ["reference transcript"],
|
||||
"ref_audio": [[[0.1, -0.1, 0.2], 16000]],
|
||||
}
|
||||
|
||||
voxcpm_server._resolve_ref_audio.assert_awaited_once_with("data:audio/wav;base64,QUJD")
|
||||
call = voxcpm_server.engine_client.generate.call_args
|
||||
assert call.kwargs["prompt"] == {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": tts_params,
|
||||
"type": "token",
|
||||
}
|
||||
|
||||
|
||||
class TestVoxCPM2Serving:
|
||||
"""Regression tests for VoxCPM2 serving behavior."""
|
||||
|
||||
def test_voxcpm2_model_type_detection(self, voxcpm2_server):
|
||||
assert voxcpm2_server._tts_model_type == "voxcpm2"
|
||||
assert voxcpm2_server._is_tts is True
|
||||
|
||||
def test_voxcpm2_default_voice_is_supported(self, voxcpm2_server):
|
||||
"""VoxCPM2 should accept voice='default' (zero-shot mode, no voice cloning).
|
||||
|
||||
Regression: previously returned 400 Invalid voice 'default'.
|
||||
"""
|
||||
assert "default" in voxcpm2_server.supported_speakers
|
||||
|
||||
def test_voxcpm2_inline_ref_audio_does_not_set_voice_name(self, voxcpm2_server):
|
||||
"""Inline ref_audio must not populate voice_name in additional_information.
|
||||
|
||||
Regression: the speaker cache keys on voice_name, so two requests with
|
||||
the same voice but different inline ref_audio would collide — the second
|
||||
request reused stale cached features from the first, causing a prefill
|
||||
length mismatch and engine crash.
|
||||
"""
|
||||
mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"}
|
||||
voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt)
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="test",
|
||||
voice="default",
|
||||
ref_audio="data:audio/wav;base64,QUJD",
|
||||
)
|
||||
|
||||
asyncio.run(voxcpm2_server._prepare_speech_generation(request))
|
||||
|
||||
additional = mock_prompt["additional_information"]
|
||||
assert "voice_name" not in additional, (
|
||||
"voice_name must not be set for inline ref_audio — "
|
||||
"it causes speaker cache collisions across different audio clips"
|
||||
)
|
||||
|
||||
def test_voxcpm2_uploaded_voice_sets_voice_name(self, voxcpm2_server):
|
||||
"""Uploaded voice without inline ref_audio must populate voice_name.
|
||||
|
||||
This ensures the speaker cache still works for the intended use case:
|
||||
a voice uploaded once and referenced by name in subsequent requests.
|
||||
"""
|
||||
mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"}
|
||||
voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt)
|
||||
voxcpm2_server.uploaded_speakers["my_voice"] = {
|
||||
"name": "my_voice",
|
||||
"file_path": "/tmp/fake.wav",
|
||||
"created_at": 12345,
|
||||
}
|
||||
voxcpm2_server.supported_speakers.add("my_voice")
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="test",
|
||||
voice="my_voice",
|
||||
)
|
||||
|
||||
asyncio.run(voxcpm2_server._prepare_speech_generation(request))
|
||||
|
||||
additional = mock_prompt["additional_information"]
|
||||
assert additional.get("voice_name") == "my_voice"
|
||||
assert "voice_created_at" in additional
|
||||
|
||||
def test_voxcpm2_uploaded_voice_with_inline_override_does_not_set_voice_name(self, voxcpm2_server):
|
||||
"""Uploaded voice with inline ref_audio override must not populate voice_name.
|
||||
|
||||
Prevents cache poisoning: an attacker could send an uploaded voice name
|
||||
with crafted ref_audio, caching malicious features under the uploaded
|
||||
voice's name for subsequent legitimate requests.
|
||||
"""
|
||||
mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"}
|
||||
voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt)
|
||||
voxcpm2_server.uploaded_speakers["my_voice"] = {
|
||||
"name": "my_voice",
|
||||
"file_path": "/tmp/fake.wav",
|
||||
"created_at": 12345,
|
||||
}
|
||||
voxcpm2_server.supported_speakers.add("my_voice")
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="test",
|
||||
voice="my_voice",
|
||||
ref_audio="data:audio/wav;base64,QUJD",
|
||||
)
|
||||
|
||||
asyncio.run(voxcpm2_server._prepare_speech_generation(request))
|
||||
|
||||
additional = mock_prompt["additional_information"]
|
||||
assert "voice_name" not in additional, (
|
||||
"voice_name must not be set when inline ref_audio overrides an "
|
||||
"uploaded voice — it would poison the cache for future requests"
|
||||
)
|
||||
@@ -313,39 +313,6 @@ class TestResolveModelConfigPath:
|
||||
assert result is not None
|
||||
assert "glm_image.yaml" in result
|
||||
|
||||
def test_voxcpm_transformers_format_resolution(self, mocker: MockerFixture):
|
||||
"""Test VoxCPM transformers config resolves to the voxcpm stage config."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.get_config",
|
||||
side_effect=ValueError("missing transformers config"),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.file_or_path_exists",
|
||||
side_effect=lambda _model, filename, revision=None: filename == "config.json",
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.get_hf_file_to_dict",
|
||||
return_value={"model_type": "voxcpm"},
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.current_omni_platform.get_default_stage_config_path",
|
||||
return_value="vllm_omni/deploy",
|
||||
)
|
||||
|
||||
original_exists = os.path.exists
|
||||
|
||||
def mock_exists(path):
|
||||
if "voxcpm.yaml" in str(path):
|
||||
return True
|
||||
return original_exists(path)
|
||||
|
||||
mocker.patch("os.path.exists", side_effect=mock_exists)
|
||||
|
||||
result = resolve_model_config_path("OpenBMB/VoxCPM1.5")
|
||||
|
||||
assert result is not None
|
||||
assert "voxcpm.yaml" in result
|
||||
|
||||
|
||||
class TestLoadAndResolveStageConfigs:
|
||||
def test_load_and_resolve_with_kwargs(self):
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""UTs for VoxCPM async-chunk stage input processing."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.stage_input_processors.voxcpm import (
|
||||
_VOXCPM_LATENT_MAGIC,
|
||||
_coerce_finished_flag,
|
||||
latent2vae_async_chunk,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _request(*, finished):
|
||||
return SimpleNamespace(is_finished=lambda: finished)
|
||||
|
||||
|
||||
def _decode_serialized_latent(codes: list[int]) -> torch.Tensor:
|
||||
assert codes[0] == _VOXCPM_LATENT_MAGIC
|
||||
latent_dim = codes[1]
|
||||
time_dim = codes[2]
|
||||
payload = torch.tensor(codes[3:], dtype=torch.int32).to(torch.uint16)
|
||||
return payload.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
(False, False),
|
||||
(True, True),
|
||||
(torch.tensor(False), False),
|
||||
(torch.tensor(True), True),
|
||||
([torch.tensor(True)], True),
|
||||
(([True],), True),
|
||||
([], False),
|
||||
],
|
||||
)
|
||||
def test_coerce_finished_flag(value, expected):
|
||||
assert _coerce_finished_flag(value) is expected
|
||||
|
||||
|
||||
def test_latent2vae_async_chunk_serializes_latent_payload():
|
||||
latent = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
|
||||
payload = latent2vae_async_chunk(
|
||||
transfer_manager=None,
|
||||
pooling_output={"latent_audio_feat": latent},
|
||||
request=_request(finished=False),
|
||||
is_finished=torch.tensor(False),
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert torch.equal(payload.meta.finished, torch.tensor(False, dtype=torch.bool))
|
||||
recovered = _decode_serialized_latent(payload.codes.audio.tolist())
|
||||
torch.testing.assert_close(recovered, latent.to(torch.bfloat16).to(torch.float32).unsqueeze(0))
|
||||
|
||||
|
||||
def test_latent2vae_async_chunk_returns_terminal_marker_without_latent():
|
||||
payload = latent2vae_async_chunk(
|
||||
transfer_manager=None,
|
||||
pooling_output=None,
|
||||
request=_request(finished=[torch.tensor(True)]),
|
||||
is_finished=False,
|
||||
)
|
||||
|
||||
assert payload.codes.audio.tolist() == []
|
||||
assert torch.equal(payload.meta.finished, torch.tensor(True, dtype=torch.bool))
|
||||
|
||||
|
||||
def test_latent2vae_async_chunk_returns_none_for_nonterminal_empty_chunk():
|
||||
payload = latent2vae_async_chunk(
|
||||
transfer_manager=None,
|
||||
pooling_output={"latent_audio_feat": torch.zeros((0,), dtype=torch.float32)},
|
||||
request=_request(finished=False),
|
||||
is_finished=False,
|
||||
)
|
||||
|
||||
assert payload is None
|
||||
@@ -1,69 +0,0 @@
|
||||
# VoxCPM two-stage (latent → VAE) without async_chunk: one-shot latent then decode.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
model_stage: latent_generator
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
# Optional persistent HF-compatible config dir for native VoxCPM models.
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.7
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
stop_token_ids: [2]
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.0
|
||||
final_output: false
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: float32
|
||||
model_stage: vae
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.15
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 4096
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 1
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
@@ -22,7 +22,6 @@ _ARCH_TO_MODEL_TYPE: dict[str, str] = {
|
||||
"CosyVoice3Model": "cosyvoice3",
|
||||
"OmniVoiceModel": "omnivoice",
|
||||
"VoxCPM2TalkerForConditionalGeneration": "voxcpm2",
|
||||
"VoxCPMForConditionalGeneration": "voxcpm",
|
||||
}
|
||||
|
||||
# Maps model architecture names to tokenizer subfolder paths within HF repos.
|
||||
|
||||
@@ -65,7 +65,6 @@ _FISH_TTS_MODEL_STAGES = {"fish_speech_slow_ar"}
|
||||
_COSYVOICE3_TTS_MODEL_STAGES = {"cosyvoice3_talker"}
|
||||
_OMNIVOICE_TTS_MODEL_STAGES = {"omnivoice_generator"}
|
||||
_COVO_AUDIO_MODEL_STAGES = {"fused_thinker_talker"}
|
||||
_VOXCPM_TTS_MODEL_STAGES = {"latent_generator", "vae"}
|
||||
_VOXCPM2_TTS_MODEL_STAGES = {"latent_generator"}
|
||||
_MING_TTS_MODEL_STAGES = {"ming_tts"}
|
||||
_MOSS_TTS_MODEL_STAGES = {"moss_tts_nano"}
|
||||
@@ -76,7 +75,6 @@ _TTS_MODEL_STAGES: set[str] = (
|
||||
| _COSYVOICE3_TTS_MODEL_STAGES
|
||||
| _OMNIVOICE_TTS_MODEL_STAGES
|
||||
| _COVO_AUDIO_MODEL_STAGES
|
||||
| _VOXCPM_TTS_MODEL_STAGES
|
||||
| _VOXCPM2_TTS_MODEL_STAGES
|
||||
| _MING_TTS_MODEL_STAGES
|
||||
| _MOSS_TTS_MODEL_STAGES
|
||||
@@ -481,8 +479,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
model_arch = getattr(self._tts_stage.engine_args, "model_arch", None)
|
||||
if model_arch == "VoxCPM2TalkerForConditionalGeneration":
|
||||
return "voxcpm2"
|
||||
if model_arch == "VoxCPMForConditionalGeneration":
|
||||
return "voxcpm"
|
||||
if model_stage in _QWEN3_TTS_MODEL_STAGES:
|
||||
return "qwen3_tts"
|
||||
if model_stage in _VOXTRAL_TTS_MODEL_STAGES:
|
||||
@@ -497,12 +493,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
model_arch = getattr(self._tts_stage.engine_args, "model_arch", None)
|
||||
if model_arch and "CovoAudio" in model_arch:
|
||||
return "covo_audio"
|
||||
if model_stage in (_VOXCPM_TTS_MODEL_STAGES | _VOXCPM2_TTS_MODEL_STAGES):
|
||||
has_vae_stage = any(
|
||||
getattr(getattr(stage, "engine_args", None), "model_stage", None) == "vae"
|
||||
for stage in self.engine_client.stage_configs
|
||||
)
|
||||
return "voxcpm" if has_vae_stage or model_stage == "vae" else "voxcpm2"
|
||||
if model_stage in _VOXCPM2_TTS_MODEL_STAGES:
|
||||
return "voxcpm2"
|
||||
if model_stage in _MING_TTS_MODEL_STAGES:
|
||||
return "ming_flash_omni_tts"
|
||||
if model_stage in _MOSS_TTS_MODEL_STAGES:
|
||||
@@ -536,8 +528,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
# is no static speaker list to surface here.
|
||||
return set()
|
||||
try:
|
||||
if self._tts_model_type == "voxcpm":
|
||||
return set()
|
||||
if self._tts_model_type == "voxcpm2":
|
||||
return {"default"}
|
||||
if self._tts_model_type == "voxtral_tts":
|
||||
@@ -599,8 +589,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
def _estimate_prompt_len(self, tts_params: dict[str, Any]) -> int:
|
||||
"""Estimate prompt length so the placeholder matches model-side embeddings."""
|
||||
try:
|
||||
if self._tts_model_type == "voxcpm":
|
||||
return 1
|
||||
from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import (
|
||||
Qwen3TTSTalkerForConditionalGeneration,
|
||||
)
|
||||
@@ -1136,8 +1124,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
return self._validate_fish_tts_request(request)
|
||||
if self._tts_model_type == "cosyvoice3":
|
||||
return self._validate_cosyvoice3_request(request)
|
||||
if self._tts_model_type == "voxcpm":
|
||||
return self._validate_voxcpm_request(request)
|
||||
if self._tts_model_type == "voxcpm2":
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
@@ -1242,43 +1228,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
return None
|
||||
|
||||
def _validate_voxcpm_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate VoxCPM request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
if request.voice is not None:
|
||||
return "'voice' is not supported for VoxCPM"
|
||||
if request.instructions is not None:
|
||||
return "'instructions' is not supported for VoxCPM"
|
||||
if request.language is not None:
|
||||
return "'language' is not supported for VoxCPM"
|
||||
if request.task_type not in (None, "Base"):
|
||||
return "VoxCPM only supports plain TTS or voice cloning with ref_audio/ref_text"
|
||||
if request.x_vector_only_mode is not None:
|
||||
return "'x_vector_only_mode' is not supported for VoxCPM"
|
||||
if request.speaker_embedding is not None:
|
||||
return "'speaker_embedding' is not supported for VoxCPM"
|
||||
if request.initial_codec_chunk_frames is not None:
|
||||
return "'initial_codec_chunk_frames' is not supported for VoxCPM"
|
||||
|
||||
if request.ref_audio is not None:
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "Voice cloning requires 'ref_text' (transcript of the reference audio)"
|
||||
elif request.ref_text is not None:
|
||||
return "'ref_text' requires 'ref_audio' for VoxCPM voice cloning"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
def _validate_qwen_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate Qwen TTS request parameters. Returns error message or None."""
|
||||
# Infer Base task when ref_audio or ref_text is provided without explicit task_type.
|
||||
@@ -1643,18 +1592,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
Processes each parameter if present, skips if not.
|
||||
Values are wrapped in lists as required by the model.
|
||||
"""
|
||||
if self._tts_model_type == "voxcpm":
|
||||
params: dict[str, Any] = {
|
||||
"text": [request.input],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [request.max_new_tokens or 4096],
|
||||
}
|
||||
if request.ref_text is not None:
|
||||
params["ref_text"] = [request.ref_text]
|
||||
return params
|
||||
|
||||
params: dict[str, Any] = {}
|
||||
|
||||
# Text content (always required)
|
||||
@@ -2169,8 +2106,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
model_type = "voxtral_tts"
|
||||
elif self._tts_model_type == "cosyvoice3":
|
||||
model_type = "cosyvoice3"
|
||||
elif self._tts_model_type == "voxcpm":
|
||||
model_type = "voxcpm"
|
||||
elif self._tts_model_type == "voxcpm2":
|
||||
model_type = "voxcpm2"
|
||||
elif self._tts_model_type == "ming_flash_omni_tts":
|
||||
|
||||
@@ -150,12 +150,6 @@ _OMNI_MODELS = {
|
||||
"fish_speech_dac_decoder",
|
||||
"FishSpeechDACDecoder",
|
||||
),
|
||||
## VoxCPM
|
||||
"VoxCPMForConditionalGeneration": (
|
||||
"voxcpm",
|
||||
"voxcpm",
|
||||
"VoxCPMForConditionalGeneration",
|
||||
),
|
||||
## VoxCPM2
|
||||
"VoxCPM2TalkerForConditionalGeneration": (
|
||||
"voxcpm2",
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
from .voxcpm import VoxCPMForConditionalGeneration
|
||||
|
||||
__all__ = [
|
||||
"VoxCPMForConditionalGeneration",
|
||||
]
|
||||
@@ -1,886 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import warnings
|
||||
import wave
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from tqdm import tqdm
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
from .voxcpm_loader import (
|
||||
_build_prompt_cache_with_soundfile,
|
||||
_device_to_string,
|
||||
_force_cuda_available_for_npu,
|
||||
_import_voxcpm_audio_vae_classes,
|
||||
_import_voxcpm_base_model_class,
|
||||
_is_torchcodec_load_error,
|
||||
_normalize_dtype_name,
|
||||
_prepare_runtime_model_dir,
|
||||
_resolve_runtime_device,
|
||||
)
|
||||
from .voxcpm_runtime_utils import resolve_voxcpm_model_dir
|
||||
from .voxcpm_stage_wrappers import _DirectVoxCPMAudioVAE, _DirectVoxCPMLatentGenerator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_VOXCPM_LATENT_MAGIC = 131071
|
||||
|
||||
|
||||
def _make_voxcpm_model_for_omni(base: type[Any]) -> type[Any]:
|
||||
"""Subclass upstream VoxCPMModel: local ``_inference`` + ``latents_only`` prompt-cache generation."""
|
||||
|
||||
from voxcpm.model.utils import get_dtype
|
||||
|
||||
class VoxCPMModelForOmni(base):
|
||||
@torch.inference_mode()
|
||||
def build_prompt_cache(self, *args: Any, **kwargs: Any):
|
||||
try:
|
||||
return super().build_prompt_cache(*args, **kwargs)
|
||||
except (ImportError, ModuleNotFoundError, RuntimeError) as exc:
|
||||
if not _is_torchcodec_load_error(exc):
|
||||
raise
|
||||
return _build_prompt_cache_with_soundfile(self, *args, **kwargs)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _inference(
|
||||
self,
|
||||
text: torch.Tensor,
|
||||
text_mask: torch.Tensor,
|
||||
feat: torch.Tensor,
|
||||
feat_mask: torch.Tensor,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 3,
|
||||
) -> Generator[tuple[torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]:
|
||||
B, _, _, _ = feat.shape
|
||||
|
||||
feat_embed = self.feat_encoder(feat)
|
||||
feat_embed = self.enc_to_lm_proj(feat_embed)
|
||||
|
||||
scale_emb = self.config.lm_config.scale_emb if self.config.lm_config.use_mup else 1.0
|
||||
text_embed = self.base_lm.embed_tokens(text) * scale_emb
|
||||
combined_embed = text_mask.unsqueeze(-1) * text_embed + feat_mask.unsqueeze(-1) * feat_embed
|
||||
|
||||
prefix_feat_cond = feat[:, -1, ...]
|
||||
pred_feat_seq: list[torch.Tensor] = []
|
||||
|
||||
audio_patch_count = int(feat_mask.sum().item())
|
||||
if audio_patch_count > 0:
|
||||
context_len = min(streaming_prefix_len - 1, audio_patch_count)
|
||||
prompt_context_patches = list(feat[:, -context_len:, :, :].split(1, dim=1))
|
||||
pred_feat_seq = prompt_context_patches + pred_feat_seq
|
||||
|
||||
enc_outputs, kv_cache_tuple = self.base_lm(
|
||||
inputs_embeds=combined_embed,
|
||||
is_causal=True,
|
||||
)
|
||||
self.base_lm.kv_cache.fill_caches(kv_cache_tuple)
|
||||
|
||||
enc_outputs = self.fsq_layer(enc_outputs) * feat_mask.unsqueeze(-1) + enc_outputs * text_mask.unsqueeze(-1)
|
||||
lm_hidden = enc_outputs[:, -1, :]
|
||||
|
||||
residual_enc_outputs, residual_kv_cache_tuple = self.residual_lm(
|
||||
inputs_embeds=enc_outputs + feat_mask.unsqueeze(-1) * feat_embed,
|
||||
is_causal=True,
|
||||
)
|
||||
self.residual_lm.kv_cache.fill_caches(residual_kv_cache_tuple)
|
||||
residual_hidden = residual_enc_outputs[:, -1, :]
|
||||
|
||||
for step_idx in tqdm(range(max_len)):
|
||||
dit_hidden = self.lm_to_dit_proj(lm_hidden) + self.res_to_dit_proj(residual_hidden)
|
||||
pred_feat = self.feat_decoder(
|
||||
mu=dit_hidden,
|
||||
patch_size=self.patch_size,
|
||||
cond=prefix_feat_cond.transpose(1, 2).contiguous(),
|
||||
n_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
).transpose(1, 2)
|
||||
|
||||
curr_embed = self.enc_to_lm_proj(self.feat_encoder(pred_feat.unsqueeze(1)))
|
||||
pred_feat_seq.append(pred_feat.unsqueeze(1))
|
||||
prefix_feat_cond = pred_feat
|
||||
|
||||
if streaming:
|
||||
pred_feat_chunk = torch.cat(pred_feat_seq[-streaming_prefix_len:], dim=1)
|
||||
feat_pred = rearrange(pred_feat_chunk, "b t p d -> b d (t p)", b=B, p=self.patch_size)
|
||||
yield feat_pred, pred_feat_seq
|
||||
|
||||
stop_flag = self.stop_head(self.stop_actn(self.stop_proj(lm_hidden))).argmax(dim=-1)[0].cpu().item()
|
||||
if step_idx > min_len and stop_flag == 1:
|
||||
break
|
||||
|
||||
lm_hidden = self.base_lm.forward_step(
|
||||
curr_embed[:, 0, :],
|
||||
torch.tensor([self.base_lm.kv_cache.step()], device=curr_embed.device),
|
||||
).clone()
|
||||
lm_hidden = self.fsq_layer(lm_hidden)
|
||||
residual_hidden = self.residual_lm.forward_step(
|
||||
lm_hidden + curr_embed[:, 0, :],
|
||||
torch.tensor([self.residual_lm.kv_cache.step()], device=curr_embed.device),
|
||||
).clone()
|
||||
|
||||
if not streaming:
|
||||
pred_feat_seq_cat = torch.cat(pred_feat_seq, dim=1)
|
||||
feat_pred = rearrange(pred_feat_seq_cat, "b t p d -> b d (t p)", b=B, p=self.patch_size)
|
||||
yield feat_pred, pred_feat_seq_cat.squeeze(0).cpu()
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate_latents_with_prompt_cache(
|
||||
self,
|
||||
target_text: str,
|
||||
prompt_cache: dict,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming_prefix_len: int = 3,
|
||||
) -> tuple[None, torch.Tensor, torch.Tensor]:
|
||||
return next(
|
||||
self._generate_with_prompt_cache(
|
||||
target_text=target_text,
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming=False,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
latents_only=True,
|
||||
)
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate_latents_with_prompt_cache_streaming(
|
||||
self,
|
||||
target_text: str,
|
||||
prompt_cache: dict,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming_prefix_len: int = 3,
|
||||
) -> Generator[tuple[None, torch.Tensor, torch.Tensor], None, None]:
|
||||
return self._generate_with_prompt_cache(
|
||||
target_text=target_text,
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming=True,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
latents_only=True,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _generate_with_prompt_cache(
|
||||
self,
|
||||
target_text: str,
|
||||
prompt_cache: dict,
|
||||
min_len: int = 2,
|
||||
max_len: int = 2000,
|
||||
inference_timesteps: int = 10,
|
||||
cfg_value: float = 2.0,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 3,
|
||||
latents_only: bool = False,
|
||||
) -> Generator[tuple[torch.Tensor | None, torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]:
|
||||
if retry_badcase and streaming:
|
||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||
retry_badcase = False
|
||||
if prompt_cache is None:
|
||||
prompt_audio_feat = torch.empty((0, self.patch_size, self.audio_vae.latent_dim), dtype=torch.float32)
|
||||
text = target_text
|
||||
else:
|
||||
prompt_audio_feat = prompt_cache["audio_feat"]
|
||||
prompt_text = prompt_cache["prompt_text"]
|
||||
text = prompt_text + target_text
|
||||
|
||||
text_token = torch.LongTensor(self.text_tokenizer(text))
|
||||
text_token = torch.cat(
|
||||
[
|
||||
text_token,
|
||||
torch.tensor([self.audio_start_token], dtype=torch.int32, device=text_token.device),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
target_text_token = torch.LongTensor(self.text_tokenizer(target_text))
|
||||
|
||||
audio_length = prompt_audio_feat.size(0)
|
||||
text_length = text_token.shape[0]
|
||||
text_pad_token = torch.zeros(audio_length, dtype=torch.int32, device=text_token.device)
|
||||
audio_pad_feat = torch.zeros(
|
||||
(text_token.shape[0], self.patch_size, self.audio_vae.latent_dim),
|
||||
dtype=torch.float32,
|
||||
device=text_token.device,
|
||||
)
|
||||
text_token = torch.cat([text_token, text_pad_token])
|
||||
audio_feat = torch.cat([audio_pad_feat, prompt_audio_feat], dim=0)
|
||||
text_mask = (
|
||||
torch.cat([torch.ones(text_length), torch.zeros(audio_length)]).type(torch.int32).to(text_token.device)
|
||||
)
|
||||
audio_mask = (
|
||||
torch.cat([torch.zeros(text_length), torch.ones(audio_length)]).type(torch.int32).to(text_token.device)
|
||||
)
|
||||
|
||||
text_token = text_token.unsqueeze(0).to(self.device)
|
||||
text_mask = text_mask.unsqueeze(0).to(self.device)
|
||||
audio_feat = audio_feat.unsqueeze(0).to(self.device).to(get_dtype(self.config.dtype))
|
||||
audio_mask = audio_mask.unsqueeze(0).to(self.device)
|
||||
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
retry_badcase_times = 0
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
audio_feat,
|
||||
audio_mask,
|
||||
min_len=min_len,
|
||||
max_len=min(int(target_text_length * retry_badcase_ratio_threshold + 10), max_len),
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
streaming=streaming,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
)
|
||||
if streaming:
|
||||
patch_len = self.patch_size * self.chunk_size
|
||||
for latent_pred, pred_audio_feat in inference_result:
|
||||
if latents_only:
|
||||
decode_audio = None
|
||||
yield (decode_audio, target_text_token, latent_pred)
|
||||
else:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
break
|
||||
|
||||
latent_pred, pred_audio_feat = next(inference_result)
|
||||
if retry_badcase and pred_audio_feat.shape[0] >= target_text_length * retry_badcase_ratio_threshold:
|
||||
ratio = pred_audio_feat.shape[0] / target_text_length
|
||||
print(f" Badcase detected, audio_text_ratio={ratio}, retrying...", file=sys.stderr)
|
||||
retry_badcase_times += 1
|
||||
continue
|
||||
break
|
||||
|
||||
if not streaming:
|
||||
if latents_only:
|
||||
decode_audio = None
|
||||
else:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
patch_len = self.patch_size * self.chunk_size
|
||||
if audio_mask.sum().item() > 0:
|
||||
decode_audio = decode_audio[..., patch_len * (streaming_prefix_len - 1) :].squeeze(1).cpu()
|
||||
else:
|
||||
decode_audio = decode_audio[..., :].squeeze(1).cpu()
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
|
||||
VoxCPMModelForOmni.__name__ = "VoxCPMModelForOmni"
|
||||
VoxCPMModelForOmni.__qualname__ = "VoxCPMModelForOmni"
|
||||
return VoxCPMModelForOmni
|
||||
|
||||
|
||||
def _import_voxcpm_model_class() -> type[Any]:
|
||||
base = _import_voxcpm_base_model_class()
|
||||
return _make_voxcpm_model_for_omni(base)
|
||||
|
||||
|
||||
def _load_native_voxcpm_model(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: str | None,
|
||||
):
|
||||
VoxCPMModel = _import_voxcpm_model_class()
|
||||
model_dir = resolve_voxcpm_model_dir(model_path)
|
||||
runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype=dtype)
|
||||
|
||||
if device.type == "npu" and hasattr(torch, "npu"):
|
||||
torch.npu.set_device(device)
|
||||
|
||||
with _force_cuda_available_for_npu(device):
|
||||
return VoxCPMModel.from_local(
|
||||
runtime_model_path,
|
||||
optimize=device.type == "cuda",
|
||||
)
|
||||
|
||||
|
||||
def _load_native_voxcpm_latent_generator(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: str | None,
|
||||
) -> _DirectVoxCPMLatentGenerator:
|
||||
return _DirectVoxCPMLatentGenerator(_load_native_voxcpm_model(model_path, device=device, dtype=dtype))
|
||||
|
||||
|
||||
def _load_native_voxcpm_audio_vae(
|
||||
model_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
) -> _DirectVoxCPMAudioVAE:
|
||||
AudioVAE, AudioVAEConfig = _import_voxcpm_audio_vae_classes()
|
||||
model_dir = resolve_voxcpm_model_dir(model_path)
|
||||
runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype="float32")
|
||||
config_dict = json.loads((Path(runtime_model_path) / "config.json").read_text())
|
||||
audio_vae_config = config_dict.get("audio_vae_config")
|
||||
audio_vae = AudioVAE(config=AudioVAEConfig(**audio_vae_config)) if audio_vae_config is not None else AudioVAE()
|
||||
|
||||
state_dict = torch.load(
|
||||
Path(runtime_model_path) / "audiovae.pth",
|
||||
map_location="cpu",
|
||||
weights_only=True,
|
||||
)["state_dict"]
|
||||
audio_vae.load_state_dict(state_dict, strict=True)
|
||||
audio_vae = audio_vae.to(device=device, dtype=torch.float32).eval()
|
||||
if device.type == "npu" and hasattr(torch, "npu"):
|
||||
torch.npu.set_device(device)
|
||||
patch_size = int(config_dict.get("patch_size", 2))
|
||||
return _DirectVoxCPMAudioVAE(audio_vae, patch_size=patch_size)
|
||||
|
||||
|
||||
class VoxCPMForConditionalGeneration(nn.Module):
|
||||
input_modalities = "audio"
|
||||
_LATENT_STAGES = {"latent_generator", "latent", "ar_dit"}
|
||||
_VAE_STAGES = {"vae", "audio_vae"}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
del prefix
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
self.model_stage = getattr(vllm_config.model_config, "model_stage", "latent_generator")
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
self.inject_omni_request_id_into_runtime_info = True
|
||||
self._pipeline = None
|
||||
self._latent_stream_gens: dict[str, Any] = {}
|
||||
self._latent_stream_terminal_pending: dict[str, int] = {}
|
||||
self._latent_stream_completed: set[str] = set()
|
||||
self._next_local_stream_key = 0
|
||||
self._ar_emit_stop_token = True
|
||||
|
||||
def _runner_hidden_device_dtype(self) -> tuple[torch.device, torch.dtype]:
|
||||
device = _resolve_runtime_device(self.vllm_config)
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
dtype = getattr(model_config, "dtype", torch.float32) if model_config is not None else torch.float32
|
||||
return device, dtype
|
||||
|
||||
def _ensure_model_loaded(self):
|
||||
if self._pipeline is not None:
|
||||
return
|
||||
|
||||
target_device = _resolve_runtime_device(self.vllm_config)
|
||||
model_dtype = getattr(self.vllm_config.model_config, "dtype", None)
|
||||
normalized_dtype = _normalize_dtype_name(model_dtype)
|
||||
if self.model_stage in self._LATENT_STAGES:
|
||||
self._pipeline = _load_native_voxcpm_latent_generator(
|
||||
self.model_path,
|
||||
device=target_device,
|
||||
dtype=normalized_dtype,
|
||||
)
|
||||
elif self.model_stage in self._VAE_STAGES:
|
||||
self._pipeline = _load_native_voxcpm_audio_vae(
|
||||
self.model_path,
|
||||
device=target_device,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported VoxCPM model_stage: {self.model_stage}. "
|
||||
"pure_voxcpm only supports split-stage latent_generator/vae inference."
|
||||
)
|
||||
|
||||
logger.info("Loaded VoxCPM stage '%s' on %s", self.model_stage, _device_to_string(target_device))
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
del weights
|
||||
self._ensure_model_loaded()
|
||||
return set()
|
||||
|
||||
@staticmethod
|
||||
def _extract_val(info: dict[str, Any], key: str, default: Any) -> Any:
|
||||
value = info.get(key, default)
|
||||
if isinstance(value, list):
|
||||
return value[0] if value else default
|
||||
return value
|
||||
|
||||
def _resolve_stream_request_key(self, info: dict[str, Any]) -> str:
|
||||
request_key = info.get("__voxcpm_stream_key")
|
||||
if request_key is not None:
|
||||
return str(request_key)
|
||||
|
||||
request_key = info.get("_omni_req_id")
|
||||
if request_key is not None:
|
||||
request_key = str(request_key)
|
||||
info["__voxcpm_stream_key"] = request_key
|
||||
return request_key
|
||||
|
||||
request_key = f"voxcpm-local-{self._next_local_stream_key}"
|
||||
self._next_local_stream_key += 1
|
||||
info["__voxcpm_stream_key"] = request_key
|
||||
return str(request_key)
|
||||
|
||||
def _recover_latent_from_input_ids(self, input_ids: torch.Tensor | None) -> torch.Tensor | None:
|
||||
if input_ids is None or input_ids.numel() == 0:
|
||||
return None
|
||||
flat_ids = input_ids.detach().reshape(-1).to("cpu")
|
||||
if flat_ids.numel() < 4 or int(flat_ids[0].item()) != _VOXCPM_LATENT_MAGIC:
|
||||
return None
|
||||
latent_dim = int(flat_ids[1].item())
|
||||
time_dim = int(flat_ids[2].item())
|
||||
payload = flat_ids[3:]
|
||||
expected = latent_dim * time_dim
|
||||
if latent_dim <= 0 or time_dim <= 0:
|
||||
raise ValueError(f"Invalid VoxCPM latent header: latent_dim={latent_dim}, time_dim={time_dim}")
|
||||
if int(payload.numel()) != expected:
|
||||
raise ValueError(
|
||||
"Invalid VoxCPM latent payload size: "
|
||||
f"expected={expected}, actual={int(payload.numel())}, "
|
||||
f"latent_dim={latent_dim}, time_dim={time_dim}"
|
||||
)
|
||||
packed = payload.to(dtype=torch.int32).to(torch.uint16)
|
||||
return packed.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim)
|
||||
|
||||
def _maybe_recover_vae_infos(
|
||||
self,
|
||||
infos: list[dict[str, Any]],
|
||||
input_ids: torch.Tensor | None,
|
||||
*,
|
||||
async_chunk: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if not async_chunk:
|
||||
return infos
|
||||
if any(self._extract_val(info, "latent_audio_feat", None) is not None for info in infos):
|
||||
return infos
|
||||
recovered = self._recover_latent_from_input_ids(input_ids)
|
||||
if recovered is None:
|
||||
return infos
|
||||
return [{"latent_audio_feat": recovered}]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_audio_samples(samples: Any) -> np.ndarray:
|
||||
if isinstance(samples, torch.Tensor):
|
||||
return samples.detach().cpu().float().reshape(-1).numpy()
|
||||
return np.asarray(samples, dtype=np.float32).reshape(-1)
|
||||
|
||||
@classmethod
|
||||
def _normalize_ref_audio(cls, ref_audio: Any) -> tuple[np.ndarray, int]:
|
||||
if isinstance(ref_audio, str):
|
||||
raise TypeError("String ref_audio should be handled as a path before waveform normalization.")
|
||||
|
||||
if isinstance(ref_audio, dict):
|
||||
sample_rate = ref_audio.get("sample_rate") or ref_audio.get("sampling_rate") or ref_audio.get("sr")
|
||||
samples = None
|
||||
for key in ("audio", "wav", "samples", "array", "waveform"):
|
||||
if key in ref_audio and ref_audio[key] is not None:
|
||||
samples = ref_audio[key]
|
||||
break
|
||||
if sample_rate is None or samples is None:
|
||||
raise ValueError("ref_audio dict must contain waveform data and sample rate.")
|
||||
return cls._normalize_audio_samples(samples), int(sample_rate)
|
||||
|
||||
if isinstance(ref_audio, (list, tuple)):
|
||||
if len(ref_audio) == 1:
|
||||
return cls._normalize_ref_audio(ref_audio[0])
|
||||
if len(ref_audio) == 2 and np.isscalar(ref_audio[1]):
|
||||
return cls._normalize_audio_samples(ref_audio[0]), int(ref_audio[1])
|
||||
|
||||
raise TypeError(f"Unsupported ref_audio format: {type(ref_audio)!r}")
|
||||
|
||||
@staticmethod
|
||||
def _write_temp_prompt_wav(waveform: np.ndarray, sample_rate: int) -> str:
|
||||
prompt_file = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
|
||||
prompt_file.close()
|
||||
|
||||
wav = np.asarray(waveform, dtype=np.float32).reshape(-1)
|
||||
wav = np.clip(wav, -1.0, 1.0)
|
||||
pcm16 = (wav * 32767.0).astype(np.int16)
|
||||
with wave.open(prompt_file.name, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(int(sample_rate))
|
||||
wav_file.writeframes(pcm16.tobytes())
|
||||
|
||||
return prompt_file.name
|
||||
|
||||
@classmethod
|
||||
def _resolve_prompt_inputs(cls, info: dict[str, Any]) -> tuple[str | None, str | None, str | None]:
|
||||
prompt_text = cls._extract_val(info, "prompt_text", None)
|
||||
prompt_wav_path = cls._extract_val(info, "prompt_wav_path", None)
|
||||
if prompt_wav_path:
|
||||
if prompt_text is None:
|
||||
prompt_text = cls._extract_val(info, "ref_text", None)
|
||||
return prompt_wav_path, prompt_text, None
|
||||
|
||||
ref_audio = cls._extract_val(info, "ref_audio", None)
|
||||
ref_text = cls._extract_val(info, "ref_text", None)
|
||||
if ref_audio is None or ref_text is None:
|
||||
return None, None, None
|
||||
if isinstance(ref_audio, str):
|
||||
return ref_audio, ref_text, None
|
||||
|
||||
waveform, sample_rate = cls._normalize_ref_audio(ref_audio)
|
||||
temp_prompt_wav = cls._write_temp_prompt_wav(waveform, sample_rate)
|
||||
return temp_prompt_wav, ref_text, temp_prompt_wav
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def _get_vocab_size(self) -> int:
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
if model_config is not None:
|
||||
getter = getattr(model_config, "get_vocab_size", None)
|
||||
if callable(getter):
|
||||
try:
|
||||
return int(getter())
|
||||
except Exception:
|
||||
pass
|
||||
hf_config = getattr(model_config, "hf_text_config", None)
|
||||
if hf_config is not None and hasattr(hf_config, "vocab_size"):
|
||||
return int(hf_config.vocab_size)
|
||||
return 32000
|
||||
|
||||
def _make_empty_output(
|
||||
self,
|
||||
*,
|
||||
output_key: str,
|
||||
payload_factory: Callable[[], torch.Tensor],
|
||||
infos: list[dict[str, Any]],
|
||||
sample_rate: int,
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
hidden_rows: int | None = None,
|
||||
) -> OmniOutput:
|
||||
if hidden_rows is None:
|
||||
hidden_rows = len(infos)
|
||||
return OmniOutput(
|
||||
text_hidden_states=torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype),
|
||||
multimodal_outputs={
|
||||
output_key: [payload_factory() for _ in infos],
|
||||
"sr": [torch.tensor(sample_rate, dtype=torch.int32) for _ in infos],
|
||||
},
|
||||
)
|
||||
|
||||
def _finalize_stage_output(
|
||||
self,
|
||||
*,
|
||||
output_key: str,
|
||||
outputs: list[torch.Tensor],
|
||||
sample_rates: list[torch.Tensor],
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
hidden_rows: int | None = None,
|
||||
) -> OmniOutput:
|
||||
multimodal_outputs: dict[str, Any] = {output_key: outputs, "sr": sample_rates}
|
||||
if hidden_rows is not None:
|
||||
text_hidden_states = torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype)
|
||||
elif outputs:
|
||||
outputs_tensor = torch.stack(outputs)
|
||||
text_hidden_states = (
|
||||
outputs_tensor.unsqueeze(-1)
|
||||
if outputs_tensor.ndim == 1
|
||||
else outputs_tensor.reshape(-1, outputs_tensor.shape[-1])
|
||||
)
|
||||
else:
|
||||
text_hidden_states = torch.zeros((0, 1), device=out_device, dtype=out_dtype)
|
||||
text_hidden_states = text_hidden_states.to(device=out_device, dtype=out_dtype)
|
||||
return OmniOutput(
|
||||
text_hidden_states=text_hidden_states,
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
)
|
||||
|
||||
def _forward_vae_stage(
|
||||
self,
|
||||
infos: list[dict[str, Any]],
|
||||
*,
|
||||
sample_rate: int,
|
||||
async_chunk: bool,
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
) -> OmniOutput:
|
||||
if all(self._extract_val(info, "latent_audio_feat", None) is None for info in infos):
|
||||
self._ar_emit_stop_token = True
|
||||
return self._make_empty_output(
|
||||
output_key="model_outputs",
|
||||
payload_factory=lambda: torch.zeros((0,), dtype=torch.float32),
|
||||
infos=infos,
|
||||
sample_rate=sample_rate,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
outputs: list[torch.Tensor] = []
|
||||
sample_rates: list[torch.Tensor] = []
|
||||
for info in infos:
|
||||
latent_audio_feat = self._extract_val(info, "latent_audio_feat", None)
|
||||
audio_tensor = self._pipeline.decode(latent_audio_feat, trim_streaming_patch=async_chunk)
|
||||
outputs.append(audio_tensor.float().cpu())
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
|
||||
self._ar_emit_stop_token = True
|
||||
return self._finalize_stage_output(
|
||||
output_key="model_outputs",
|
||||
outputs=outputs,
|
||||
sample_rates=sample_rates,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
def _forward_latent_stage(
|
||||
self,
|
||||
infos: list[dict[str, Any]],
|
||||
*,
|
||||
sample_rate: int,
|
||||
async_chunk: bool,
|
||||
out_device: torch.device,
|
||||
out_dtype: torch.dtype,
|
||||
hidden_rows: int,
|
||||
) -> OmniOutput:
|
||||
texts = [self._extract_val(info, "text", "") for info in infos]
|
||||
if all(not text for text in texts):
|
||||
self._ar_emit_stop_token = True
|
||||
return self._make_empty_output(
|
||||
output_key="latent_audio_feat",
|
||||
payload_factory=lambda: torch.zeros((0,), dtype=torch.float32),
|
||||
infos=infos,
|
||||
sample_rate=sample_rate,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
hidden_rows=hidden_rows,
|
||||
)
|
||||
|
||||
outputs: list[torch.Tensor] = []
|
||||
sample_rates: list[torch.Tensor] = []
|
||||
last_chunk_flags: list[bool] | None = [] if async_chunk else None
|
||||
payload_finished_flags: list[bool] | None = [] if async_chunk else None
|
||||
for info in infos:
|
||||
text = self._extract_val(info, "text", "")
|
||||
cfg_value = float(self._extract_val(info, "cfg_value", 2.0))
|
||||
inference_timesteps = int(self._extract_val(info, "inference_timesteps", 10))
|
||||
min_len = int(self._extract_val(info, "min_len", 2))
|
||||
max_len = int(self._extract_val(info, "max_len", self._extract_val(info, "max_new_tokens", 4096)))
|
||||
retry_badcase = bool(self._extract_val(info, "retry_badcase", True))
|
||||
retry_badcase_max_times = int(self._extract_val(info, "retry_badcase_max_times", 3))
|
||||
retry_badcase_ratio_threshold = float(self._extract_val(info, "retry_badcase_ratio_threshold", 6.0))
|
||||
streaming_prefix_len = int(self._extract_val(info, "streaming_prefix_len", 3))
|
||||
|
||||
request_key = self._resolve_stream_request_key(info)
|
||||
created_temp: str | None = None
|
||||
|
||||
if async_chunk:
|
||||
terminal_pending = self._latent_stream_terminal_pending.get(request_key, 0)
|
||||
if terminal_pending > 0:
|
||||
outputs.append(torch.zeros((0,), dtype=torch.float32))
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(True)
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(terminal_pending == 1)
|
||||
if terminal_pending == 1:
|
||||
self._latent_stream_terminal_pending.pop(request_key, None)
|
||||
else:
|
||||
self._latent_stream_terminal_pending[request_key] = terminal_pending - 1
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
continue
|
||||
|
||||
if request_key in self._latent_stream_completed:
|
||||
outputs.append(torch.zeros((0,), dtype=torch.float32))
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(True)
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(False)
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
continue
|
||||
|
||||
if request_key not in self._latent_stream_gens:
|
||||
prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info)
|
||||
created_temp = temp_prompt_wav
|
||||
self._latent_stream_gens[request_key] = self._pipeline.iter_latent_chunks_streaming(
|
||||
text=text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
prompt_text=prompt_text,
|
||||
cfg_value=cfg_value,
|
||||
inference_timesteps=inference_timesteps,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
retry_badcase=False,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
)
|
||||
generator = self._latent_stream_gens[request_key]
|
||||
try:
|
||||
chunk_latent, is_last = next(generator)
|
||||
except StopIteration:
|
||||
self._latent_stream_gens.pop(request_key, None)
|
||||
self._latent_stream_terminal_pending[request_key] = 1
|
||||
self._latent_stream_completed.add(request_key)
|
||||
outputs.append(torch.zeros((0,), dtype=torch.float32))
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(True)
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(True)
|
||||
else:
|
||||
if is_last:
|
||||
self._latent_stream_gens.pop(request_key, None)
|
||||
self._latent_stream_terminal_pending[request_key] = 1
|
||||
self._latent_stream_completed.add(request_key)
|
||||
outputs.append(chunk_latent.detach().float().cpu())
|
||||
assert last_chunk_flags is not None
|
||||
last_chunk_flags.append(bool(is_last))
|
||||
assert payload_finished_flags is not None
|
||||
payload_finished_flags.append(False)
|
||||
finally:
|
||||
if created_temp is not None and os.path.exists(created_temp):
|
||||
os.unlink(created_temp)
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
continue
|
||||
|
||||
prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info)
|
||||
try:
|
||||
latent_audio_feat = self._pipeline.generate_latents(
|
||||
text=text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
prompt_text=prompt_text,
|
||||
cfg_value=cfg_value,
|
||||
inference_timesteps=inference_timesteps,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
)
|
||||
outputs.append(latent_audio_feat.float().cpu())
|
||||
finally:
|
||||
if temp_prompt_wav is not None and os.path.exists(temp_prompt_wav):
|
||||
os.unlink(temp_prompt_wav)
|
||||
|
||||
sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32))
|
||||
|
||||
self._ar_emit_stop_token = all(last_chunk_flags) if async_chunk and last_chunk_flags else True
|
||||
output = self._finalize_stage_output(
|
||||
output_key="latent_audio_feat",
|
||||
outputs=outputs,
|
||||
sample_rates=sample_rates,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
hidden_rows=hidden_rows,
|
||||
)
|
||||
if async_chunk and payload_finished_flags is not None:
|
||||
output.multimodal_outputs["finished"] = [
|
||||
torch.tensor(flag, dtype=torch.bool) for flag in payload_finished_flags
|
||||
]
|
||||
return output
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> torch.Tensor:
|
||||
del sampling_metadata
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if hidden_states is None:
|
||||
device, dtype = self._runner_hidden_device_dtype()
|
||||
hidden_states = torch.zeros((0, 1), device=device, dtype=dtype)
|
||||
if hidden_states.ndim == 1:
|
||||
hidden_states = hidden_states.unsqueeze(-1)
|
||||
elif hidden_states.ndim > 2:
|
||||
hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
|
||||
vocab_size = self._get_vocab_size()
|
||||
num_rows = int(hidden_states.shape[0])
|
||||
logits = torch.zeros((num_rows, vocab_size), dtype=torch.float32, device=hidden_states.device)
|
||||
eos_id = 2 if vocab_size > 2 else 0
|
||||
safe_id = 1 if vocab_size > 1 and 1 != eos_id else 0
|
||||
emit_stop = getattr(self, "_ar_emit_stop_token", True)
|
||||
if num_rows > 0:
|
||||
if emit_stop:
|
||||
logits[:, eos_id] = 1.0e6
|
||||
else:
|
||||
logits[:, eos_id] = -1.0e9
|
||||
logits[:, safe_id] = 1.0e6
|
||||
return logits
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
model_intermediate_buffer: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
del positions, intermediate_tensors, inputs_embeds, kwargs
|
||||
self._ensure_model_loaded()
|
||||
out_device, out_dtype = self._runner_hidden_device_dtype()
|
||||
if input_ids is not None and input_ids.device.type == out_device.type:
|
||||
out_device = input_ids.device
|
||||
|
||||
infos = model_intermediate_buffer or runtime_additional_information or [{}]
|
||||
hidden_rows = len(infos)
|
||||
if input_ids is not None and len(input_ids.shape) > 0:
|
||||
hidden_rows = max(hidden_rows, int(input_ids.shape[0]))
|
||||
sample_rate = int(getattr(self._pipeline, "sample_rate", 24000))
|
||||
async_chunk = bool(getattr(self.vllm_config.model_config, "async_chunk", False))
|
||||
if self.model_stage in self._VAE_STAGES:
|
||||
infos = self._maybe_recover_vae_infos(infos, input_ids, async_chunk=async_chunk)
|
||||
return self._forward_vae_stage(
|
||||
infos,
|
||||
sample_rate=sample_rate,
|
||||
async_chunk=async_chunk,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
if self.model_stage in self._LATENT_STAGES:
|
||||
return self._forward_latent_stage(
|
||||
infos,
|
||||
sample_rate=sample_rate,
|
||||
async_chunk=async_chunk,
|
||||
out_device=out_device,
|
||||
out_dtype=out_dtype,
|
||||
hidden_rows=hidden_rows,
|
||||
)
|
||||
raise ValueError(f"Unsupported VoxCPM model_stage at runtime: {self.model_stage}")
|
||||
|
||||
def make_empty_intermediate_tensors(
|
||||
self, batch_size: int, dtype: torch.dtype, device: torch.device
|
||||
) -> IntermediateTensors:
|
||||
del batch_size, dtype, device
|
||||
return {}
|
||||
|
||||
|
||||
__all__ = ["VoxCPMForConditionalGeneration"]
|
||||
@@ -1,247 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _iter_voxcpm_src_candidates() -> list[Path]:
|
||||
candidates: list[Path] = []
|
||||
env_path = os.environ.get("VLLM_OMNI_VOXCPM_CODE_PATH")
|
||||
if env_path:
|
||||
candidates.append(Path(env_path).expanduser())
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
candidates.append(repo_root.parent / "VoxCPM" / "src")
|
||||
|
||||
unique_candidates: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
candidate_key = str(candidate)
|
||||
if candidate_key in seen:
|
||||
continue
|
||||
seen.add(candidate_key)
|
||||
unique_candidates.append(candidate)
|
||||
return unique_candidates
|
||||
|
||||
|
||||
def _prepend_voxcpm_src(candidate: Path) -> None:
|
||||
candidate_str = str(candidate)
|
||||
if candidate_str not in sys.path:
|
||||
sys.path.insert(0, candidate_str)
|
||||
|
||||
|
||||
def _import_voxcpm_attrs(module_name: str, *attr_names: str) -> tuple[Any, ...]:
|
||||
last_exc: ImportError | None = None
|
||||
for candidate in _iter_voxcpm_src_candidates():
|
||||
if not candidate.exists():
|
||||
continue
|
||||
_prepend_voxcpm_src(candidate)
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
return tuple(getattr(module, attr_name) for attr_name in attr_names)
|
||||
except ImportError as exc:
|
||||
last_exc = exc
|
||||
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
return tuple(getattr(module, attr_name) for attr_name in attr_names)
|
||||
except ImportError as exc:
|
||||
last_exc = exc
|
||||
|
||||
raise ImportError(f"Failed to import {module_name}.") from last_exc
|
||||
|
||||
|
||||
def _import_voxcpm_base_model_class():
|
||||
"""Import upstream ``VoxCPMModel`` from ``VoxCPM/src/voxcpm`` (env, sibling tree, or pip)."""
|
||||
try:
|
||||
(VoxCPMModel,) = _import_voxcpm_attrs("voxcpm.model.voxcpm", "VoxCPMModel")
|
||||
return VoxCPMModel
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Failed to import VoxCPMModel. Install the `voxcpm` package or set "
|
||||
"`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory "
|
||||
"(the parent of the `voxcpm` package that contains `model/` and `modules/`)."
|
||||
) from exc
|
||||
|
||||
|
||||
def _import_voxcpm_audio_vae_classes():
|
||||
try:
|
||||
return _import_voxcpm_attrs("voxcpm.modules.audiovae", "AudioVAE", "AudioVAEConfig")
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Failed to import VoxCPM AudioVAE. Install the `voxcpm` package or set "
|
||||
"`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory."
|
||||
) from exc
|
||||
|
||||
|
||||
def _device_to_string(device: torch.device) -> str:
|
||||
if device.index is None:
|
||||
return device.type
|
||||
return f"{device.type}:{device.index}"
|
||||
|
||||
|
||||
def _normalize_dtype_name(dtype: Any) -> str | None:
|
||||
if dtype is None:
|
||||
return None
|
||||
if isinstance(dtype, torch.dtype):
|
||||
mapping = {
|
||||
torch.bfloat16: "bfloat16",
|
||||
torch.float16: "float16",
|
||||
torch.float32: "float32",
|
||||
}
|
||||
return mapping.get(dtype, str(dtype).removeprefix("torch."))
|
||||
dtype_str = str(dtype)
|
||||
return dtype_str.removeprefix("torch.")
|
||||
|
||||
|
||||
def _resolve_runtime_device(vllm_config: VllmConfig) -> torch.device:
|
||||
try:
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
return current_omni_platform.get_torch_device()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
device = getattr(getattr(vllm_config, "device_config", None), "device", None)
|
||||
if isinstance(device, torch.device):
|
||||
return device
|
||||
if device:
|
||||
return torch.device(device)
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def _prepare_runtime_model_dir(
|
||||
model_path: str | Path,
|
||||
*,
|
||||
target_device: torch.device,
|
||||
target_dtype: str | None,
|
||||
) -> str:
|
||||
source_dir = Path(model_path)
|
||||
config_path = source_dir / "config.json"
|
||||
if not config_path.exists():
|
||||
return str(source_dir)
|
||||
|
||||
config_text = config_path.read_text()
|
||||
config_dict = json.loads(config_text)
|
||||
desired_device = target_device.type
|
||||
desired_dtype = target_dtype or config_dict.get("dtype")
|
||||
|
||||
if config_dict.get("device") == desired_device and config_dict.get("dtype") == desired_dtype:
|
||||
return str(source_dir)
|
||||
|
||||
digest = sha256(f"{source_dir.resolve()}:{config_text}:{desired_device}:{desired_dtype}".encode()).hexdigest()[:16]
|
||||
runtime_dir = Path(tempfile.gettempdir()) / "vllm_omni_voxcpm_runtime" / digest
|
||||
runtime_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for entry in source_dir.iterdir():
|
||||
target = runtime_dir / entry.name
|
||||
if entry.name == "config.json" or target.exists():
|
||||
continue
|
||||
try:
|
||||
target.symlink_to(entry, target_is_directory=entry.is_dir())
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Falling back to copying VoxCPM runtime artifact %s into %s because symlink creation failed: %s",
|
||||
entry,
|
||||
runtime_dir,
|
||||
exc,
|
||||
)
|
||||
if entry.is_dir():
|
||||
shutil.copytree(entry, target, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(entry, target)
|
||||
|
||||
patched_config = dict(config_dict)
|
||||
patched_config["device"] = desired_device
|
||||
if desired_dtype is not None:
|
||||
patched_config["dtype"] = desired_dtype
|
||||
(runtime_dir / "config.json").write_text(json.dumps(patched_config, indent=2, sort_keys=True))
|
||||
return str(runtime_dir)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _force_cuda_available_for_npu(device: torch.device):
|
||||
if device.type != "npu":
|
||||
yield
|
||||
return
|
||||
|
||||
with patch("torch.cuda.is_available", return_value=True):
|
||||
yield
|
||||
|
||||
|
||||
def _is_torchcodec_load_error(exc: BaseException) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "torchcodec" in message or "load_with_torchcodec" in message
|
||||
|
||||
|
||||
def _load_audio_with_soundfile(
|
||||
prompt_wav_path: str,
|
||||
*,
|
||||
sample_rate: int,
|
||||
) -> torch.Tensor:
|
||||
try:
|
||||
import soundfile as sf
|
||||
except ImportError:
|
||||
raise
|
||||
|
||||
audio_np, source_sr = sf.read(prompt_wav_path, dtype="float32", always_2d=True)
|
||||
audio = torch.from_numpy(np.ascontiguousarray(audio_np.T))
|
||||
|
||||
if audio.size(0) > 1:
|
||||
audio = audio.mean(dim=0, keepdim=True)
|
||||
|
||||
if int(source_sr) != int(sample_rate):
|
||||
try:
|
||||
import torchaudio
|
||||
except ImportError as exc:
|
||||
raise ImportError("torchaudio is required for resampling prompt audio.") from exc
|
||||
audio = torchaudio.functional.resample(audio, int(source_sr), int(sample_rate))
|
||||
|
||||
return audio
|
||||
|
||||
|
||||
def _build_prompt_cache_with_soundfile(model: Any, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
if args:
|
||||
prompt_text = args[0]
|
||||
prompt_wav_path = args[1] if len(args) > 1 else kwargs.get("prompt_wav_path")
|
||||
else:
|
||||
prompt_text = kwargs.get("prompt_text")
|
||||
prompt_wav_path = kwargs.get("prompt_wav_path")
|
||||
|
||||
if not prompt_text or not prompt_wav_path:
|
||||
raise ValueError("prompt_text and prompt_wav_path are required")
|
||||
|
||||
audio = _load_audio_with_soundfile(prompt_wav_path, sample_rate=int(model.sample_rate))
|
||||
|
||||
patch_len = model.patch_size * model.chunk_size
|
||||
if audio.size(1) % patch_len != 0:
|
||||
padding_size = patch_len - audio.size(1) % patch_len
|
||||
audio = torch.nn.functional.pad(audio, (padding_size, 0))
|
||||
|
||||
audio_feat = model.audio_vae.encode(audio.to(model.device), model.sample_rate).cpu()
|
||||
audio_feat = audio_feat.view(
|
||||
model.audio_vae.latent_dim,
|
||||
-1,
|
||||
model.patch_size,
|
||||
).permute(1, 2, 0)
|
||||
|
||||
return {
|
||||
"prompt_text": prompt_text,
|
||||
"audio_feat": audio_feat,
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def resolve_voxcpm_model_dir(model: str) -> Path:
|
||||
model_path = Path(model).expanduser()
|
||||
if model_path.exists():
|
||||
return model_path
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
return Path(snapshot_download(repo_id=model))
|
||||
|
||||
|
||||
def prepare_voxcpm_hf_config_dir(model_dir: str | Path, hf_config_dir: str | Path) -> Path:
|
||||
model_dir = Path(model_dir).expanduser()
|
||||
hf_config_dir = Path(hf_config_dir).expanduser()
|
||||
hf_config_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
source_config_path = model_dir / "config.json"
|
||||
if not source_config_path.exists():
|
||||
raise FileNotFoundError(f"VoxCPM config.json not found under {model_dir}")
|
||||
|
||||
config_path = hf_config_dir / "config.json"
|
||||
shutil.copy2(source_config_path, config_path)
|
||||
|
||||
source_generation_config_path = model_dir / "generation_config.json"
|
||||
if source_generation_config_path.exists():
|
||||
shutil.copy2(source_generation_config_path, hf_config_dir / "generation_config.json")
|
||||
|
||||
config_dict = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
config_dict["model_type"] = "voxcpm"
|
||||
config_dict.setdefault("architectures", ["VoxCPMForConditionalGeneration"])
|
||||
config_path.write_text(json.dumps(config_dict, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
return hf_config_dir
|
||||
|
||||
|
||||
__all__ = [
|
||||
"prepare_voxcpm_hf_config_dir",
|
||||
"resolve_voxcpm_model_dir",
|
||||
]
|
||||
@@ -1,185 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
|
||||
|
||||
class _DirectVoxCPMLatentGenerator:
|
||||
def __init__(self, tts_model: Any):
|
||||
self.tts_model = tts_model
|
||||
self.sample_rate = int(getattr(tts_model, "sample_rate", 24000))
|
||||
|
||||
def generate_latents(
|
||||
self,
|
||||
*,
|
||||
text: str,
|
||||
prompt_wav_path: str | None = None,
|
||||
prompt_text: str | None = None,
|
||||
cfg_value: float = 2.0,
|
||||
inference_timesteps: int = 10,
|
||||
min_len: int = 2,
|
||||
max_len: int = 4096,
|
||||
retry_badcase: bool = True,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
) -> torch.Tensor:
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("target text must be a non-empty string")
|
||||
if (prompt_wav_path is None) != (prompt_text is None):
|
||||
raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None")
|
||||
if prompt_wav_path is not None and not os.path.exists(prompt_wav_path):
|
||||
raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}")
|
||||
|
||||
prompt_cache = None
|
||||
if prompt_wav_path is not None and prompt_text is not None:
|
||||
prompt_cache = self.tts_model.build_prompt_cache(
|
||||
prompt_text=prompt_text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
)
|
||||
|
||||
gen_kw = dict(
|
||||
target_text=" ".join(text.split()),
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
)
|
||||
latent_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache", None)
|
||||
if latent_entry is not None:
|
||||
_, _, pred_audio_feat = latent_entry(**gen_kw)
|
||||
else:
|
||||
try:
|
||||
_, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(
|
||||
**gen_kw,
|
||||
latents_only=True,
|
||||
)
|
||||
except TypeError:
|
||||
_, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(**gen_kw)
|
||||
return pred_audio_feat.detach().cpu().to(torch.float32)
|
||||
|
||||
def iter_latent_chunks_streaming(
|
||||
self,
|
||||
*,
|
||||
text: str,
|
||||
prompt_wav_path: str | None = None,
|
||||
prompt_text: str | None = None,
|
||||
cfg_value: float = 2.0,
|
||||
inference_timesteps: int = 10,
|
||||
min_len: int = 2,
|
||||
max_len: int = 4096,
|
||||
streaming_prefix_len: int = 3,
|
||||
retry_badcase: bool = False,
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
) -> Generator[tuple[torch.Tensor, bool], None, None]:
|
||||
"""Yield ``(latent_window, is_last_chunk)`` for Omni async_chunk latent to VAE."""
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("target text must be a non-empty string")
|
||||
if (prompt_wav_path is None) != (prompt_text is None):
|
||||
raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None")
|
||||
if prompt_wav_path is not None and not os.path.exists(prompt_wav_path):
|
||||
raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}")
|
||||
|
||||
prompt_cache = None
|
||||
if prompt_wav_path is not None and prompt_text is not None:
|
||||
prompt_cache = self.tts_model.build_prompt_cache(
|
||||
prompt_text=prompt_text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
)
|
||||
|
||||
gen_kw = dict(
|
||||
target_text=" ".join(text.split()),
|
||||
prompt_cache=prompt_cache,
|
||||
min_len=min_len,
|
||||
max_len=max_len,
|
||||
inference_timesteps=inference_timesteps,
|
||||
cfg_value=cfg_value,
|
||||
retry_badcase=retry_badcase,
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming_prefix_len=streaming_prefix_len,
|
||||
)
|
||||
stream_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache_streaming", None)
|
||||
if stream_entry is not None:
|
||||
gen = stream_entry(**gen_kw)
|
||||
else:
|
||||
fallback_stream_entry = getattr(self.tts_model, "generate_with_prompt_cache_streaming", None)
|
||||
if fallback_stream_entry is not None:
|
||||
gen = fallback_stream_entry(**gen_kw, latents_only=True)
|
||||
else:
|
||||
gen = self.tts_model._generate_with_prompt_cache(streaming=True, latents_only=True, **gen_kw)
|
||||
|
||||
iterator = iter(gen)
|
||||
previous = next(iterator, None)
|
||||
while previous is not None:
|
||||
current = next(iterator, None)
|
||||
_, _target_tok, chunk_latent = previous
|
||||
if not isinstance(chunk_latent, torch.Tensor):
|
||||
chunk_latent = torch.as_tensor(chunk_latent)
|
||||
yield chunk_latent, current is None
|
||||
previous = current
|
||||
|
||||
|
||||
class _DirectVoxCPMAudioVAE:
|
||||
def __init__(self, audio_vae: nn.Module, *, patch_size: int = 2):
|
||||
self.audio_vae = audio_vae
|
||||
self.sample_rate = int(getattr(audio_vae, "sample_rate", 24000))
|
||||
self.latent_dim = int(getattr(audio_vae, "latent_dim", 64))
|
||||
self.patch_size = int(patch_size)
|
||||
self._chunk_size = int(getattr(audio_vae, "chunk_size", 1))
|
||||
self._stream_audio_patch_samples = max(1, self.patch_size * self._chunk_size)
|
||||
|
||||
def _prepare_latents_for_decode(self, latent_audio_feat: Any) -> torch.Tensor:
|
||||
latents = latent_audio_feat
|
||||
if not isinstance(latents, torch.Tensor):
|
||||
latents = torch.tensor(latents, dtype=torch.float32)
|
||||
latents = latents.detach().to(torch.float32)
|
||||
|
||||
if latents.ndim == 3:
|
||||
if latents.shape[-1] == self.latent_dim:
|
||||
latents = rearrange(latents, "t p d -> 1 d (t p)")
|
||||
elif latents.shape[1] == self.latent_dim:
|
||||
latents = latents.contiguous()
|
||||
else:
|
||||
raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}")
|
||||
elif latents.ndim == 2:
|
||||
if latents.shape[0] == self.latent_dim:
|
||||
latents = latents.unsqueeze(0)
|
||||
elif latents.shape[1] == self.latent_dim:
|
||||
latents = rearrange(latents, "t d -> 1 d t")
|
||||
else:
|
||||
raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}")
|
||||
else:
|
||||
raise ValueError(f"Unsupported latent_audio_feat ndim: {latents.ndim}")
|
||||
|
||||
return latents
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latent_audio_feat: Any, *, trim_streaming_patch: bool = False) -> torch.Tensor:
|
||||
latents = self._prepare_latents_for_decode(latent_audio_feat)
|
||||
device = next(self.audio_vae.parameters()).device
|
||||
raw = self.audio_vae.decode(latents.to(device=device, dtype=torch.float32))
|
||||
if isinstance(raw, dict):
|
||||
audio = raw.get("audio")
|
||||
if audio is None:
|
||||
audio = next(v for v in raw.values() if isinstance(v, torch.Tensor))
|
||||
else:
|
||||
audio = raw
|
||||
if audio.dim() == 3:
|
||||
stream = audio.squeeze(1)
|
||||
elif audio.dim() == 2:
|
||||
stream = audio
|
||||
else:
|
||||
stream = audio.reshape(audio.shape[0], -1)
|
||||
if trim_streaming_patch:
|
||||
stream = stream[..., -self._stream_audio_patch_samples :]
|
||||
return stream.reshape(-1).detach().cpu().to(torch.float32)
|
||||
@@ -1,95 +0,0 @@
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: qwen3_tts
|
||||
max_num_seqs: 10
|
||||
model_arch: Qwen3TTSTalkerForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.3
|
||||
max_num_batched_tokens: 512
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
|
||||
# Use named connector to apply runtime.connectors.extra.
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 10
|
||||
model_arch: Qwen3TTSCode2Wav
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.3
|
||||
# Must be divisible by num_code_groups and cover (left_context + chunk).
|
||||
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep headroom past 32k.
|
||||
max_num_batched_tokens: 65536
|
||||
# async_chunk appends windows per step; max_model_len must cover accumulated flat codec stream.
|
||||
max_model_len: 65536
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
# Distributed connector configuration
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
tts_args:
|
||||
max_instructions_length: 500
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
# Frame-aligned codec streaming transport.
|
||||
codec_streaming: true
|
||||
# Connector polling / timeout (unit: loop count, sleep interval in seconds).
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
# Match the decoder sliding attention window to avoid chunk-boundary noise.
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 72
|
||||
# Emit only the first audio chunk early, then return to codec_chunk_frames.
|
||||
initial_codec_chunk_frames: 1
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -1,98 +0,0 @@
|
||||
# VoxCPM two-stage streaming (align with qwen3_tts.yaml async_chunk pattern).
|
||||
# Stage0 (latent_generator) emits latent in time chunks; Stage1 (VAE) decodes as chunks arrive.
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
model_stage: latent_generator
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
# Optional persistent HF-compatible config dir for native VoxCPM models.
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.7
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae_async_chunk
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
stop_token_ids: [2]
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.0
|
||||
final_output: false
|
||||
output_connectors:
|
||||
to_stage_1: voxcpm_shm
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: float32
|
||||
model_stage: vae
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.15
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 4096
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: voxcpm_shm
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
|
||||
connectors:
|
||||
voxcpm_shm:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
# Frame-aligned codec streaming transport.
|
||||
codec_streaming: true
|
||||
# Connector polling / timeout (unit: loop count, sleep interval in seconds).
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
# Align with Omni: small chunks with sufficient context overlap.
|
||||
codec_chunk_frames: 1
|
||||
codec_left_context_frames: 1
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -1,115 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.inputs import TextPrompt
|
||||
|
||||
from vllm_omni.data_entry_keys import (
|
||||
CodesStruct,
|
||||
MetaStruct,
|
||||
OmniPayloadStruct,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
|
||||
_VOXCPM_LATENT_MAGIC = 131071
|
||||
|
||||
|
||||
def _serialize_latent_to_codes(latent: Any) -> list[int]:
|
||||
latent_tensor = latent if isinstance(latent, torch.Tensor) else torch.as_tensor(latent)
|
||||
latent_tensor = latent_tensor.detach().cpu().contiguous()
|
||||
if latent_tensor.ndim == 3:
|
||||
if latent_tensor.shape[0] != 1:
|
||||
raise ValueError(f"Expected batch=1 latent tensor, got shape={tuple(latent_tensor.shape)}")
|
||||
latent_tensor = latent_tensor.squeeze(0)
|
||||
if latent_tensor.ndim != 2:
|
||||
raise ValueError(f"Unsupported latent_audio_feat shape for async chunk: {tuple(latent_tensor.shape)}")
|
||||
latent_dim, time_dim = int(latent_tensor.shape[0]), int(latent_tensor.shape[1])
|
||||
packed = latent_tensor.to(torch.bfloat16).contiguous().view(torch.uint16).reshape(-1).to(torch.int32)
|
||||
return [_VOXCPM_LATENT_MAGIC, latent_dim, time_dim, *packed.tolist()]
|
||||
|
||||
|
||||
def _coerce_finished_flag(value: Any) -> bool:
|
||||
"""Normalize VoxCPM async-chunk finished markers to a Python bool."""
|
||||
if value is None:
|
||||
return False
|
||||
if isinstance(value, torch.Tensor):
|
||||
if value.numel() != 1:
|
||||
raise ValueError(f"finished tensor must be scalar, got shape={tuple(value.shape)}")
|
||||
return bool(value.detach().cpu().item())
|
||||
if isinstance(value, (list, tuple)):
|
||||
if not value:
|
||||
return False
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"finished container must have one element, got len={len(value)}")
|
||||
return _coerce_finished_flag(value[0])
|
||||
return bool(value)
|
||||
|
||||
|
||||
def latent2vae(
|
||||
source_outputs: list[Any],
|
||||
_prompt: OmniTokensPrompt | TextPrompt | None = None,
|
||||
_requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
vae_inputs: list[OmniTokensPrompt] = []
|
||||
for source_output in source_outputs:
|
||||
output = source_output.outputs[0]
|
||||
multimodal_output = getattr(output, "multimodal_output", None)
|
||||
if not isinstance(multimodal_output, dict) or "latent_audio_feat" not in multimodal_output:
|
||||
raise ValueError(
|
||||
"VoxCPM latent stage output missing 'latent_audio_feat'. "
|
||||
f"request_id={getattr(source_output, 'request_id', None)}"
|
||||
)
|
||||
|
||||
additional_information = {
|
||||
"latent_audio_feat": multimodal_output["latent_audio_feat"],
|
||||
}
|
||||
if "sr" in multimodal_output:
|
||||
additional_information["sample_rate"] = [int(multimodal_output["sr"])]
|
||||
|
||||
vae_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0],
|
||||
additional_information=additional_information,
|
||||
multi_modal_data=None,
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
|
||||
return vae_inputs
|
||||
|
||||
|
||||
def _eof_payload() -> OmniPayloadStruct:
|
||||
return OmniPayloadStruct(
|
||||
codes=CodesStruct(audio=torch.empty(0, dtype=torch.long)),
|
||||
meta=MetaStruct(finished=torch.tensor(True, dtype=torch.bool)),
|
||||
)
|
||||
|
||||
|
||||
def latent2vae_async_chunk(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any] | None,
|
||||
request: Any,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
"""Stage-0 latent → stage-1 VAE under ``async_chunk`` (connector payload)."""
|
||||
# Kept for callback signature compatibility with OmniChunkTransferAdapter.
|
||||
_ = transfer_manager
|
||||
finished_request = _coerce_finished_flag(is_finished)
|
||||
if callable(getattr(request, "is_finished", None)):
|
||||
finished_request = finished_request or _coerce_finished_flag(request.is_finished())
|
||||
if not isinstance(pooling_output, dict):
|
||||
return _eof_payload() if finished_request else None
|
||||
|
||||
latent = pooling_output.get("latent_audio_feat")
|
||||
if isinstance(latent, torch.Tensor) and latent.numel() == 0:
|
||||
latent = None
|
||||
|
||||
if latent is None:
|
||||
return _eof_payload() if finished_request else None
|
||||
|
||||
serialized_codes = _serialize_latent_to_codes(latent)
|
||||
return OmniPayloadStruct(
|
||||
codes=CodesStruct(audio=torch.tensor(serialized_codes, dtype=torch.long)),
|
||||
meta=MetaStruct(finished=torch.tensor(finished_request, dtype=torch.bool)),
|
||||
)
|
||||
@@ -1,67 +0,0 @@
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
model_stage: latent_generator
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
# Optional persistent HF-compatible config dir for native VoxCPM models.
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.75
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.0
|
||||
final_output: false
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: float32
|
||||
model_stage: vae
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 4096
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 1
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
@@ -1,89 +0,0 @@
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
model_stage: latent_generator
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
# Optional persistent HF-compatible config dir for native VoxCPM models.
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.75
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.0
|
||||
final_output: false
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
dtype: float32
|
||||
model_stage: vae
|
||||
model_arch: VoxCPMForConditionalGeneration
|
||||
hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 4096
|
||||
engine_input_source: [0]
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 1
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: false
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -17,7 +17,6 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"FishSpeechConfig": "vllm_omni.transformers_utils.configs.fish_speech",
|
||||
"FishSpeechSlowARConfig": "vllm_omni.transformers_utils.configs.fish_speech",
|
||||
"FishSpeechFastARConfig": "vllm_omni.transformers_utils.configs.fish_speech",
|
||||
"VoxCPMConfig": "vllm_omni.transformers_utils.configs.voxcpm",
|
||||
"VoxCPM2Config": "vllm_omni.transformers_utils.configs.voxcpm2",
|
||||
"VoxtralTTSConfig": "vllm_omni.transformers_utils.configs.voxtral_tts",
|
||||
"CosyVoice3Config": "vllm_omni.transformers_utils.configs.cosyvoice3",
|
||||
@@ -37,7 +36,6 @@ __all__ = [
|
||||
"FishSpeechConfig",
|
||||
"FishSpeechSlowARConfig",
|
||||
"FishSpeechFastARConfig",
|
||||
"VoxCPMConfig",
|
||||
"VoxCPM2Config",
|
||||
"VoxtralTTSConfig",
|
||||
"CosyVoice3Config",
|
||||
@@ -70,6 +68,5 @@ from vllm_omni.transformers_utils.configs import fish_speech as _fish_speech #
|
||||
from vllm_omni.transformers_utils.configs import mammoth_moda2 as _mammoth_moda2 # noqa: F401, E402
|
||||
from vllm_omni.transformers_utils.configs import ming_flash_omni as _ming_flash_omni # noqa: F401, E402
|
||||
from vllm_omni.transformers_utils.configs import omnivoice as _omnivoice # noqa: F401, E402
|
||||
from vllm_omni.transformers_utils.configs import voxcpm as _voxcpm # noqa: F401, E402
|
||||
from vllm_omni.transformers_utils.configs import voxcpm2 as _voxcpm2 # noqa: F401, E402
|
||||
from vllm_omni.transformers_utils.configs import voxtral_tts as _voxtral_tts # noqa: F401, E402
|
||||
@@ -1,68 +0,0 @@
|
||||
from transformers import AutoConfig
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
class VoxCPMConfig(PretrainedConfig):
|
||||
model_type = "voxcpm"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bos_token_id: int = 1,
|
||||
eos_token_id: int = 2,
|
||||
vocab_size: int = 32000,
|
||||
hidden_size: int = 1024,
|
||||
intermediate_size: int = 4096,
|
||||
max_position_embeddings: int = 4096,
|
||||
num_attention_heads: int = 16,
|
||||
num_hidden_layers: int = 24,
|
||||
num_key_value_heads: int = 16,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
rope_theta: float = 10000.0,
|
||||
rope_scaling: dict | None = None,
|
||||
lm_config: dict | None = None,
|
||||
encoder_config: dict | None = None,
|
||||
dit_config: dict | None = None,
|
||||
audio_vae_config: dict | None = None,
|
||||
patch_size: int = 2,
|
||||
feat_dim: int = 64,
|
||||
residual_lm_num_layers: int = 6,
|
||||
scalar_quantization_latent_dim: int = 256,
|
||||
scalar_quantization_scale: int = 9,
|
||||
max_length: int = 4096,
|
||||
device: str = "cuda",
|
||||
dtype: str = "bfloat16",
|
||||
dit_mean_mode: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
|
||||
self.lm_config = lm_config or {}
|
||||
self.encoder_config = encoder_config or {}
|
||||
self.dit_config = dit_config or {}
|
||||
self.audio_vae_config = audio_vae_config
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.feat_dim = feat_dim
|
||||
self.residual_lm_num_layers = residual_lm_num_layers
|
||||
self.scalar_quantization_latent_dim = scalar_quantization_latent_dim
|
||||
self.scalar_quantization_scale = scalar_quantization_scale
|
||||
self.max_length = max_length
|
||||
self.device = device
|
||||
self.dtype = dtype
|
||||
self.dit_mean_mode = dit_mean_mode
|
||||
|
||||
|
||||
AutoConfig.register("voxcpm", VoxCPMConfig)
|
||||
|
||||
__all__ = ["VoxCPMConfig"]
|
||||
Reference in new issue
Block a user