From fc8486cf20d9014bb189ffceb042c109f7e6d340 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Wed, 20 May 2026 03:24:06 -0400 Subject: [PATCH] [Doc][TTS] CosyVoice3 online docs + residual TTS yaml cleanup + remove VoxCPM v1 (#3748) Signed-off-by: Yueqian Lin --- benchmarks/tts/README.md | 1 - benchmarks/tts/bench_voxcpm_offline.py | 922 ------------------ docs/api/README.md | 1 - docs/serving/speech_api.md | 7 + .../offline_inference/text_to_speech.md | 60 -- .../examples/online_serving/text_to_speech.md | 136 ++- .../text_to_speech/README.md | 55 +- .../text_to_speech/voxcpm/end2end.py | 220 ----- .../online_serving/text_to_speech/README.md | 71 +- .../text_to_speech/cosyvoice3/run_server.sh | 28 + .../cosyvoice3/speech_client.py | 142 +++ .../voxcpm/openai_speech_client.py | 155 --- .../text_to_speech/voxcpm/run_server.sh | 38 - pyproject.toml | 2 +- .../test_voxcpm_expansion.py | 99 -- tests/engine/test_arg_utils.py | 18 - .../openai_api/test_serving_speech_voxcpm.py | 275 ------ tests/entrypoints/test_utils.py | 33 - .../test_voxcpm_async_chunk.py | 85 -- vllm_omni/deploy/voxcpm.yaml | 69 -- vllm_omni/engine/arg_utils.py | 1 - .../entrypoints/openai/serving_speech.py | 69 +- vllm_omni/model_executor/models/registry.py | 6 - .../model_executor/models/voxcpm/__init__.py | 5 - .../model_executor/models/voxcpm/voxcpm.py | 886 ----------------- .../models/voxcpm/voxcpm_loader.py | 247 ----- .../models/voxcpm/voxcpm_runtime_utils.py | 44 - .../models/voxcpm/voxcpm_stage_wrappers.py | 185 ---- .../stage_configs/qwen3_tts_uniproc.yaml | 95 -- .../stage_configs/voxcpm_async_chunk.yaml | 98 -- .../stage_input_processors/voxcpm.py | 115 --- .../platforms/npu/stage_configs/voxcpm.yaml | 67 -- .../npu/stage_configs/voxcpm_async_chunk.yaml | 89 -- .../transformers_utils/configs/__init__.py | 3 - .../transformers_utils/configs/voxcpm.py | 68 -- 35 files changed, 248 insertions(+), 4147 deletions(-) delete mode 100644 benchmarks/tts/bench_voxcpm_offline.py delete mode 100644 examples/offline_inference/text_to_speech/voxcpm/end2end.py create mode 100755 examples/online_serving/text_to_speech/cosyvoice3/run_server.sh create mode 100644 examples/online_serving/text_to_speech/cosyvoice3/speech_client.py delete mode 100644 examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py delete mode 100755 examples/online_serving/text_to_speech/voxcpm/run_server.sh delete mode 100644 tests/e2e/offline_inference/test_voxcpm_expansion.py delete mode 100644 tests/entrypoints/openai_api/test_serving_speech_voxcpm.py delete mode 100644 tests/model_executor/stage_input_processors/test_voxcpm_async_chunk.py delete mode 100644 vllm_omni/deploy/voxcpm.yaml delete mode 100644 vllm_omni/model_executor/models/voxcpm/__init__.py delete mode 100644 vllm_omni/model_executor/models/voxcpm/voxcpm.py delete mode 100644 vllm_omni/model_executor/models/voxcpm/voxcpm_loader.py delete mode 100644 vllm_omni/model_executor/models/voxcpm/voxcpm_runtime_utils.py delete mode 100644 vllm_omni/model_executor/models/voxcpm/voxcpm_stage_wrappers.py delete mode 100644 vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml delete mode 100644 vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml delete mode 100644 vllm_omni/model_executor/stage_input_processors/voxcpm.py delete mode 100644 vllm_omni/platforms/npu/stage_configs/voxcpm.yaml delete mode 100644 vllm_omni/platforms/npu/stage_configs/voxcpm_async_chunk.yaml delete mode 100644 vllm_omni/transformers_utils/configs/voxcpm.py diff --git a/benchmarks/tts/README.md b/benchmarks/tts/README.md index d8841f382..f6930a866 100644 --- a/benchmarks/tts/README.md +++ b/benchmarks/tts/README.md @@ -233,7 +233,6 @@ sentinel for regressions in this area. benchmarks/tts/ ├── README.md (this file) ├── bench_tts.py CLI — serve-mode benchmark driver -├── bench_voxcpm_offline.py CLI — offline VoxCPM benchmark (sync + streaming) ├── plot_results.py Generate per-task / per-concurrency curves └── model_configs.yaml Model registry (supported tasks + extra body) ``` diff --git a/benchmarks/tts/bench_voxcpm_offline.py b/benchmarks/tts/bench_voxcpm_offline.py deleted file mode 100644 index 9ab0fa508..000000000 --- a/benchmarks/tts/bench_voxcpm_offline.py +++ /dev/null @@ -1,922 +0,0 @@ -"""Offline VoxCPM benchmark for vLLM Omni. - -Supports both: -- sync one-shot (Omni.generate) -- streaming (AsyncOmni.generate with async_chunk config) -- text-only synthesis -- voice cloning -- text/clone batch inputs from txt or jsonl - -Usage:: - - # Sync (default voice) - python benchmarks/tts/bench_voxcpm_offline.py \\ - --model /path/to/VoxCPM \\ - --text "Hello world" \\ - --output-dir results/audio/ - - # Streaming (async_chunk) - python benchmarks/tts/bench_voxcpm_offline.py \\ - --model /path/to/VoxCPM \\ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \\ - --txt-prompts prompts.txt \\ - --output-dir results/audio/ - - # Voice cloning batch via JSONL - python benchmarks/tts/bench_voxcpm_offline.py \\ - --model /path/to/VoxCPM \\ - --jsonl-prompts prompts.jsonl \\ - --output-dir results/audio/ -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import os -import tempfile -import time -import uuid -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import torch -from vllm.utils.argparse_utils import FlexibleArgumentParser - -from vllm_omni import AsyncOmni, Omni - - -def _find_repo_root(start: Path) -> Path: - """Walk up from ``start`` until a repo marker is found. - - Falls back to ``parents[2]`` for backwards compatibility if no marker hits - (which can only happen in unusual checkouts — the tree should always have - pyproject.toml + vllm_omni/ at the top level). - """ - for candidate in [start, *start.parents]: - if (candidate / "pyproject.toml").is_file() and (candidate / "vllm_omni").is_dir(): - return candidate - return start.parents[2] - - -REPO_ROOT = _find_repo_root(Path(__file__).resolve()) -DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml" -DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "deploy" / "voxcpm.yaml" - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True, slots=True) -class PromptSpec: - text: str - label: str - ref_audio: str | None = None - ref_text: str | None = None - - -def _require_soundfile(): - try: - import soundfile as sf # type: ignore - except ModuleNotFoundError as exc: - raise RuntimeError( - "soundfile is required to write VoxCPM benchmark WAV outputs. Install it with: pip install soundfile" - ) from exc - return sf - - -def _build_prompt( - args, - *, - text: str, - ref_audio: str | None = None, - ref_text: str | None = None, - global_request_id: str | None = None, -) -> dict[str, Any]: - additional_information: dict[str, list[Any]] = { - "text": [text], - "cfg_value": [args.cfg_value], - "inference_timesteps": [args.inference_timesteps], - "min_len": [args.min_len], - "max_new_tokens": [args.max_new_tokens], - } - if args.streaming_prefix_len is not None: - additional_information["streaming_prefix_len"] = [args.streaming_prefix_len] - - if ref_audio: - additional_information["ref_audio"] = [ref_audio] - if ref_text: - additional_information["ref_text"] = [ref_text] - if global_request_id is not None: - additional_information["global_request_id"] = [global_request_id] - - return { - "prompt_token_ids": [1], - "additional_information": additional_information, - } - - -def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor: - audio = mm.get("audio", mm.get("model_outputs")) - if audio is None: - raise ValueError("No audio output found in multimodal output.") - if isinstance(audio, list): - parts = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio] - audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0) - if not isinstance(audio, torch.Tensor): - audio = torch.as_tensor(audio) - return audio.float().cpu().reshape(-1) - - -def _extract_sample_rate(mm: dict[str, Any]) -> int: - sr_raw = mm.get("sr", 24000) - if isinstance(sr_raw, list) and sr_raw: - sr_raw = sr_raw[-1] - if hasattr(sr_raw, "item"): - return int(sr_raw.item()) - return int(sr_raw) - - -def _emit_offline_metrics( - *, - request_id: str, - elapsed_s: float, - first_audio_elapsed: float | None, - audio_duration_s: float, -) -> None: - metrics = { - "request_id": request_id, - "ttfp_ms": round(first_audio_elapsed * 1000.0, 3) if first_audio_elapsed is not None else None, - "audio_duration_s": round(audio_duration_s, 6), - "rtf": round(elapsed_s / audio_duration_s, 6) if audio_duration_s > 0 else None, - } - print(f"[OfflineMetrics] {metrics}") - - -def _write_audio_tensor(output_path: Path, audio_tensor: Any, sample_rate: int) -> None: - sf = _require_soundfile() - if isinstance(audio_tensor, torch.Tensor): - audio_np = audio_tensor.float().cpu().clamp(-1.0, 1.0).numpy() - else: - audio_np = torch.as_tensor(audio_tensor).float().cpu().clamp(-1.0, 1.0).numpy() - sf.write( - output_path, - audio_np, - sample_rate, - format="WAV", - subtype="PCM_16", - ) - - -def _save_wav(mm: dict[str, Any], output_dir: Path, request_id: str) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / f"output_{request_id}.wav" - _write_audio_tensor(output_path, _extract_audio_tensor(mm), _extract_sample_rate(mm)) - return output_path - - -def _iter_request_multimodal_outputs(request_output: Any): - outputs = getattr(request_output, "outputs", None) - if outputs: - for output in outputs: - mm = getattr(output, "multimodal_output", None) - if isinstance(mm, dict): - yield mm - - mm = getattr(request_output, "multimodal_output", None) - if isinstance(mm, dict): - yield mm - - -def _read_non_empty_lines(path: str) -> list[str]: - with open(path, encoding="utf-8") as f: - return [line.strip() for line in f if line.strip()] - - -def _load_prompt_specs(args) -> list[PromptSpec]: - specs: list[PromptSpec] = [] - - if args.txt_prompts is not None: - texts = _read_non_empty_lines(args.txt_prompts) - if not texts: - raise ValueError(f"No prompts found in {args.txt_prompts}") - for idx, text in enumerate(texts, start=1): - specs.append( - PromptSpec( - text=text, - label=f"item{idx:03d}", - ref_audio=args.ref_audio, - ref_text=args.ref_text, - ) - ) - return specs - - if args.jsonl_prompts is not None: - with open(args.jsonl_prompts, encoding="utf-8") as f: - for line_no, raw_line in enumerate(f, start=1): - line = raw_line.strip() - if not line: - continue - try: - item = json.loads(line) - except json.JSONDecodeError as exc: - raise ValueError(f"{args.jsonl_prompts}:{line_no} is not valid JSON: {exc}") from exc - if not isinstance(item, dict): - raise ValueError(f"{args.jsonl_prompts}:{line_no} must be a JSON object") - - text = item.get("text") - if not isinstance(text, str) or not text.strip(): - raise ValueError(f"{args.jsonl_prompts}:{line_no} requires non-empty string field 'text'") - - ref_audio = item.get("ref_audio", args.ref_audio) - ref_text = item.get("ref_text", args.ref_text) - if (ref_audio is None) != (ref_text is None): - raise ValueError( - f"{args.jsonl_prompts}:{line_no} must provide both 'ref_audio' and 'ref_text' together" - ) - - specs.append( - PromptSpec( - text=text.strip(), - label=f"item{len(specs) + 1:03d}", - ref_audio=ref_audio, - ref_text=ref_text, - ) - ) - - if not specs: - raise ValueError(f"No prompts found in {args.jsonl_prompts}") - return specs - - specs.append( - PromptSpec( - text=args.text, - label="item001", - ref_audio=args.ref_audio, - ref_text=args.ref_text, - ) - ) - return specs - - -def _build_prompt_for_spec(args, spec: PromptSpec, *, global_request_id: str | None = None) -> dict[str, Any]: - return _build_prompt( - args, - text=spec.text, - ref_audio=spec.ref_audio, - ref_text=spec.ref_text, - global_request_id=global_request_id, - ) - - -def _count_voice_clone_prompts(prompt_specs: list[PromptSpec]) -> int: - return sum(1 for spec in prompt_specs if spec.ref_audio is not None) - - -def _get_warmup_specs(prompt_specs: list[PromptSpec]) -> list[PromptSpec]: - return prompt_specs[:1] - - -def _extract_stream_finished(stage_output: Any) -> bool: - request_output = getattr(stage_output, "request_output", None) - request_finished = getattr(request_output, "finished", None) - if request_finished is not None: - return bool(request_finished) - return bool(getattr(stage_output, "finished", False)) - - -def _build_profiled_stage_config( - stage_configs_path: str, - profiler_dir: str, -) -> str: - stage_config_path = Path(stage_configs_path) - yaml_text = stage_config_path.read_text(encoding="utf-8") - injected_lines: list[str] = [] - injected_count = 0 - - for line in yaml_text.splitlines(): - injected_lines.append(line) - if line.strip() != "engine_args:": - continue - indent = line[: len(line) - len(line.lstrip())] - child_indent = indent + " " - grandchild_indent = child_indent + " " - injected_lines.extend( - [ - f"{child_indent}profiler_config:", - f'{grandchild_indent}profiler: "torch"', - f'{grandchild_indent}torch_profiler_dir: "{profiler_dir}"', - f"{grandchild_indent}torch_profiler_with_stack: true", - ] - ) - injected_count += 1 - - if injected_count == 0: - raise ValueError(f"No engine_args block found in stage config: {stage_configs_path}") - - tmp = tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - delete=False, - suffix=".yaml", - prefix=f"{stage_config_path.stem}_profile_", - ) - tmp.write("\n".join(injected_lines) + "\n") - tmp.close() - return tmp.name - - -def parse_args(): - parser = FlexibleArgumentParser( - description="Offline split-stage VoxCPM inference with vLLM Omni (auto sync/streaming by stage config)" - ) - parser.add_argument( - "--model", - type=str, - default=os.environ.get("VOXCPM_MODEL"), - help="Local VoxCPM model directory. Defaults to $VOXCPM_MODEL.", - ) - parser.add_argument( - "--text", - type=str, - default="This is a split-stage VoxCPM synthesis example running on vLLM Omni.", - help="Text to synthesize. Ignored when --txt-prompts or --jsonl-prompts is used.", - ) - parser.add_argument( - "--txt-prompts", - type=str, - default=None, - help="Path to a .txt file with one synthesis text per line.", - ) - parser.add_argument( - "--jsonl-prompts", - type=str, - default=None, - help=( - "Path to a .jsonl file. Each line must contain at least {'text': ...}; " - "clone rows can also set ref_audio/ref_text, and ref_text must be the " - "real transcript of ref_audio." - ), - ) - parser.add_argument( - "--ref-audio", - type=str, - default=None, - help=( - "Optional reference audio path for voice cloning. With --txt-prompts, " - "the same reference is applied to every line." - ), - ) - parser.add_argument( - "--ref-text", - type=str, - default=None, - help=( - "Real transcript of the reference audio. Placeholder text or mismatched " - "text will usually produce noisy/electronic clone audio." - ), - ) - parser.add_argument( - "--stage-configs-path", - type=str, - default=str(DEFAULT_STAGE_SYNC), - help="Stage config YAML path. Routing is selected only from this path.", - ) - parser.add_argument( - "--cfg-value", - type=float, - default=2.0, - help="Classifier-free guidance value for VoxCPM.", - ) - parser.add_argument( - "--inference-timesteps", - type=int, - default=10, - help="Number of inference timesteps.", - ) - parser.add_argument( - "--min-len", - type=int, - default=2, - help="Minimum generated token length.", - ) - parser.add_argument( - "--max-new-tokens", - type=int, - default=4096, - help="Maximum generated token length.", - ) - parser.add_argument( - "--streaming-prefix-len", - type=int, - default=None, - help="VoxCPM streaming window (optional, streaming mode only).", - ) - parser.add_argument( - "--output-dir", - type=str, - default=None, - help="Directory for output WAV files.", - ) - parser.add_argument( - "--stage-init-timeout", - type=int, - default=600, - help="Stage initialization timeout in seconds.", - ) - parser.add_argument( - "--log-stats", - dest="log_stats", - action="store_true", - help="Enable vLLM Omni stats logging.", - ) - parser.add_argument( - "--no-log-stats", - dest="log_stats", - action="store_false", - help="Disable vLLM Omni stats logging.", - ) - parser.set_defaults(log_stats=True) - parser.add_argument( - "--num-runs", - type=int, - default=1, - help="Number of full inference runs (same prompt each time). Default 1.", - ) - parser.add_argument( - "--warmup-runs", - type=int, - default=0, - help=( - "Optional number of warmup passes before measured runs. Warmup uses only " - "the first prompt and does not save outputs." - ), - ) - parser.add_argument( - "--enable-profiler", - action="store_true", - help=( - "Enable torch profiler for the configured stages. A temporary profiled " - "stage config is generated automatically." - ), - ) - parser.add_argument( - "--profiler-dir", - type=str, - default=None, - help="Directory for profiler traces. Defaults to /profiler when profiling is enabled.", - ) - parser.add_argument( - "--profiler-stages", - type=int, - nargs="*", - default=None, - help="Optional stage ids to profile. Defaults to all stages that have profiler_config.", - ) - parser.add_argument( - "--profiler-wait-seconds", - type=float, - default=30.0, - help="Seconds to wait after stop_profile for trace files to flush.", - ) - args = parser.parse_args() - - if not args.model: - parser.error("--model is required unless $VOXCPM_MODEL is set") - if args.txt_prompts is not None and args.jsonl_prompts is not None: - parser.error("--txt-prompts and --jsonl-prompts are mutually exclusive") - if (args.ref_audio is None) != (args.ref_text is None): - parser.error("--ref-audio and --ref-text must be provided together") - if args.num_runs < 1: - parser.error("--num-runs must be >= 1") - if args.warmup_runs < 0: - parser.error("--warmup-runs must be >= 0") - if args.output_dir is None: - args.output_dir = ( - "output_audio_streaming" if _is_streaming_stage_config(args.stage_configs_path) else "output_audio" - ) - if args.enable_profiler and args.profiler_dir is None: - args.profiler_dir = str(Path(args.output_dir) / "profiler") - try: - args.prompt_specs = _load_prompt_specs(args) - except ValueError as exc: - parser.error(str(exc)) - - return args - - -def _is_streaming_stage_config(stage_configs_path: str) -> bool: - cfg_name = Path(stage_configs_path).name.lower() - return "async_chunk" in cfg_name - - -async def _collect_streaming_audio( - omni: AsyncOmni, - args: Any, - spec: PromptSpec, - request_id: str, - *, - phase_label: str, - prompt_index: int, - prompt_count: int, - print_prompt: bool = False, -) -> tuple[torch.Tensor, int, float, float | None]: - prompt = _build_prompt_for_spec(args, spec, global_request_id=request_id) - delta_chunks: list[torch.Tensor] = [] - sample_rate = 24000 - chunk_i = 0 - prev_total_samples = 0 - t_start = time.perf_counter() - first_audio_elapsed: float | None = None - - if print_prompt: - print(f"---prompt---:{prompt}") - - async for stage_output in omni.generate(prompt, request_id=request_id): - mm = getattr(stage_output, "multimodal_output", None) - if not isinstance(mm, dict): - ro = getattr(stage_output, "request_output", None) - if ro is None: - continue - mm = getattr(ro, "multimodal_output", None) - if not isinstance(mm, dict) and getattr(ro, "outputs", None): - seq = ro.outputs[0] - mm = getattr(seq, "multimodal_output", None) - if not isinstance(mm, dict): - continue - sample_rate = _extract_sample_rate(mm) - try: - w = _extract_audio_tensor(mm) - n = int(w.numel()) - if n == 0: - continue - finished = _extract_stream_finished(stage_output) - if n > prev_total_samples: - delta = w.reshape(-1)[prev_total_samples:] - prev_total_samples = n - elif finished and n == prev_total_samples: - delta = w.reshape(-1)[:0] - else: - delta = w.reshape(-1) - prev_total_samples += int(delta.numel()) - if int(delta.numel()) > 0: - delta_chunks.append(delta) - if first_audio_elapsed is None and int(delta.numel()) > 0: - first_audio_elapsed = time.perf_counter() - t_start - logger.info( - "%s prompt=%d/%d chunk=%d delta_samples=%d buf_len=%d finished=%s", - phase_label, - prompt_index + 1, - prompt_count, - chunk_i, - int(delta.numel()), - n, - finished, - ) - chunk_i += 1 - except ValueError: - if not _extract_stream_finished(stage_output): - logger.debug("skip non-audio partial output chunk=%d", chunk_i) - - if not delta_chunks: - raise RuntimeError("No audio chunks received; check stage config and logs.") - - audio_cat = torch.cat([c.reshape(-1) for c in delta_chunks], dim=0) - elapsed = time.perf_counter() - t_start - return audio_cat, sample_rate, elapsed, first_audio_elapsed - - -async def _abort_streaming_residual_work( - omni: AsyncOmni, - request_id: str, - *, - settle_seconds: float = 0.1, -) -> None: - """Stop any late stage-0 work once the final audio has been collected.""" - await omni.engine.abort_async([request_id]) - if settle_seconds > 0: - await asyncio.sleep(settle_seconds) - - -async def _run_streaming_single( - omni: AsyncOmni, - args: Any, - spec: PromptSpec, - output_dir: Path, - request_id: str, - *, - run_index: int, - num_runs: int, - prompt_index: int, - prompt_count: int, -) -> Path: - audio_cat, sample_rate, elapsed, first_audio_elapsed = await _collect_streaming_audio( - omni, - args, - spec, - request_id, - phase_label=f"run={run_index + 1}/{num_runs}", - prompt_index=prompt_index, - prompt_count=prompt_count, - print_prompt=(run_index == 0 and prompt_index == 0), - ) - await _abort_streaming_residual_work(omni, request_id) - output_path = output_dir / f"output_run{run_index + 1}_{spec.label}.wav" - _write_audio_tensor(output_path, audio_cat, sample_rate) - audio_duration_s = float(audio_cat.numel()) / float(sample_rate) if sample_rate > 0 else 0.0 - ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else "" - rtf_text = f", rtf={elapsed / audio_duration_s:.3f}" if audio_duration_s > 0 else "" - print( - f"Saved (streaming) run {run_index + 1}/{num_runs}, " - f"prompt {prompt_index + 1}/{prompt_count}: {output_path} ({elapsed:.2f}s{ttfp_text}{rtf_text})" - ) - _emit_offline_metrics( - request_id=request_id, - elapsed_s=elapsed, - first_audio_elapsed=first_audio_elapsed, - audio_duration_s=audio_duration_s, - ) - return output_path - - -async def _run_streaming_warmup(args, omni: AsyncOmni) -> None: - if args.warmup_runs == 0: - return - - warmup_specs = _get_warmup_specs(args.prompt_specs) - print( - f"Warmup: {args.warmup_runs} run(s) using the first prompt " - f"({len(warmup_specs)} prompt(s)); outputs will be discarded." - ) - for warmup_index in range(args.warmup_runs): - t_warmup = time.perf_counter() - tasks = [] - request_ids: list[str] = [] - for prompt_index, spec in enumerate(warmup_specs): - request_id = f"warmup_stream_{warmup_index + 1}_{spec.label}_{uuid.uuid4().hex[:8]}" - request_ids.append(request_id) - tasks.append( - _collect_streaming_audio( - omni, - args, - spec, - request_id, - phase_label=f"warmup={warmup_index + 1}/{args.warmup_runs}", - prompt_index=prompt_index, - prompt_count=len(warmup_specs), - ) - ) - results = await asyncio.gather(*tasks) - for request_id in request_ids: - await _abort_streaming_residual_work(omni, request_id) - total_samples = sum(int(audio.numel()) for audio, _, _, _ in results) - warmup_ttfps = [ttfp for _, _, _, ttfp in results if ttfp is not None] - ttfp_text = f", ttfp={min(warmup_ttfps):.2f}s" if warmup_ttfps else "" - print( - f"Warmup (streaming) {warmup_index + 1}/{args.warmup_runs} finished: " - f"{len(results)} prompt(s), {total_samples} sample(s) " - f"({time.perf_counter() - t_warmup:.2f}s{ttfp_text})" - ) - - -async def _run_streaming(args) -> list[Path]: - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - omni = AsyncOmni( - model=args.model, - stage_configs_path=args.stage_configs_path, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) - - await _run_streaming_warmup(args, omni) - profiler_started = False - if args.enable_profiler: - profile_prefix = f"voxcpm_streaming_{int(time.time())}" - stages_text = args.profiler_stages if args.profiler_stages is not None else "all-configured" - print(f"Starting profiler (streaming): stages={stages_text}, dir={args.profiler_dir}") - await omni.start_profile(profile_prefix=profile_prefix, stages=args.profiler_stages) - profiler_started = True - t_total = time.perf_counter() - total_elapsed = 0.0 - paths: list[Path] = [] - prompt_specs: list[PromptSpec] = args.prompt_specs - try: - for run in range(args.num_runs): - for prompt_index, spec in enumerate(prompt_specs): - request_id = f"stream_{run + 1}_{spec.label}_{uuid.uuid4().hex[:8]}" - paths.append( - await _run_streaming_single( - omni, - args, - spec, - output_dir, - request_id, - run_index=run, - num_runs=args.num_runs, - prompt_index=prompt_index, - prompt_count=len(prompt_specs), - ) - ) - total_elapsed = time.perf_counter() - t_total - finally: - if profiler_started: - print("Stopping profiler (streaming)...") - await omni.stop_profile(stages=args.profiler_stages) - if args.profiler_wait_seconds > 0: - print(f"Waiting {args.profiler_wait_seconds:.1f}s for profiler traces to flush...") - await asyncio.sleep(args.profiler_wait_seconds) - - print( - f"All streaming runs finished: {args.num_runs} run(s), " - f"{len(prompt_specs)} prompt(s), {len(paths)} file(s) in {total_elapsed:.2f}s total" - ) - return paths - - -def _run_sync(args) -> list[Path]: - output_dir = Path(args.output_dir) - - omni = Omni( - model=args.model, - stage_configs_path=args.stage_configs_path, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) - - def _run_sync_single( - spec: PromptSpec, - *, - request_prefix: str, - save_outputs: bool, - run_index: int | None = None, - ) -> tuple[list[Path], int, float | None, float, float, str]: - global_request_id = f"{request_prefix}_{spec.label}" - prompt = _build_prompt_for_spec(args, spec, global_request_id=global_request_id) - if save_outputs and run_index == 0 and spec.label == "item001": - print(f"---prompt---:{prompt}") - - saved_paths: list[Path] = [] - output_count = 0 - first_audio_elapsed: float | None = None - total_audio_duration_s = 0.0 - metrics_request_id = global_request_id - t_start = time.perf_counter() - for stage_outputs in omni.generate(prompt): - request_output = stage_outputs.request_output - if request_output is None: - continue - request_output_id = getattr(request_output, "request_id", None) - if isinstance(request_output_id, str) and request_output_id: - metrics_request_id = request_output_id - for j, mm in enumerate(_iter_request_multimodal_outputs(request_output)): - output_count += 1 - if first_audio_elapsed is None: - try: - audio_tensor = _extract_audio_tensor(mm) - if int(audio_tensor.numel()) > 0: - first_audio_elapsed = time.perf_counter() - t_start - total_audio_duration_s += float(audio_tensor.numel()) / float(_extract_sample_rate(mm)) - except ValueError: - pass - else: - try: - audio_tensor = _extract_audio_tensor(mm) - total_audio_duration_s += float(audio_tensor.numel()) / float(_extract_sample_rate(mm)) - except ValueError: - pass - if not save_outputs: - continue - save_stem = f"run{run_index + 1}_{spec.label}" if j == 0 else f"run{run_index + 1}_{spec.label}_{j}" - saved_paths.append(_save_wav(mm, output_dir, save_stem)) - - if output_count == 0: - raise RuntimeError("No output from Omni.generate") - elapsed_s = time.perf_counter() - t_start - return saved_paths, output_count, first_audio_elapsed, elapsed_s, total_audio_duration_s, metrics_request_id - - if args.warmup_runs: - warmup_specs = _get_warmup_specs(args.prompt_specs) - print( - f"Warmup: {args.warmup_runs} run(s) using the first prompt " - f"({len(warmup_specs)} prompt(s)); outputs will be discarded." - ) - for warmup_index in range(args.warmup_runs): - t_warmup = time.perf_counter() - _, output_count, first_audio_elapsed, elapsed_s, audio_duration_s, _ = _run_sync_single( - warmup_specs[0], - request_prefix=f"warmup_sync{warmup_index + 1}", - save_outputs=False, - ) - ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else "" - rtf_text = f", rtf={elapsed_s / audio_duration_s:.3f}" if audio_duration_s > 0 else "" - print( - f"Warmup (sync) {warmup_index + 1}/{args.warmup_runs} finished: " - f"{output_count} output(s) ({time.perf_counter() - t_warmup:.2f}s{ttfp_text}{rtf_text})" - ) - - profiler_started = False - if args.enable_profiler: - profile_prefix = f"voxcpm_sync_{int(time.time())}" - stages_text = args.profiler_stages if args.profiler_stages is not None else "all-configured" - print(f"Starting profiler (sync): stages={stages_text}, dir={args.profiler_dir}") - omni.start_profile(profile_prefix=profile_prefix, stages=args.profiler_stages) - profiler_started = True - - t_total = time.perf_counter() - total_elapsed = 0.0 - saved_paths: list[Path] = [] - prompt_specs: list[PromptSpec] = args.prompt_specs - try: - for run in range(args.num_runs): - t_run = time.perf_counter() - run_paths: list[Path] = [] - for prompt_index, spec in enumerate(prompt_specs): - prompt_paths, _, first_audio_elapsed, elapsed_s, audio_duration_s, metrics_request_id = ( - _run_sync_single( - spec, - request_prefix=f"sync_run{run + 1}_{prompt_index + 1:03d}", - save_outputs=True, - run_index=run, - ) - ) - run_paths.extend(prompt_paths) - ttfp_text = f", ttfp={first_audio_elapsed:.2f}s" if first_audio_elapsed is not None else "" - rtf_text = f", rtf={elapsed_s / audio_duration_s:.3f}" if audio_duration_s > 0 else "" - print( - f"Saved (sync) run {run + 1}/{args.num_runs}, " - f"prompt {prompt_index + 1}/{len(prompt_specs)}: {len(prompt_paths)} file(s){ttfp_text}{rtf_text}" - ) - _emit_offline_metrics( - request_id=metrics_request_id, - elapsed_s=elapsed_s, - first_audio_elapsed=first_audio_elapsed, - audio_duration_s=audio_duration_s, - ) - - saved_paths.extend(run_paths) - print( - f"Run {run + 1}/{args.num_runs} finished: {len(run_paths)} file(s) ({time.perf_counter() - t_run:.2f}s)" - ) - for path in run_paths: - print(f" {path}") - - total_elapsed = time.perf_counter() - t_total - finally: - if profiler_started: - print("Stopping profiler (sync)...") - omni.stop_profile(stages=args.profiler_stages) - if args.profiler_wait_seconds > 0: - print(f"Waiting {args.profiler_wait_seconds:.1f}s for profiler traces to flush...") - time.sleep(args.profiler_wait_seconds) - - print( - f"All sync runs finished: {args.num_runs} run(s), " - f"{len(prompt_specs)} prompt(s), {len(saved_paths)} file(s) in {total_elapsed:.2f}s total" - ) - return saved_paths - - -def main(args) -> int: - logging.basicConfig(level=logging.INFO) - profiled_stage_config_path: str | None = None - original_stage_config_path = args.stage_configs_path - if args.enable_profiler: - Path(args.profiler_dir).mkdir(parents=True, exist_ok=True) - profiled_stage_config_path = _build_profiled_stage_config( - args.stage_configs_path, - str(Path(args.profiler_dir).resolve()), - ) - args.stage_configs_path = profiled_stage_config_path - - is_streaming = _is_streaming_stage_config(args.stage_configs_path) - voice_clone_count = _count_voice_clone_prompts(args.prompt_specs) - print(f"Model: {args.model}") - print(f"Stage config: {original_stage_config_path}") - print(f"Route: {'streaming' if is_streaming else 'sync'} (from stage-configs-path)") - print(f"Prompt count: {len(args.prompt_specs)}") - print("Batch mode: sequential (aligned with native VoxCPM)") - print(f"Warmup runs: {args.warmup_runs}") - print(f"Voice cloning prompts: {voice_clone_count}/{len(args.prompt_specs)}") - if args.enable_profiler: - print(f"Profiler: enabled (dir={args.profiler_dir}, stages={args.profiler_stages or 'all-configured'})") - print(f"Profiled stage config: {args.stage_configs_path}") - if voice_clone_count: - print("Voice cloning note: --ref-text/ref_text must match the spoken content of the reference audio.") - print(f"Num runs: {args.num_runs}") - try: - if is_streaming: - asyncio.run(_run_streaming(args)) - else: - _run_sync(args) - finally: - if profiled_stage_config_path is not None and os.path.exists(profiled_stage_config_path): - os.unlink(profiled_stage_config_path) - return 0 - - -if __name__ == "__main__": - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - raise SystemExit(main(parse_args())) diff --git a/docs/api/README.md b/docs/api/README.md index 11401dc02..dce02e13d 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -121,7 +121,6 @@ Configuration classes. - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][] - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][] - [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][] -- [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][] - [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][] ## Workers diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 60d6df718..23f89bad5 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -5,6 +5,9 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. - **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz. - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz. - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz. +- **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz. + +See the [Supported Models](#supported-models) section below for the full list, including OmniVoice, VoxCPM2, and MOSS-TTS-Nano. Each server instance runs a single model (specified at startup via `vllm serve --omni`). @@ -26,6 +29,10 @@ vllm serve fishaudio/s2-pro --omni --port 8091 # Voxtral TTS vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091 + +# CosyVoice3 (voice cloning only — supply ref_audio + ref_text per request) +vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 \ + --omni --port 8091 --trust-remote-code ``` ### Generate Speech diff --git a/docs/user_guide/examples/offline_inference/text_to_speech.md b/docs/user_guide/examples/offline_inference/text_to_speech.md index e88a68220..7ad3d2d40 100644 --- a/docs/user_guide/examples/offline_inference/text_to_speech.md +++ b/docs/user_guide/examples/offline_inference/text_to_speech.md @@ -17,7 +17,6 @@ list of supported architectures across all modalities, see | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate | |---|---|---|---|---|---|---| | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz | -| VoxCPM | local model dir | split | ✓ | ✓ (`voxcpm_async_chunk.yaml`) | — | 24 kHz | | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz | | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz | | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design (`--instruct`), language (`--lang`) | 24 kHz | @@ -72,61 +71,6 @@ python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \ --- -## VoxCPM - -Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement. - -### Prerequisites -```bash -pip install voxcpm soundfile -# or use a local source tree: -export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src -``` - -If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`: - -```bash -export VOXCPM_MODEL=/path/to/voxcpm-model -export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config -mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH" -cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json" -cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true -python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)' -``` - -### Quick start -```bash -python examples/offline_inference/text_to_speech/voxcpm/end2end.py \ - --model "$VOXCPM_MODEL" \ - --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni." -``` - -### Voice cloning -```bash -python examples/offline_inference/text_to_speech/voxcpm/end2end.py \ - --model "$VOXCPM_MODEL" \ - --text "This sentence is synthesized with a cloned voice." \ - --ref-audio /path/to/reference.wav \ - --ref-text "The exact transcript spoken in reference.wav." -``` - -### Streaming -Pass the async-chunk stage config: -```bash -python examples/offline_inference/text_to_speech/voxcpm/end2end.py \ - --model "$VOXCPM_MODEL" \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \ - --text "This is a split-stage VoxCPM streaming example running on vLLM Omni." -``` - -### Notes -- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming. -- Streaming is currently single-request oriented. -- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality. -- For online serving, see [`examples/online_serving/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm/README.md). For benchmark reporting, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md). - ---- - ## CosyVoice3 2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz. @@ -381,10 +325,6 @@ Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-T ``````py --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/end2end.py" `````` -??? abstract "voxcpm/end2end.py" - ``````py - --8<-- "examples/offline_inference/text_to_speech/voxcpm/end2end.py" - `````` ??? abstract "voxcpm2/end2end.py" ``````py --8<-- "examples/offline_inference/text_to_speech/voxcpm2/end2end.py" diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index 703b7cf7a..ff22dd0fd 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -17,14 +17,14 @@ For the full list of supported architectures across all modalities, see | Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo | |---|---|---|---|---|---| +| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | — | | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ | | OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) | -| VoxCPM | local model dir | ✓ | ✓ (PCM stream) | — | — | | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ | -CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3) instead. +For offline inference of any of these models, see the [offline TTS hub](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md). ## Common Quick Start @@ -96,6 +96,54 @@ For full request-shape documentation (all parameters, response formats, error co --- +## CosyVoice3 + +2-stage TTS (`talker` + flow-matching `code2wav`) at 24 kHz. Voice cloning only — every request needs `ref_audio` + `ref_text`; there are no built-in voice presets. + +### Prerequisites +```bash +huggingface-cli download FunAudioLLM/Fun-CosyVoice3-0.5B-2512 +``` + +If your downloaded checkpoint lacks `config.json`, add one with `{"model_type": "cosyvoice3", "architectures": ["CosyVoice3Model"]}` (the loader reads `model_type` to select the class). + +### Launch +```bash +vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code +# or: +./cosyvoice3/run_server.sh +``` + +Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path. + +### CLI client +The client defaults to the official upstream zero-shot prompt, so it runs without extra flags: +```bash +cd examples/online_serving/text_to_speech/cosyvoice3 +python speech_client.py --text "收到好友从远方寄来的生日礼物。" +``` + +Pass your own reference clip and transcript for a different voice: +```bash +python speech_client.py --text "Hello, this is a cloned voice." \ + --ref-audio /path/to/reference.wav \ + --ref-text "Transcript of the reference audio." +``` + +Stream PCM instead of WAV: +```bash +python speech_client.py --text "Hello world" --stream --output output.pcm +``` + +The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-text`, `--response-format`, `--stream`, `--output`. + +### Notes +- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform. +- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config ` to override. +- For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3). + +--- + ## Fish Speech S2 Pro 4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec. @@ -196,10 +244,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091 ./qwen3_tts/run_server.sh Base ``` -### Choosing an executor backend (uniproc vs mp) -Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation. +### Executor backend +Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config ` to override. -To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor. +To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor. ### Sending requests ```bash @@ -281,71 +329,11 @@ python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000 `qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement. ### Notes -- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above. +- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background. - `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N- `. --- -## VoxCPM - -Split-stage TTS at 24 kHz. - -### Prerequisites -```bash -pip install voxcpm -# or use a local source tree: -export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src -``` - -If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir: -```bash -export VOXCPM_MODEL=/path/to/voxcpm-model -export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config -mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH" -cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json" -cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true -python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)' -``` - -### Launch -```bash -export VOXCPM_MODEL=/path/to/voxcpm-model -./voxcpm/run_server.sh # async-chunk streaming (default) -./voxcpm/run_server.sh sync # non-streaming -``` -Or directly: -```bash -vllm serve "$VOXCPM_MODEL" \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \ - --trust-remote-code --enforce-eager --omni --port 8091 -``` - -### Sending requests -```bash -# Basic TTS -python voxcpm/openai_speech_client.py \ - --model "$VOXCPM_MODEL" \ - --text "This is a VoxCPM online text-to-speech example." - -# Voice cloning -python voxcpm/openai_speech_client.py \ - --model "$VOXCPM_MODEL" \ - --text "This sentence is synthesized with a cloned voice." \ - --ref-audio /path/to/reference.wav \ - --ref-text "The exact transcript spoken in reference.wav." - -# Streaming PCM -python voxcpm/openai_speech_client.py \ - --model "$VOXCPM_MODEL" \ - --text "This is a streaming VoxCPM request." \ - --stream --output voxcpm_stream.pcm -``` - -### Notes -- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`. -- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`. -- For benchmark measurement, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md). - --- ## VoxCPM2 @@ -407,6 +395,14 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts ## Example materials +??? abstract "cosyvoice3/run_server.sh" + + --8<-- "examples/online_serving/text_to_speech/cosyvoice3/run_server.sh" + +??? abstract "cosyvoice3/speech_client.py" + + --8<-- "examples/online_serving/text_to_speech/cosyvoice3/speech_client.py" + ??? abstract "fish_speech/gradio_demo.py" ``````py --8<-- "examples/online_serving/text_to_speech/fish_speech/gradio_demo.py" @@ -463,14 +459,6 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts ``````py --8<-- "examples/online_serving/text_to_speech/qwen3_tts/tts_common.py" `````` -??? abstract "voxcpm/openai_speech_client.py" - ``````py - --8<-- "examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py" - `````` -??? abstract "voxcpm/run_server.sh" - ``````sh - --8<-- "examples/online_serving/text_to_speech/voxcpm/run_server.sh" - `````` ??? abstract "voxcpm2/gradio_demo.py" ``````py --8<-- "examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py" diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index bfdee543c..722b9eeb3 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -14,13 +14,12 @@ list of supported architectures across all modalities, see | Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate | |---|---|---|---|---|---|---| -| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 22.05 kHz | +| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz | | Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz | | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | single (talker only) | — (caption-controlled) | — | style / IP / basic captions | 44.1 kHz | | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | single (AR + codec) | ✓ (required) | ✓ | voice_clone, continuation | 48 kHz | | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz | -| VoxCPM | `openbmb/VoxCPM-0.5B` | split | ✓ | ✓ | — | 24 kHz | | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz | @@ -41,7 +40,7 @@ python examples/offline_inference/text_to_speech//end2end.py \ ## CosyVoice3 -2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz. +2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz. ### Prerequisites ```bash @@ -308,56 +307,6 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \ --- -## VoxCPM - -Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement. - -### Prerequisites -```bash -pip install voxcpm soundfile -# or use a local source tree: -export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src -``` - -### Quick start -```bash -python examples/offline_inference/text_to_speech/voxcpm/end2end.py \ - --model openbmb/VoxCPM-0.5B \ - --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni." -``` - -### Voice cloning -```bash -python examples/offline_inference/text_to_speech/voxcpm/end2end.py \ - --model openbmb/VoxCPM-0.5B \ - --text "This sentence is synthesized with a cloned voice." \ - --ref-audio /path/to/reference.wav \ - --ref-text "The exact transcript spoken in reference.wav." -``` - -### Streaming -Pass `--streaming` together with the legacy async-chunk stage config: -```bash -python examples/offline_inference/text_to_speech/voxcpm/end2end.py \ - --model openbmb/VoxCPM-0.5B \ - --streaming \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \ - --text "This is a split-stage VoxCPM streaming example running on vLLM Omni." -``` - -### Persistent HF config (optional) -The engine auto-patches a missing `model_type` in the checkpoint's `config.json` in-memory at startup (see `vllm_omni/engine/arg_utils.py`), so no manual setup is required. - -If you'd rather maintain a persistent patched config dir (e.g. to share across runs or to add fields the auto-patcher doesn't set), export `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` to its location — both the bundled deploy and async-chunk yamls read it via `${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}`. - -### Notes -- Non-streaming auto-loads `vllm_omni/deploy/voxcpm.yaml` from the model directory name / HF `model_type`. Streaming uses the legacy `voxcpm_async_chunk.yaml` passed via `--stage-configs-path`. -- Streaming is currently single-request oriented. -- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality. -- For online serving, see the [VoxCPM section in the online hub](../../online_serving/text_to_speech/README.md#voxcpm). For benchmark reporting, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md). - ---- - ## VoxCPM2 Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`. diff --git a/examples/offline_inference/text_to_speech/voxcpm/end2end.py b/examples/offline_inference/text_to_speech/voxcpm/end2end.py deleted file mode 100644 index 79efa3123..000000000 --- a/examples/offline_inference/text_to_speech/voxcpm/end2end.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Minimal offline VoxCPM example for vLLM Omni.""" - -from __future__ import annotations - -import asyncio -import time -from pathlib import Path -from typing import Any - -import soundfile as sf -import torch -from vllm.utils.argparse_utils import FlexibleArgumentParser - -from vllm_omni import AsyncOmni, Omni -from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults - - -def _build_prompt(args) -> dict[str, Any]: - additional_information: dict[str, list[Any]] = { - "text": [args.text], - "cfg_value": [args.cfg_value], - "inference_timesteps": [args.inference_timesteps], - "min_len": [args.min_len], - "max_new_tokens": [args.max_new_tokens], - } - if args.streaming_prefix_len is not None: - additional_information["streaming_prefix_len"] = [args.streaming_prefix_len] - if args.ref_audio is not None: - additional_information["ref_audio"] = [args.ref_audio] - if args.ref_text is not None: - additional_information["ref_text"] = [args.ref_text] - return { - "prompt_token_ids": [1], - "additional_information": additional_information, - } - - -def _extract_audio_tensor(mm: dict[str, Any]) -> torch.Tensor: - audio = mm.get("audio", mm.get("model_outputs")) - if audio is None: - raise ValueError("No audio output found in multimodal output.") - if isinstance(audio, list): - parts = [torch.as_tensor(item).float().cpu().reshape(-1) for item in audio] - audio = torch.cat(parts, dim=-1) if parts else torch.zeros(0) - if not isinstance(audio, torch.Tensor): - audio = torch.as_tensor(audio) - return audio.float().cpu().reshape(-1) - - -def _extract_sample_rate(mm: dict[str, Any]) -> int: - sr_raw = mm.get("sr", 24000) - if isinstance(sr_raw, list) and sr_raw: - sr_raw = sr_raw[-1] - if hasattr(sr_raw, "item"): - return int(sr_raw.item()) - return int(sr_raw) - - -def _save_audio(audio: torch.Tensor, sample_rate: int, output_dir: Path, request_id: str) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - output_path = output_dir / f"output_{request_id}.wav" - sf.write( - output_path, - audio.float().cpu().clamp(-1.0, 1.0).numpy(), - sample_rate, - format="WAV", - subtype="PCM_16", - ) - return output_path - - -async def _run_streaming(args) -> Path: - prompt = _build_prompt(args) - output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio_streaming") - request_id = "streaming_example" - sample_rate = 24000 - buffered_samples = 0 - chunks: list[torch.Tensor] = [] - started = time.perf_counter() - omni = AsyncOmni( - model=args.model, - stage_configs_path=args.stage_configs_path, - deploy_config=args.deploy_config, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) - try: - async for stage_output in omni.generate(prompt, request_id=request_id): - mm = getattr(stage_output, "multimodal_output", None) - if not isinstance(mm, dict): - request_output = getattr(stage_output, "request_output", None) - if request_output is None: - continue - mm = getattr(request_output, "multimodal_output", None) - if not isinstance(mm, dict) and getattr(request_output, "outputs", None): - mm = getattr(request_output.outputs[0], "multimodal_output", None) - if not isinstance(mm, dict): - continue - audio = _extract_audio_tensor(mm) - if audio.numel() == 0: - continue - sample_rate = _extract_sample_rate(mm) - if audio.numel() > buffered_samples: - delta = audio[buffered_samples:] - buffered_samples = int(audio.numel()) - else: - delta = audio - buffered_samples += int(delta.numel()) - if delta.numel() > 0: - chunks.append(delta) - if not chunks: - raise RuntimeError("No streaming audio chunks received from VoxCPM.") - output_audio = torch.cat(chunks, dim=0) - output_path = _save_audio(output_audio, sample_rate, output_dir, request_id) - print(f"Saved streaming audio to: {output_path} ({time.perf_counter() - started:.2f}s)") - return output_path - finally: - omni.shutdown() - - -def _run_sync(args) -> Path: - prompt = _build_prompt(args) - output_dir = Path(args.output_dir) if args.output_dir is not None else Path("output_audio") - request_id = "sync_example" - started = time.perf_counter() - last_mm: dict[str, Any] | None = None - omni = Omni( - model=args.model, - stage_configs_path=args.stage_configs_path, - deploy_config=args.deploy_config, - log_stats=args.log_stats, - stage_init_timeout=args.stage_init_timeout, - ) - for stage_outputs in omni.generate(prompt): - request_output = getattr(stage_outputs, "request_output", None) - if request_output is None: - continue - outputs = getattr(request_output, "outputs", None) - if outputs: - for output in outputs: - mm = getattr(output, "multimodal_output", None) - if isinstance(mm, dict): - last_mm = mm - mm = getattr(request_output, "multimodal_output", None) - if isinstance(mm, dict): - last_mm = mm - if last_mm is None: - raise RuntimeError("No audio output received from VoxCPM.") - output_path = _save_audio( - _extract_audio_tensor(last_mm), - _extract_sample_rate(last_mm), - output_dir, - request_id, - ) - print(f"Saved audio to: {output_path} ({time.perf_counter() - started:.2f}s)") - return output_path - - -def parse_args(): - parser = FlexibleArgumentParser(description="Minimal offline VoxCPM example for vLLM Omni.") - parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.") - parser.add_argument( - "--deploy-config", - type=str, - default=None, - help=("Override the deploy config path."), - ) - parser.add_argument( - "--stage-configs-path", - type=str, - default=None, - help=( - "Legacy stage_args yaml path. Required for streaming " - "(vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml); " - "leave unset for non-streaming to use the auto-resolved deploy config." - ), - ) - parser.add_argument( - "--streaming", - action="store_true", - default=False, - help="Stream audio chunks as they arrive via AsyncOmni.", - ) - parser.add_argument("--text", type=str, required=True, help="Input text for synthesis.") - parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.") - parser.add_argument("--ref-text", type=str, default=None, help="Transcript of the reference audio.") - parser.add_argument("--output-dir", type=str, default=None, help="Output directory for generated wav files.") - parser.add_argument("--cfg-value", type=float, default=2.0, help="Guidance value passed to VoxCPM.") - parser.add_argument("--inference-timesteps", type=int, default=10, help="Number of diffusion timesteps.") - parser.add_argument("--min-len", type=int, default=2, help="Minimum latent length.") - parser.add_argument("--max-new-tokens", type=int, default=4096, help="Maximum latent length.") - parser.add_argument( - "--streaming-prefix-len", - type=int, - default=3, - help="Streaming prefix length used by voxcpm_async_chunk.yaml.", - ) - parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.") - parser.add_argument("--log-stats", action="store_true", help="Enable vLLM Omni stats logging.") - nullify_stage_engine_defaults(parser) - args = parser.parse_args() - if (args.ref_audio is None) != (args.ref_text is None): - raise ValueError("Voice cloning requires --ref-audio and --ref-text together.") - return args - - -def main(args) -> None: - route = "streaming" if args.streaming else "sync" - print(f"Model: {args.model}") - print(f"Deploy config: {args.deploy_config or ''}") - print(f"Stage configs path: {args.stage_configs_path or ''}") - print(f"Route: {route}") - if route == "streaming": - asyncio.run(_run_streaming(args)) - else: - _run_sync(args) - - -if __name__ == "__main__": - main(parse_args()) diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index 8481e723e..75b85580d 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -19,7 +19,6 @@ For the full list of supported architectures across all modalities, see | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ | | OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) | -| VoxCPM | `openbmb/VoxCPM-0.5B` | ✓ | ✓ (PCM stream) | — | — | | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ | @@ -294,10 +293,10 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091 ./qwen3_tts/run_server.sh Base ``` -### Choosing an executor backend (uniproc vs mp) -Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation. +### Executor backend +Single-GPU serves now default to the uniproc executor (lower IPC overhead, the Base cloning use case from [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) / [#2604](https://github.com/vllm-project/vllm-omni/pull/2604)). `vllm_omni/deploy/qwen3_tts.yaml` is the only Qwen3-TTS deploy config; pass `--deploy-config ` to override. -To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor. +To opt out of chunked streaming, pass `--no-async-chunk` — the pipeline auto-dispatches to the end-to-end codec processor. ### Sending requests ```bash @@ -375,73 +374,11 @@ python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-de `qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement. ### Notes -- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above. +- Base voice cloning has per-request reference-audio cost; the uniproc default keeps IPC overhead off the critical path. See the executor-backend section above for background. - `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N- `. --- -## VoxCPM - -Split-stage TTS at 24 kHz. - -### Prerequisites -```bash -pip install voxcpm -# or use a local source tree: -export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src -``` - -If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir: -```bash -export VOXCPM_MODEL=/path/to/voxcpm-model -export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config -mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH" -cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json" -cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true -python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)' -``` - -### Launch -```bash -export VOXCPM_MODEL=/path/to/voxcpm-model -./voxcpm/run_server.sh # async-chunk streaming (default) -./voxcpm/run_server.sh sync # non-streaming -``` -Or directly: -```bash -vllm serve "$VOXCPM_MODEL" \ - --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \ - --trust-remote-code --enforce-eager --omni --port 8091 -``` - -### Sending requests -```bash -# Basic TTS -python voxcpm/openai_speech_client.py \ - --model "$VOXCPM_MODEL" \ - --text "This is a VoxCPM online text-to-speech example." - -# Voice cloning -python voxcpm/openai_speech_client.py \ - --model "$VOXCPM_MODEL" \ - --text "This sentence is synthesized with a cloned voice." \ - --ref-audio /path/to/reference.wav \ - --ref-text "The exact transcript spoken in reference.wav." - -# Streaming PCM -python voxcpm/openai_speech_client.py \ - --model "$VOXCPM_MODEL" \ - --text "This is a streaming VoxCPM request." \ - --stream --output voxcpm_stream.pcm -``` - -### Notes -- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`. -- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`. -- For benchmark measurement, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md). - ---- - ## VoxCPM2 Single-stage native AR TTS at 48 kHz. diff --git a/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh b/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh new file mode 100755 index 000000000..e40b1bb13 --- /dev/null +++ b/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Launch vLLM-Omni server for CosyVoice3 TTS +# +# Usage: +# ./run_server.sh +# CUDA_VISIBLE_DEVICES=0 ./run_server.sh +# +# Streaming (async-chunk) is on by default via vllm_omni/deploy/cosyvoice3.yaml. +# Set NO_ASYNC_CHUNK=1 to use the legacy synchronous path. + +set -e + +MODEL="${MODEL:-FunAudioLLM/Fun-CosyVoice3-0.5B-2512}" +PORT="${PORT:-8091}" + +EXTRA_ARGS=() +if [[ -n "${NO_ASYNC_CHUNK:-}" ]]; then + EXTRA_ARGS+=(--no-async-chunk) +fi + +echo "Starting CosyVoice3 server with model: $MODEL" + +vllm serve "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --trust-remote-code \ + --omni \ + "${EXTRA_ARGS[@]}" diff --git a/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py b/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py new file mode 100644 index 000000000..a837d9243 --- /dev/null +++ b/examples/online_serving/text_to_speech/cosyvoice3/speech_client.py @@ -0,0 +1,142 @@ +"""Client for CosyVoice3 TTS via /v1/audio/speech endpoint. + +CosyVoice3 has no built-in voice presets: every request is voice cloning +driven by ``ref_audio`` + ``ref_text``. The defaults below point at the +official upstream zero-shot prompt so the script runs out of the box. + +Examples: + # Voice cloning with the default upstream prompt + python speech_client.py --text "收到好友从远方寄来的生日礼物。" + + # Custom reference clip + transcript + python speech_client.py --text "Hello, this is a cloned voice." \ + --ref-audio /path/to/reference.wav \ + --ref-text "Transcript of the reference audio." + + # Streaming PCM output + python speech_client.py --text "Hello world" --stream --output output.pcm +""" + +import argparse +import base64 +import os + +import httpx + +DEFAULT_API_BASE = "http://localhost:8091" +DEFAULT_API_KEY = "EMPTY" +DEFAULT_MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512" + +# Official CosyVoice zero-shot prompt and its transcript. +DEFAULT_REF_AUDIO = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav" +DEFAULT_REF_TEXT = "希望你以后能够做的比我还好呦。" + + +def encode_audio_to_base64(audio_path: str) -> str: + """Encode a local audio file to a base64 data URL.""" + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + ext = audio_path.lower().rsplit(".", 1)[-1] + mime_map = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"} + mime_type = mime_map.get(ext, "audio/wav") + with open(audio_path, "rb") as f: + audio_b64 = base64.b64encode(f.read()).decode("utf-8") + return f"data:{mime_type};base64,{audio_b64}" + + +def run_tts(args) -> None: + """Generate speech via the /v1/audio/speech API.""" + payload = { + "model": args.model, + "input": args.text, + "response_format": args.response_format, + } + + if args.ref_audio.startswith(("http://", "https://")): + payload["ref_audio"] = args.ref_audio + else: + payload["ref_audio"] = encode_audio_to_base64(args.ref_audio) + payload["ref_text"] = args.ref_text + + if args.stream: + payload["stream"] = True + payload["response_format"] = "pcm" + + print(f"Model: {args.model}") + print(f"Text: {args.text}") + print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}") + print("Generating audio...") + + api_url = f"{args.api_base}/v1/audio/speech" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {args.api_key}", + } + + if args.stream: + output_path = args.output or "output.pcm" + with httpx.Client(timeout=300.0) as client: + with client.stream("POST", api_url, json=payload, headers=headers) as resp: + if resp.status_code != 200: + print(f"Error: {resp.status_code}") + print(resp.read().decode()) + return + total_bytes = 0 + with open(output_path, "wb") as f: + for chunk in resp.iter_bytes(): + f.write(chunk) + total_bytes += len(chunk) + print(f"Streamed {total_bytes} bytes to: {output_path}") + else: + with httpx.Client(timeout=300.0) as client: + response = client.post(api_url, json=payload, headers=headers) + + if response.status_code != 200: + print(f"Error: {response.status_code}") + print(response.text) + return + + try: + text = response.content.decode("utf-8") + if text.startswith('{"error"'): + print(f"Error: {text}") + return + except UnicodeDecodeError: + pass + + output_path = args.output or "output.wav" + with open(output_path, "wb") as f: + f.write(response.content) + print(f"Audio saved to: {output_path}") + + +def main(): + parser = argparse.ArgumentParser(description="CosyVoice3 TTS client") + parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL") + parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key") + parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name") + parser.add_argument("--text", required=True, help="Text to synthesize") + parser.add_argument( + "--ref-audio", + default=DEFAULT_REF_AUDIO, + help="Reference audio for voice cloning (path or URL)", + ) + parser.add_argument( + "--ref-text", + default=DEFAULT_REF_TEXT, + help="Transcript of the reference audio", + ) + parser.add_argument("--stream", action="store_true", help="Enable streaming (PCM output)") + parser.add_argument( + "--response-format", + default="wav", + choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], + help="Audio format (default: wav)", + ) + parser.add_argument("--output", "-o", default=None, help="Output file path") + args = parser.parse_args() + run_tts(args) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py b/examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py deleted file mode 100644 index c400114e8..000000000 --- a/examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py +++ /dev/null @@ -1,155 +0,0 @@ -"""OpenAI-compatible client for VoxCPM via /v1/audio/speech. - -Examples: - # Basic text-to-speech - python openai_speech_client.py --text "Hello from VoxCPM" - - # Voice cloning - python openai_speech_client.py \ - --text "This sentence uses the cloned voice." \ - --ref-audio /path/to/reference.wav \ - --ref-text "The exact transcript spoken in the reference audio." - - # Streaming PCM output - python openai_speech_client.py \ - --text "This is a streaming VoxCPM request." \ - --stream \ - --output output.pcm -""" - -import argparse -import base64 -import os - -import httpx - -DEFAULT_API_BASE = "http://localhost:8091" -DEFAULT_API_KEY = "EMPTY" -DEFAULT_MODEL = "OpenBMB/VoxCPM1.5" - - -def encode_audio_to_base64(audio_path: str) -> str: - """Encode a local audio file to base64 data URL.""" - if not os.path.exists(audio_path): - raise FileNotFoundError(f"Audio file not found: {audio_path}") - - ext = audio_path.lower().rsplit(".", 1)[-1] - mime_map = { - "wav": "audio/wav", - "mp3": "audio/mpeg", - "flac": "audio/flac", - "ogg": "audio/ogg", - } - mime_type = mime_map.get(ext, "audio/wav") - - with open(audio_path, "rb") as f: - audio_b64 = base64.b64encode(f.read()).decode("utf-8") - return f"data:{mime_type};base64,{audio_b64}" - - -def build_payload(args) -> dict[str, object]: - payload: dict[str, object] = { - "model": args.model, - "input": args.text, - "response_format": "pcm" if args.stream else args.response_format, - } - - if args.ref_audio: - if args.ref_audio.startswith(("http://", "https://", "data:")): - payload["ref_audio"] = args.ref_audio - else: - payload["ref_audio"] = encode_audio_to_base64(args.ref_audio) - if args.ref_text: - payload["ref_text"] = args.ref_text - if args.max_new_tokens is not None: - payload["max_new_tokens"] = args.max_new_tokens - if args.stream: - payload["stream"] = True - - return payload - - -def run_tts(args) -> None: - payload = build_payload(args) - api_url = f"{args.api_base}/v1/audio/speech" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {args.api_key}", - } - - print(f"Model: {args.model}") - print(f"Text: {args.text}") - if args.ref_audio: - print("Mode: voice cloning") - print(f"Reference audio: {args.ref_audio}") - else: - print("Mode: text-to-speech") - - if args.stream: - output_path = args.output or "voxcpm_output.pcm" - with httpx.Client(timeout=300.0) as client: - with client.stream("POST", api_url, json=payload, headers=headers) as response: - if response.status_code != 200: - print(f"Error: {response.status_code}") - print(response.read().decode("utf-8", errors="ignore")) - return - - total_bytes = 0 - with open(output_path, "wb") as f: - for chunk in response.iter_bytes(): - if not chunk: - continue - f.write(chunk) - total_bytes += len(chunk) - print(f"Streamed {total_bytes} bytes to: {output_path}") - return - - with httpx.Client(timeout=300.0) as client: - response = client.post(api_url, json=payload, headers=headers) - - if response.status_code != 200: - print(f"Error: {response.status_code}") - print(response.text) - return - - try: - text = response.content.decode("utf-8") - if text.startswith('{"error"'): - print(f"Error: {text}") - return - except UnicodeDecodeError: - pass - - output_path = args.output or "voxcpm_output.wav" - with open(output_path, "wb") as f: - f.write(response.content) - print(f"Audio saved to: {output_path}") - - -def main(): - parser = argparse.ArgumentParser(description="VoxCPM OpenAI-compatible speech client") - parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL") - parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key") - parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name or path") - parser.add_argument("--text", required=True, help="Text to synthesize") - parser.add_argument("--ref-audio", default=None, help="Reference audio path, URL, or data URL") - parser.add_argument( - "--ref-text", - default=None, - help="The exact transcript spoken in the reference audio", - ) - parser.add_argument("--stream", action="store_true", help="Enable streaming PCM output") - parser.add_argument( - "--response-format", - default="wav", - choices=["wav", "pcm", "flac", "mp3", "aac", "opus"], - help="Audio format for non-streaming mode (default: wav)", - ) - parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate") - parser.add_argument("--output", "-o", default=None, help="Output file path") - args = parser.parse_args() - run_tts(args) - - -if __name__ == "__main__": - main() diff --git a/examples/online_serving/text_to_speech/voxcpm/run_server.sh b/examples/online_serving/text_to_speech/voxcpm/run_server.sh deleted file mode 100755 index c5ac08b9e..000000000 --- a/examples/online_serving/text_to_speech/voxcpm/run_server.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Launch vLLM-Omni server for VoxCPM online speech serving. -# -# Usage: -# ./run_server.sh # default: async_chunk stage config -# ./run_server.sh async # async_chunk stage config -# ./run_server.sh sync # no-async-chunk stage config -# VOXCPM_MODEL=/path/to/model ./run_server.sh - -set -e - -MODE="${1:-async}" -MODEL="${VOXCPM_MODEL:-OpenBMB/VoxCPM1.5}" - -case "$MODE" in - async) - STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml" - ;; - sync) - STAGE_CONFIG="vllm_omni/deploy/voxcpm.yaml" - ;; - *) - echo "Unknown mode: $MODE" - echo "Supported: async, sync" - exit 1 - ;; -esac - -echo "Starting VoxCPM server with model: $MODEL" -echo "Stage config: $STAGE_CONFIG" - -vllm serve "$MODEL" \ - --stage-configs-path "$STAGE_CONFIG" \ - --host 0.0.0.0 \ - --port 8091 \ - --trust-remote-code \ - --enforce-eager \ - --omni diff --git a/pyproject.toml b/pyproject.toml index 95e95fed8..35d333fd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ dev = [ "funasr>=1.0.0", "lpips>=0.1.4", # for quantization fp8 "orjson>=3.11.7", # to make v1/embeddings API fast - "voxcpm>=2.0.2", # for voxcpm model Test + "voxcpm>=2.0.2", # for VoxCPM2 path-locating in tests "torchdiffeq>=0.2.5" # for covo_audio Test ] diff --git a/tests/e2e/offline_inference/test_voxcpm_expansion.py b/tests/e2e/offline_inference/test_voxcpm_expansion.py deleted file mode 100644 index 66639a17f..000000000 --- a/tests/e2e/offline_inference/test_voxcpm_expansion.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""E2E test for VoxCPM offline inference.""" - -from typing import Any - -import numpy as np -import pytest -import torch - -from tests.helpers.mark import hardware_test -from tests.helpers.runtime import OmniRunner -from tests.helpers.stage_config import get_deploy_config_path - -VOXCPM_MODEL = "OpenBMB/VoxCPM1.5" -STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml") -SAMPLE_RATE = 24000 - -# (model, stage_config_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)`` -_OMNI_RUNNER_PARAM = (VOXCPM_MODEL, STAGE_CONFIG) - -pytestmark = [ - pytest.mark.full_model, - pytest.mark.tts, - pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True), -] - - -def _build_prompt(text: str) -> dict[str, Any]: - return { - "prompt_token_ids": [1], - "additional_information": { - "text": [text], - "cfg_value": [2.0], - "inference_timesteps": [10], - "min_len": [2], - "max_new_tokens": [1024], - }, - } - - -def _extract_audio_tensor(multimodal_output: dict[str, Any]) -> torch.Tensor: - audio = multimodal_output.get("audio", multimodal_output.get("model_outputs")) - assert audio is not None, f"No audio output found, keys={list(multimodal_output.keys())}" - - if isinstance(audio, list): - parts: list[torch.Tensor] = [] - for item in audio: - if item is None: - continue - tensor = torch.as_tensor(item) - if tensor.numel() == 0: - continue - parts.append(tensor.float().cpu().reshape(-1)) - return torch.cat(parts, dim=-1) if parts else torch.zeros((0,), dtype=torch.float32) - - return torch.as_tensor(audio).float().cpu().reshape(-1) - - -def _extract_final_multimodal_output(outputs) -> dict[str, Any]: - for item in reversed(outputs): - request_output = getattr(item, "request_output", None) - if request_output is not None: - multimodal_output = getattr(request_output, "multimodal_output", None) - if isinstance(multimodal_output, dict): - return multimodal_output - completions = getattr(request_output, "outputs", None) or [] - for completion in completions: - multimodal_output = getattr(completion, "multimodal_output", None) - if isinstance(multimodal_output, dict): - return multimodal_output - - multimodal_output = getattr(item, "multimodal_output", None) - if isinstance(multimodal_output, dict): - return multimodal_output - - raise AssertionError("No multimodal audio output found in VoxCPM generate results") - - -@hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_voxcpm_zero_shot_001(omni_runner: OmniRunner) -> None: - outputs = list(omni_runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test."))) - - assert outputs, "No outputs returned" - - multimodal_output = _extract_final_multimodal_output(outputs) - audio = _extract_audio_tensor(multimodal_output) - assert audio.numel() > SAMPLE_RATE // 2, f"Audio too short: {audio.numel()} samples" - - duration_s = audio.shape[0] / SAMPLE_RATE - assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s" - - peak = float(torch.max(torch.abs(audio)).item()) if audio.numel() > 0 else 0.0 - assert peak > 0.01, "Generated audio appears to be silence" - - audio_np = audio.numpy() - rms = float(np.sqrt(np.mean(np.square(audio_np)))) if audio_np.size else 0.0 - assert rms > 1e-4, "Generated audio RMS too low" diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 8dec9c5a9..cf77e2256 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -232,24 +232,6 @@ def test_stage_configs_path_field(): assert args.stage_configs_path == "/some/path.yaml" -def test_voxcpm_model_arch_injects_model_type_override(mocker): - """Ensure VoxCPM model_arch injects hf_overrides for config resolution.""" - mocker.patch.object(OmniEngineArgs, "_ensure_omni_models_registered", return_value=True) - mocker.patch.object(OmniEngineArgs, "_patch_empty_hf_config") - mocker.patch.object(EngineArgs, "create_model_config", return_value=Mock()) - mocker.patch.object(OmniModelConfig, "from_vllm_model_config", return_value=Mock()) - - args = OmniEngineArgs( - model="OpenBMB/VoxCPM1.5", - model_arch="VoxCPMForConditionalGeneration", - ) - args.create_model_config() - - assert args.hf_overrides["architectures"] == ["VoxCPMForConditionalGeneration"] - assert args.hf_overrides["model_type"] == "voxcpm" - args._patch_empty_hf_config.assert_called_once_with("voxcpm") - - def test_strip_single_engine_args(): """_strip_single_engine_args should remove EngineArgs fields but keep omni fields.""" kwargs = { diff --git a/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py b/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py deleted file mode 100644 index f49c1d9ce..000000000 --- a/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py +++ /dev/null @@ -1,275 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""UTs for VoxCPM OpenAI speech serving behavior.""" - -import asyncio -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest -from pytest_mock import MockerFixture - -from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest -from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -@pytest.fixture -def voxcpm_server(mocker: MockerFixture): - mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set()) - mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) - - mock_engine_client = mocker.MagicMock() - mock_engine_client.errored = False - mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM1.5") - mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] - mock_engine_client.tts_batch_max_items = 32 - mock_engine_client.generate = mocker.MagicMock(return_value="generator") - mock_engine_client.stage_configs = [ - SimpleNamespace( - engine_args=SimpleNamespace( - model_stage="latent_generator", - model_arch="VoxCPMForConditionalGeneration", - ), - tts_args={}, - ), - SimpleNamespace( - engine_args=SimpleNamespace(model_stage="vae"), - tts_args={}, - ), - ] - - mock_models = mocker.MagicMock() - mock_models.is_base_model.return_value = True - - return OmniOpenAIServingSpeech( - engine_client=mock_engine_client, - models=mock_models, - request_logger=mocker.MagicMock(), - ) - - -@pytest.fixture -def voxcpm2_server(mocker: MockerFixture): - mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) - - mock_engine_client = mocker.MagicMock() - mock_engine_client.errored = False - mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM2") - mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] - mock_engine_client.tts_batch_max_items = 32 - mock_engine_client.generate = mocker.MagicMock(return_value="generator") - mock_engine_client.stage_configs = [ - SimpleNamespace( - engine_args=SimpleNamespace( - model_stage="latent_generator", - model_arch="VoxCPM2TalkerForConditionalGeneration", - ), - tts_args={}, - ), - SimpleNamespace( - engine_args=SimpleNamespace(model_stage="vae"), - tts_args={}, - ), - ] - - mock_models = mocker.MagicMock() - mock_models.is_base_model.return_value = True - - return OmniOpenAIServingSpeech( - engine_client=mock_engine_client, - models=mock_models, - request_logger=mocker.MagicMock(), - ) - - -class TestVoxCPMServing: - def test_voxcpm_model_type_detection(self, voxcpm_server): - assert voxcpm_server._tts_model_type == "voxcpm" - assert voxcpm_server._is_tts is True - assert voxcpm_server.supported_speakers == set() - - @pytest.mark.parametrize( - ("request_kwargs", "expected_substring"), - [ - ({"voice": "alice"}, "voice"), - ({"instructions": "whisper"}, "instructions"), - ({"language": "en"}, "language"), - ({"task_type": "CustomVoice"}, "plain tts"), - ({"x_vector_only_mode": True}, "x_vector_only_mode"), - ({"speaker_embedding": [0.1, 0.2]}, "speaker_embedding"), - ({"initial_codec_chunk_frames": 4}, "initial_codec_chunk_frames"), - ({"ref_text": "reference"}, "ref_audio"), - ], - ) - def test_validate_voxcpm_rejects_unsupported_fields(self, voxcpm_server, request_kwargs, expected_substring): - request = OpenAICreateSpeechRequest(input="hello voxcpm", **request_kwargs) - error = voxcpm_server._validate_voxcpm_request(request) - assert error is not None - assert expected_substring in error.lower() - - def test_validate_voxcpm_accepts_plain_tts_request(self, voxcpm_server): - request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=256) - assert voxcpm_server._validate_voxcpm_request(request) is None - - def test_validate_voxcpm_accepts_voice_clone_request(self, voxcpm_server): - request = OpenAICreateSpeechRequest( - input="clone this voice", - ref_audio="data:audio/wav;base64,QUJD", - ref_text="reference transcript", - max_new_tokens=256, - ) - assert voxcpm_server._validate_voxcpm_request(request) is None - - def test_prepare_speech_generation_voxcpm_text_only(self, voxcpm_server): - request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=321) - - request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request)) - - assert request_id.startswith("speech-") - assert generator == "generator" - assert tts_params == { - "text": ["hello voxcpm"], - "cfg_value": [2.0], - "inference_timesteps": [10], - "min_len": [2], - "max_new_tokens": [321], - } - - voxcpm_server.engine_client.generate.assert_called_once() - call = voxcpm_server.engine_client.generate.call_args - assert call.kwargs["prompt"] == { - "prompt_token_ids": [1], - "additional_information": tts_params, - "type": "token", - } - assert call.kwargs["output_modalities"] == ["audio"] - - def test_prepare_speech_generation_voxcpm_voice_clone_resolves_ref_audio(self, voxcpm_server): - voxcpm_server._resolve_ref_audio = AsyncMock(return_value=([0.1, -0.1, 0.2], 16000)) - request = OpenAICreateSpeechRequest( - input="clone this voice", - ref_audio="data:audio/wav;base64,QUJD", - ref_text="reference transcript", - max_new_tokens=512, - ) - - request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request)) - - assert request_id.startswith("speech-") - assert generator == "generator" - assert tts_params == { - "text": ["clone this voice"], - "cfg_value": [2.0], - "inference_timesteps": [10], - "min_len": [2], - "max_new_tokens": [512], - "ref_text": ["reference transcript"], - "ref_audio": [[[0.1, -0.1, 0.2], 16000]], - } - - voxcpm_server._resolve_ref_audio.assert_awaited_once_with("data:audio/wav;base64,QUJD") - call = voxcpm_server.engine_client.generate.call_args - assert call.kwargs["prompt"] == { - "prompt_token_ids": [1], - "additional_information": tts_params, - "type": "token", - } - - -class TestVoxCPM2Serving: - """Regression tests for VoxCPM2 serving behavior.""" - - def test_voxcpm2_model_type_detection(self, voxcpm2_server): - assert voxcpm2_server._tts_model_type == "voxcpm2" - assert voxcpm2_server._is_tts is True - - def test_voxcpm2_default_voice_is_supported(self, voxcpm2_server): - """VoxCPM2 should accept voice='default' (zero-shot mode, no voice cloning). - - Regression: previously returned 400 Invalid voice 'default'. - """ - assert "default" in voxcpm2_server.supported_speakers - - def test_voxcpm2_inline_ref_audio_does_not_set_voice_name(self, voxcpm2_server): - """Inline ref_audio must not populate voice_name in additional_information. - - Regression: the speaker cache keys on voice_name, so two requests with - the same voice but different inline ref_audio would collide — the second - request reused stale cached features from the first, causing a prefill - length mismatch and engine crash. - """ - mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"} - voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt) - - request = OpenAICreateSpeechRequest( - input="test", - voice="default", - ref_audio="data:audio/wav;base64,QUJD", - ) - - asyncio.run(voxcpm2_server._prepare_speech_generation(request)) - - additional = mock_prompt["additional_information"] - assert "voice_name" not in additional, ( - "voice_name must not be set for inline ref_audio — " - "it causes speaker cache collisions across different audio clips" - ) - - def test_voxcpm2_uploaded_voice_sets_voice_name(self, voxcpm2_server): - """Uploaded voice without inline ref_audio must populate voice_name. - - This ensures the speaker cache still works for the intended use case: - a voice uploaded once and referenced by name in subsequent requests. - """ - mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"} - voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt) - voxcpm2_server.uploaded_speakers["my_voice"] = { - "name": "my_voice", - "file_path": "/tmp/fake.wav", - "created_at": 12345, - } - voxcpm2_server.supported_speakers.add("my_voice") - - request = OpenAICreateSpeechRequest( - input="test", - voice="my_voice", - ) - - asyncio.run(voxcpm2_server._prepare_speech_generation(request)) - - additional = mock_prompt["additional_information"] - assert additional.get("voice_name") == "my_voice" - assert "voice_created_at" in additional - - def test_voxcpm2_uploaded_voice_with_inline_override_does_not_set_voice_name(self, voxcpm2_server): - """Uploaded voice with inline ref_audio override must not populate voice_name. - - Prevents cache poisoning: an attacker could send an uploaded voice name - with crafted ref_audio, caching malicious features under the uploaded - voice's name for subsequent legitimate requests. - """ - mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"} - voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt) - voxcpm2_server.uploaded_speakers["my_voice"] = { - "name": "my_voice", - "file_path": "/tmp/fake.wav", - "created_at": 12345, - } - voxcpm2_server.supported_speakers.add("my_voice") - - request = OpenAICreateSpeechRequest( - input="test", - voice="my_voice", - ref_audio="data:audio/wav;base64,QUJD", - ) - - asyncio.run(voxcpm2_server._prepare_speech_generation(request)) - - additional = mock_prompt["additional_information"] - assert "voice_name" not in additional, ( - "voice_name must not be set when inline ref_audio overrides an " - "uploaded voice — it would poison the cache for future requests" - ) diff --git a/tests/entrypoints/test_utils.py b/tests/entrypoints/test_utils.py index b52b49d68..479ada963 100644 --- a/tests/entrypoints/test_utils.py +++ b/tests/entrypoints/test_utils.py @@ -313,39 +313,6 @@ class TestResolveModelConfigPath: assert result is not None assert "glm_image.yaml" in result - def test_voxcpm_transformers_format_resolution(self, mocker: MockerFixture): - """Test VoxCPM transformers config resolves to the voxcpm stage config.""" - mocker.patch( - "vllm_omni.entrypoints.utils.get_config", - side_effect=ValueError("missing transformers config"), - ) - mocker.patch( - "vllm_omni.entrypoints.utils.file_or_path_exists", - side_effect=lambda _model, filename, revision=None: filename == "config.json", - ) - mocker.patch( - "vllm_omni.entrypoints.utils.get_hf_file_to_dict", - return_value={"model_type": "voxcpm"}, - ) - mocker.patch( - "vllm_omni.entrypoints.utils.current_omni_platform.get_default_stage_config_path", - return_value="vllm_omni/deploy", - ) - - original_exists = os.path.exists - - def mock_exists(path): - if "voxcpm.yaml" in str(path): - return True - return original_exists(path) - - mocker.patch("os.path.exists", side_effect=mock_exists) - - result = resolve_model_config_path("OpenBMB/VoxCPM1.5") - - assert result is not None - assert "voxcpm.yaml" in result - class TestLoadAndResolveStageConfigs: def test_load_and_resolve_with_kwargs(self): diff --git a/tests/model_executor/stage_input_processors/test_voxcpm_async_chunk.py b/tests/model_executor/stage_input_processors/test_voxcpm_async_chunk.py deleted file mode 100644 index 6738c763a..000000000 --- a/tests/model_executor/stage_input_processors/test_voxcpm_async_chunk.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""UTs for VoxCPM async-chunk stage input processing.""" - -from types import SimpleNamespace - -import pytest -import torch - -from vllm_omni.model_executor.stage_input_processors.voxcpm import ( - _VOXCPM_LATENT_MAGIC, - _coerce_finished_flag, - latent2vae_async_chunk, -) - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - - -def _request(*, finished): - return SimpleNamespace(is_finished=lambda: finished) - - -def _decode_serialized_latent(codes: list[int]) -> torch.Tensor: - assert codes[0] == _VOXCPM_LATENT_MAGIC - latent_dim = codes[1] - time_dim = codes[2] - payload = torch.tensor(codes[3:], dtype=torch.int32).to(torch.uint16) - return payload.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim) - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - (None, False), - (False, False), - (True, True), - (torch.tensor(False), False), - (torch.tensor(True), True), - ([torch.tensor(True)], True), - (([True],), True), - ([], False), - ], -) -def test_coerce_finished_flag(value, expected): - assert _coerce_finished_flag(value) is expected - - -def test_latent2vae_async_chunk_serializes_latent_payload(): - latent = torch.arange(6, dtype=torch.float32).reshape(2, 3) - - payload = latent2vae_async_chunk( - transfer_manager=None, - pooling_output={"latent_audio_feat": latent}, - request=_request(finished=False), - is_finished=torch.tensor(False), - ) - - assert payload is not None - assert torch.equal(payload.meta.finished, torch.tensor(False, dtype=torch.bool)) - recovered = _decode_serialized_latent(payload.codes.audio.tolist()) - torch.testing.assert_close(recovered, latent.to(torch.bfloat16).to(torch.float32).unsqueeze(0)) - - -def test_latent2vae_async_chunk_returns_terminal_marker_without_latent(): - payload = latent2vae_async_chunk( - transfer_manager=None, - pooling_output=None, - request=_request(finished=[torch.tensor(True)]), - is_finished=False, - ) - - assert payload.codes.audio.tolist() == [] - assert torch.equal(payload.meta.finished, torch.tensor(True, dtype=torch.bool)) - - -def test_latent2vae_async_chunk_returns_none_for_nonterminal_empty_chunk(): - payload = latent2vae_async_chunk( - transfer_manager=None, - pooling_output={"latent_audio_feat": torch.zeros((0,), dtype=torch.float32)}, - request=_request(finished=False), - is_finished=False, - ) - - assert payload is None diff --git a/vllm_omni/deploy/voxcpm.yaml b/vllm_omni/deploy/voxcpm.yaml deleted file mode 100644 index a5f324f66..000000000 --- a/vllm_omni/deploy/voxcpm.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# VoxCPM two-stage (latent → VAE) without async_chunk: one-shot latent then decode. -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: bfloat16 - model_stage: latent_generator - model_arch: VoxCPMForConditionalGeneration - # Optional persistent HF-compatible config dir for native VoxCPM models. - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: latent - gpu_memory_utilization: 0.7 - distributed_executor_backend: "mp" - max_num_batched_tokens: 4096 - max_model_len: 4096 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 4096 - stop_token_ids: [2] - seed: 42 - detokenize: false - repetition_penalty: 1.0 - final_output: false - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: float32 - model_stage: vae - model_arch: VoxCPMForConditionalGeneration - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.15 - distributed_executor_backend: "mp" - max_num_batched_tokens: 8192 - max_model_len: 4096 - engine_input_source: [0] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: true - repetition_penalty: 1.0 diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 14d0d43e8..12a5a450a 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -22,7 +22,6 @@ _ARCH_TO_MODEL_TYPE: dict[str, str] = { "CosyVoice3Model": "cosyvoice3", "OmniVoiceModel": "omnivoice", "VoxCPM2TalkerForConditionalGeneration": "voxcpm2", - "VoxCPMForConditionalGeneration": "voxcpm", } # Maps model architecture names to tokenizer subfolder paths within HF repos. diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 70a491a94..3b7a2e747 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -65,7 +65,6 @@ _FISH_TTS_MODEL_STAGES = {"fish_speech_slow_ar"} _COSYVOICE3_TTS_MODEL_STAGES = {"cosyvoice3_talker"} _OMNIVOICE_TTS_MODEL_STAGES = {"omnivoice_generator"} _COVO_AUDIO_MODEL_STAGES = {"fused_thinker_talker"} -_VOXCPM_TTS_MODEL_STAGES = {"latent_generator", "vae"} _VOXCPM2_TTS_MODEL_STAGES = {"latent_generator"} _MING_TTS_MODEL_STAGES = {"ming_tts"} _MOSS_TTS_MODEL_STAGES = {"moss_tts_nano"} @@ -76,7 +75,6 @@ _TTS_MODEL_STAGES: set[str] = ( | _COSYVOICE3_TTS_MODEL_STAGES | _OMNIVOICE_TTS_MODEL_STAGES | _COVO_AUDIO_MODEL_STAGES - | _VOXCPM_TTS_MODEL_STAGES | _VOXCPM2_TTS_MODEL_STAGES | _MING_TTS_MODEL_STAGES | _MOSS_TTS_MODEL_STAGES @@ -481,8 +479,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): model_arch = getattr(self._tts_stage.engine_args, "model_arch", None) if model_arch == "VoxCPM2TalkerForConditionalGeneration": return "voxcpm2" - if model_arch == "VoxCPMForConditionalGeneration": - return "voxcpm" if model_stage in _QWEN3_TTS_MODEL_STAGES: return "qwen3_tts" if model_stage in _VOXTRAL_TTS_MODEL_STAGES: @@ -497,12 +493,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): model_arch = getattr(self._tts_stage.engine_args, "model_arch", None) if model_arch and "CovoAudio" in model_arch: return "covo_audio" - if model_stage in (_VOXCPM_TTS_MODEL_STAGES | _VOXCPM2_TTS_MODEL_STAGES): - has_vae_stage = any( - getattr(getattr(stage, "engine_args", None), "model_stage", None) == "vae" - for stage in self.engine_client.stage_configs - ) - return "voxcpm" if has_vae_stage or model_stage == "vae" else "voxcpm2" + if model_stage in _VOXCPM2_TTS_MODEL_STAGES: + return "voxcpm2" if model_stage in _MING_TTS_MODEL_STAGES: return "ming_flash_omni_tts" if model_stage in _MOSS_TTS_MODEL_STAGES: @@ -536,8 +528,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # is no static speaker list to surface here. return set() try: - if self._tts_model_type == "voxcpm": - return set() if self._tts_model_type == "voxcpm2": return {"default"} if self._tts_model_type == "voxtral_tts": @@ -599,8 +589,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def _estimate_prompt_len(self, tts_params: dict[str, Any]) -> int: """Estimate prompt length so the placeholder matches model-side embeddings.""" try: - if self._tts_model_type == "voxcpm": - return 1 from vllm_omni.model_executor.models.qwen3_tts.qwen3_tts_talker import ( Qwen3TTSTalkerForConditionalGeneration, ) @@ -1136,8 +1124,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return self._validate_fish_tts_request(request) if self._tts_model_type == "cosyvoice3": return self._validate_cosyvoice3_request(request) - if self._tts_model_type == "voxcpm": - return self._validate_voxcpm_request(request) if self._tts_model_type == "voxcpm2": if request.max_new_tokens is not None: if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: @@ -1242,43 +1228,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return None - def _validate_voxcpm_request(self, request: OpenAICreateSpeechRequest) -> str | None: - """Validate VoxCPM request parameters. Returns error message or None.""" - if not request.input or not request.input.strip(): - return "Input text cannot be empty" - - if request.voice is not None: - return "'voice' is not supported for VoxCPM" - if request.instructions is not None: - return "'instructions' is not supported for VoxCPM" - if request.language is not None: - return "'language' is not supported for VoxCPM" - if request.task_type not in (None, "Base"): - return "VoxCPM only supports plain TTS or voice cloning with ref_audio/ref_text" - if request.x_vector_only_mode is not None: - return "'x_vector_only_mode' is not supported for VoxCPM" - if request.speaker_embedding is not None: - return "'speaker_embedding' is not supported for VoxCPM" - if request.initial_codec_chunk_frames is not None: - return "'initial_codec_chunk_frames' is not supported for VoxCPM" - - if request.ref_audio is not None: - fmt_err = self._validate_ref_audio_format(request.ref_audio) - if fmt_err: - return fmt_err - if not request.ref_text or not request.ref_text.strip(): - return "Voice cloning requires 'ref_text' (transcript of the reference audio)" - elif request.ref_text is not None: - return "'ref_text' requires 'ref_audio' for VoxCPM voice cloning" - - if request.max_new_tokens is not None: - if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: - return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" - if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: - return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" - - return None - def _validate_qwen_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: """Validate Qwen TTS request parameters. Returns error message or None.""" # Infer Base task when ref_audio or ref_text is provided without explicit task_type. @@ -1643,18 +1592,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): Processes each parameter if present, skips if not. Values are wrapped in lists as required by the model. """ - if self._tts_model_type == "voxcpm": - params: dict[str, Any] = { - "text": [request.input], - "cfg_value": [2.0], - "inference_timesteps": [10], - "min_len": [2], - "max_new_tokens": [request.max_new_tokens or 4096], - } - if request.ref_text is not None: - params["ref_text"] = [request.ref_text] - return params - params: dict[str, Any] = {} # Text content (always required) @@ -2169,8 +2106,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): model_type = "voxtral_tts" elif self._tts_model_type == "cosyvoice3": model_type = "cosyvoice3" - elif self._tts_model_type == "voxcpm": - model_type = "voxcpm" elif self._tts_model_type == "voxcpm2": model_type = "voxcpm2" elif self._tts_model_type == "ming_flash_omni_tts": diff --git a/vllm_omni/model_executor/models/registry.py b/vllm_omni/model_executor/models/registry.py index cc900d471..4f29c0ae3 100644 --- a/vllm_omni/model_executor/models/registry.py +++ b/vllm_omni/model_executor/models/registry.py @@ -150,12 +150,6 @@ _OMNI_MODELS = { "fish_speech_dac_decoder", "FishSpeechDACDecoder", ), - ## VoxCPM - "VoxCPMForConditionalGeneration": ( - "voxcpm", - "voxcpm", - "VoxCPMForConditionalGeneration", - ), ## VoxCPM2 "VoxCPM2TalkerForConditionalGeneration": ( "voxcpm2", diff --git a/vllm_omni/model_executor/models/voxcpm/__init__.py b/vllm_omni/model_executor/models/voxcpm/__init__.py deleted file mode 100644 index ffde6dee6..000000000 --- a/vllm_omni/model_executor/models/voxcpm/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .voxcpm import VoxCPMForConditionalGeneration - -__all__ = [ - "VoxCPMForConditionalGeneration", -] diff --git a/vllm_omni/model_executor/models/voxcpm/voxcpm.py b/vllm_omni/model_executor/models/voxcpm/voxcpm.py deleted file mode 100644 index 6fa36fc42..000000000 --- a/vllm_omni/model_executor/models/voxcpm/voxcpm.py +++ /dev/null @@ -1,886 +0,0 @@ -from __future__ import annotations - -import json -import os -import sys -import tempfile -import warnings -import wave -from collections.abc import Callable, Generator, Iterable -from pathlib import Path -from typing import Any - -import numpy as np -import torch -import torch.nn as nn -from einops import rearrange -from tqdm import tqdm -from vllm.config import VllmConfig -from vllm.logger import init_logger -from vllm.sequence import IntermediateTensors - -from vllm_omni.model_executor.models.output_templates import OmniOutput - -from .voxcpm_loader import ( - _build_prompt_cache_with_soundfile, - _device_to_string, - _force_cuda_available_for_npu, - _import_voxcpm_audio_vae_classes, - _import_voxcpm_base_model_class, - _is_torchcodec_load_error, - _normalize_dtype_name, - _prepare_runtime_model_dir, - _resolve_runtime_device, -) -from .voxcpm_runtime_utils import resolve_voxcpm_model_dir -from .voxcpm_stage_wrappers import _DirectVoxCPMAudioVAE, _DirectVoxCPMLatentGenerator - -logger = init_logger(__name__) -_VOXCPM_LATENT_MAGIC = 131071 - - -def _make_voxcpm_model_for_omni(base: type[Any]) -> type[Any]: - """Subclass upstream VoxCPMModel: local ``_inference`` + ``latents_only`` prompt-cache generation.""" - - from voxcpm.model.utils import get_dtype - - class VoxCPMModelForOmni(base): - @torch.inference_mode() - def build_prompt_cache(self, *args: Any, **kwargs: Any): - try: - return super().build_prompt_cache(*args, **kwargs) - except (ImportError, ModuleNotFoundError, RuntimeError) as exc: - if not _is_torchcodec_load_error(exc): - raise - return _build_prompt_cache_with_soundfile(self, *args, **kwargs) - - @torch.inference_mode() - def _inference( - self, - text: torch.Tensor, - text_mask: torch.Tensor, - feat: torch.Tensor, - feat_mask: torch.Tensor, - min_len: int = 2, - max_len: int = 2000, - inference_timesteps: int = 10, - cfg_value: float = 2.0, - streaming: bool = False, - streaming_prefix_len: int = 3, - ) -> Generator[tuple[torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]: - B, _, _, _ = feat.shape - - feat_embed = self.feat_encoder(feat) - feat_embed = self.enc_to_lm_proj(feat_embed) - - scale_emb = self.config.lm_config.scale_emb if self.config.lm_config.use_mup else 1.0 - text_embed = self.base_lm.embed_tokens(text) * scale_emb - combined_embed = text_mask.unsqueeze(-1) * text_embed + feat_mask.unsqueeze(-1) * feat_embed - - prefix_feat_cond = feat[:, -1, ...] - pred_feat_seq: list[torch.Tensor] = [] - - audio_patch_count = int(feat_mask.sum().item()) - if audio_patch_count > 0: - context_len = min(streaming_prefix_len - 1, audio_patch_count) - prompt_context_patches = list(feat[:, -context_len:, :, :].split(1, dim=1)) - pred_feat_seq = prompt_context_patches + pred_feat_seq - - enc_outputs, kv_cache_tuple = self.base_lm( - inputs_embeds=combined_embed, - is_causal=True, - ) - self.base_lm.kv_cache.fill_caches(kv_cache_tuple) - - enc_outputs = self.fsq_layer(enc_outputs) * feat_mask.unsqueeze(-1) + enc_outputs * text_mask.unsqueeze(-1) - lm_hidden = enc_outputs[:, -1, :] - - residual_enc_outputs, residual_kv_cache_tuple = self.residual_lm( - inputs_embeds=enc_outputs + feat_mask.unsqueeze(-1) * feat_embed, - is_causal=True, - ) - self.residual_lm.kv_cache.fill_caches(residual_kv_cache_tuple) - residual_hidden = residual_enc_outputs[:, -1, :] - - for step_idx in tqdm(range(max_len)): - dit_hidden = self.lm_to_dit_proj(lm_hidden) + self.res_to_dit_proj(residual_hidden) - pred_feat = self.feat_decoder( - mu=dit_hidden, - patch_size=self.patch_size, - cond=prefix_feat_cond.transpose(1, 2).contiguous(), - n_timesteps=inference_timesteps, - cfg_value=cfg_value, - ).transpose(1, 2) - - curr_embed = self.enc_to_lm_proj(self.feat_encoder(pred_feat.unsqueeze(1))) - pred_feat_seq.append(pred_feat.unsqueeze(1)) - prefix_feat_cond = pred_feat - - if streaming: - pred_feat_chunk = torch.cat(pred_feat_seq[-streaming_prefix_len:], dim=1) - feat_pred = rearrange(pred_feat_chunk, "b t p d -> b d (t p)", b=B, p=self.patch_size) - yield feat_pred, pred_feat_seq - - stop_flag = self.stop_head(self.stop_actn(self.stop_proj(lm_hidden))).argmax(dim=-1)[0].cpu().item() - if step_idx > min_len and stop_flag == 1: - break - - lm_hidden = self.base_lm.forward_step( - curr_embed[:, 0, :], - torch.tensor([self.base_lm.kv_cache.step()], device=curr_embed.device), - ).clone() - lm_hidden = self.fsq_layer(lm_hidden) - residual_hidden = self.residual_lm.forward_step( - lm_hidden + curr_embed[:, 0, :], - torch.tensor([self.residual_lm.kv_cache.step()], device=curr_embed.device), - ).clone() - - if not streaming: - pred_feat_seq_cat = torch.cat(pred_feat_seq, dim=1) - feat_pred = rearrange(pred_feat_seq_cat, "b t p d -> b d (t p)", b=B, p=self.patch_size) - yield feat_pred, pred_feat_seq_cat.squeeze(0).cpu() - - @torch.inference_mode() - def generate_latents_with_prompt_cache( - self, - target_text: str, - prompt_cache: dict, - min_len: int = 2, - max_len: int = 2000, - inference_timesteps: int = 10, - cfg_value: float = 2.0, - retry_badcase: bool = False, - retry_badcase_max_times: int = 3, - retry_badcase_ratio_threshold: float = 6.0, - streaming_prefix_len: int = 3, - ) -> tuple[None, torch.Tensor, torch.Tensor]: - return next( - self._generate_with_prompt_cache( - target_text=target_text, - prompt_cache=prompt_cache, - min_len=min_len, - max_len=max_len, - inference_timesteps=inference_timesteps, - cfg_value=cfg_value, - retry_badcase=retry_badcase, - retry_badcase_max_times=retry_badcase_max_times, - retry_badcase_ratio_threshold=retry_badcase_ratio_threshold, - streaming=False, - streaming_prefix_len=streaming_prefix_len, - latents_only=True, - ) - ) - - @torch.inference_mode() - def generate_latents_with_prompt_cache_streaming( - self, - target_text: str, - prompt_cache: dict, - min_len: int = 2, - max_len: int = 2000, - inference_timesteps: int = 10, - cfg_value: float = 2.0, - retry_badcase: bool = False, - retry_badcase_max_times: int = 3, - retry_badcase_ratio_threshold: float = 6.0, - streaming_prefix_len: int = 3, - ) -> Generator[tuple[None, torch.Tensor, torch.Tensor], None, None]: - return self._generate_with_prompt_cache( - target_text=target_text, - prompt_cache=prompt_cache, - min_len=min_len, - max_len=max_len, - inference_timesteps=inference_timesteps, - cfg_value=cfg_value, - retry_badcase=retry_badcase, - retry_badcase_max_times=retry_badcase_max_times, - retry_badcase_ratio_threshold=retry_badcase_ratio_threshold, - streaming=True, - streaming_prefix_len=streaming_prefix_len, - latents_only=True, - ) - - @torch.inference_mode() - def _generate_with_prompt_cache( - self, - target_text: str, - prompt_cache: dict, - min_len: int = 2, - max_len: int = 2000, - inference_timesteps: int = 10, - cfg_value: float = 2.0, - retry_badcase: bool = False, - retry_badcase_max_times: int = 3, - retry_badcase_ratio_threshold: float = 6.0, - streaming: bool = False, - streaming_prefix_len: int = 3, - latents_only: bool = False, - ) -> Generator[tuple[torch.Tensor | None, torch.Tensor, torch.Tensor | list[torch.Tensor]], None, None]: - if retry_badcase and streaming: - warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.") - retry_badcase = False - if prompt_cache is None: - prompt_audio_feat = torch.empty((0, self.patch_size, self.audio_vae.latent_dim), dtype=torch.float32) - text = target_text - else: - prompt_audio_feat = prompt_cache["audio_feat"] - prompt_text = prompt_cache["prompt_text"] - text = prompt_text + target_text - - text_token = torch.LongTensor(self.text_tokenizer(text)) - text_token = torch.cat( - [ - text_token, - torch.tensor([self.audio_start_token], dtype=torch.int32, device=text_token.device), - ], - dim=-1, - ) - target_text_token = torch.LongTensor(self.text_tokenizer(target_text)) - - audio_length = prompt_audio_feat.size(0) - text_length = text_token.shape[0] - text_pad_token = torch.zeros(audio_length, dtype=torch.int32, device=text_token.device) - audio_pad_feat = torch.zeros( - (text_token.shape[0], self.patch_size, self.audio_vae.latent_dim), - dtype=torch.float32, - device=text_token.device, - ) - text_token = torch.cat([text_token, text_pad_token]) - audio_feat = torch.cat([audio_pad_feat, prompt_audio_feat], dim=0) - text_mask = ( - torch.cat([torch.ones(text_length), torch.zeros(audio_length)]).type(torch.int32).to(text_token.device) - ) - audio_mask = ( - torch.cat([torch.zeros(text_length), torch.ones(audio_length)]).type(torch.int32).to(text_token.device) - ) - - text_token = text_token.unsqueeze(0).to(self.device) - text_mask = text_mask.unsqueeze(0).to(self.device) - audio_feat = audio_feat.unsqueeze(0).to(self.device).to(get_dtype(self.config.dtype)) - audio_mask = audio_mask.unsqueeze(0).to(self.device) - - target_text_length = len(self.text_tokenizer(target_text)) - retry_badcase_times = 0 - while retry_badcase_times < retry_badcase_max_times: - inference_result = self._inference( - text_token, - text_mask, - audio_feat, - audio_mask, - min_len=min_len, - max_len=min(int(target_text_length * retry_badcase_ratio_threshold + 10), max_len), - inference_timesteps=inference_timesteps, - cfg_value=cfg_value, - streaming=streaming, - streaming_prefix_len=streaming_prefix_len, - ) - if streaming: - patch_len = self.patch_size * self.chunk_size - for latent_pred, pred_audio_feat in inference_result: - if latents_only: - decode_audio = None - yield (decode_audio, target_text_token, latent_pred) - else: - decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)) - decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu() - yield (decode_audio, target_text_token, pred_audio_feat) - break - - latent_pred, pred_audio_feat = next(inference_result) - if retry_badcase and pred_audio_feat.shape[0] >= target_text_length * retry_badcase_ratio_threshold: - ratio = pred_audio_feat.shape[0] / target_text_length - print(f" Badcase detected, audio_text_ratio={ratio}, retrying...", file=sys.stderr) - retry_badcase_times += 1 - continue - break - - if not streaming: - if latents_only: - decode_audio = None - else: - decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)) - patch_len = self.patch_size * self.chunk_size - if audio_mask.sum().item() > 0: - decode_audio = decode_audio[..., patch_len * (streaming_prefix_len - 1) :].squeeze(1).cpu() - else: - decode_audio = decode_audio[..., :].squeeze(1).cpu() - yield (decode_audio, target_text_token, pred_audio_feat) - - VoxCPMModelForOmni.__name__ = "VoxCPMModelForOmni" - VoxCPMModelForOmni.__qualname__ = "VoxCPMModelForOmni" - return VoxCPMModelForOmni - - -def _import_voxcpm_model_class() -> type[Any]: - base = _import_voxcpm_base_model_class() - return _make_voxcpm_model_for_omni(base) - - -def _load_native_voxcpm_model( - model_path: str, - *, - device: torch.device, - dtype: str | None, -): - VoxCPMModel = _import_voxcpm_model_class() - model_dir = resolve_voxcpm_model_dir(model_path) - runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype=dtype) - - if device.type == "npu" and hasattr(torch, "npu"): - torch.npu.set_device(device) - - with _force_cuda_available_for_npu(device): - return VoxCPMModel.from_local( - runtime_model_path, - optimize=device.type == "cuda", - ) - - -def _load_native_voxcpm_latent_generator( - model_path: str, - *, - device: torch.device, - dtype: str | None, -) -> _DirectVoxCPMLatentGenerator: - return _DirectVoxCPMLatentGenerator(_load_native_voxcpm_model(model_path, device=device, dtype=dtype)) - - -def _load_native_voxcpm_audio_vae( - model_path: str, - *, - device: torch.device, -) -> _DirectVoxCPMAudioVAE: - AudioVAE, AudioVAEConfig = _import_voxcpm_audio_vae_classes() - model_dir = resolve_voxcpm_model_dir(model_path) - runtime_model_path = _prepare_runtime_model_dir(model_dir, target_device=device, target_dtype="float32") - config_dict = json.loads((Path(runtime_model_path) / "config.json").read_text()) - audio_vae_config = config_dict.get("audio_vae_config") - audio_vae = AudioVAE(config=AudioVAEConfig(**audio_vae_config)) if audio_vae_config is not None else AudioVAE() - - state_dict = torch.load( - Path(runtime_model_path) / "audiovae.pth", - map_location="cpu", - weights_only=True, - )["state_dict"] - audio_vae.load_state_dict(state_dict, strict=True) - audio_vae = audio_vae.to(device=device, dtype=torch.float32).eval() - if device.type == "npu" and hasattr(torch, "npu"): - torch.npu.set_device(device) - patch_size = int(config_dict.get("patch_size", 2)) - return _DirectVoxCPMAudioVAE(audio_vae, patch_size=patch_size) - - -class VoxCPMForConditionalGeneration(nn.Module): - input_modalities = "audio" - _LATENT_STAGES = {"latent_generator", "latent", "ar_dit"} - _VAE_STAGES = {"vae", "audio_vae"} - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - del prefix - self.vllm_config = vllm_config - self.model_path = vllm_config.model_config.model - self.model_stage = getattr(vllm_config.model_config, "model_stage", "latent_generator") - self.have_multimodal_outputs = True - self.has_preprocess = False - self.has_postprocess = False - self.enable_update_additional_information = True - self.requires_raw_input_tokens = True - self.inject_omni_request_id_into_runtime_info = True - self._pipeline = None - self._latent_stream_gens: dict[str, Any] = {} - self._latent_stream_terminal_pending: dict[str, int] = {} - self._latent_stream_completed: set[str] = set() - self._next_local_stream_key = 0 - self._ar_emit_stop_token = True - - def _runner_hidden_device_dtype(self) -> tuple[torch.device, torch.dtype]: - device = _resolve_runtime_device(self.vllm_config) - model_config = getattr(self.vllm_config, "model_config", None) - dtype = getattr(model_config, "dtype", torch.float32) if model_config is not None else torch.float32 - return device, dtype - - def _ensure_model_loaded(self): - if self._pipeline is not None: - return - - target_device = _resolve_runtime_device(self.vllm_config) - model_dtype = getattr(self.vllm_config.model_config, "dtype", None) - normalized_dtype = _normalize_dtype_name(model_dtype) - if self.model_stage in self._LATENT_STAGES: - self._pipeline = _load_native_voxcpm_latent_generator( - self.model_path, - device=target_device, - dtype=normalized_dtype, - ) - elif self.model_stage in self._VAE_STAGES: - self._pipeline = _load_native_voxcpm_audio_vae( - self.model_path, - device=target_device, - ) - else: - raise ValueError( - f"Unsupported VoxCPM model_stage: {self.model_stage}. " - "pure_voxcpm only supports split-stage latent_generator/vae inference." - ) - - logger.info("Loaded VoxCPM stage '%s' on %s", self.model_stage, _device_to_string(target_device)) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - del weights - self._ensure_model_loaded() - return set() - - @staticmethod - def _extract_val(info: dict[str, Any], key: str, default: Any) -> Any: - value = info.get(key, default) - if isinstance(value, list): - return value[0] if value else default - return value - - def _resolve_stream_request_key(self, info: dict[str, Any]) -> str: - request_key = info.get("__voxcpm_stream_key") - if request_key is not None: - return str(request_key) - - request_key = info.get("_omni_req_id") - if request_key is not None: - request_key = str(request_key) - info["__voxcpm_stream_key"] = request_key - return request_key - - request_key = f"voxcpm-local-{self._next_local_stream_key}" - self._next_local_stream_key += 1 - info["__voxcpm_stream_key"] = request_key - return str(request_key) - - def _recover_latent_from_input_ids(self, input_ids: torch.Tensor | None) -> torch.Tensor | None: - if input_ids is None or input_ids.numel() == 0: - return None - flat_ids = input_ids.detach().reshape(-1).to("cpu") - if flat_ids.numel() < 4 or int(flat_ids[0].item()) != _VOXCPM_LATENT_MAGIC: - return None - latent_dim = int(flat_ids[1].item()) - time_dim = int(flat_ids[2].item()) - payload = flat_ids[3:] - expected = latent_dim * time_dim - if latent_dim <= 0 or time_dim <= 0: - raise ValueError(f"Invalid VoxCPM latent header: latent_dim={latent_dim}, time_dim={time_dim}") - if int(payload.numel()) != expected: - raise ValueError( - "Invalid VoxCPM latent payload size: " - f"expected={expected}, actual={int(payload.numel())}, " - f"latent_dim={latent_dim}, time_dim={time_dim}" - ) - packed = payload.to(dtype=torch.int32).to(torch.uint16) - return packed.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim) - - def _maybe_recover_vae_infos( - self, - infos: list[dict[str, Any]], - input_ids: torch.Tensor | None, - *, - async_chunk: bool, - ) -> list[dict[str, Any]]: - if not async_chunk: - return infos - if any(self._extract_val(info, "latent_audio_feat", None) is not None for info in infos): - return infos - recovered = self._recover_latent_from_input_ids(input_ids) - if recovered is None: - return infos - return [{"latent_audio_feat": recovered}] - - @staticmethod - def _normalize_audio_samples(samples: Any) -> np.ndarray: - if isinstance(samples, torch.Tensor): - return samples.detach().cpu().float().reshape(-1).numpy() - return np.asarray(samples, dtype=np.float32).reshape(-1) - - @classmethod - def _normalize_ref_audio(cls, ref_audio: Any) -> tuple[np.ndarray, int]: - if isinstance(ref_audio, str): - raise TypeError("String ref_audio should be handled as a path before waveform normalization.") - - if isinstance(ref_audio, dict): - sample_rate = ref_audio.get("sample_rate") or ref_audio.get("sampling_rate") or ref_audio.get("sr") - samples = None - for key in ("audio", "wav", "samples", "array", "waveform"): - if key in ref_audio and ref_audio[key] is not None: - samples = ref_audio[key] - break - if sample_rate is None or samples is None: - raise ValueError("ref_audio dict must contain waveform data and sample rate.") - return cls._normalize_audio_samples(samples), int(sample_rate) - - if isinstance(ref_audio, (list, tuple)): - if len(ref_audio) == 1: - return cls._normalize_ref_audio(ref_audio[0]) - if len(ref_audio) == 2 and np.isscalar(ref_audio[1]): - return cls._normalize_audio_samples(ref_audio[0]), int(ref_audio[1]) - - raise TypeError(f"Unsupported ref_audio format: {type(ref_audio)!r}") - - @staticmethod - def _write_temp_prompt_wav(waveform: np.ndarray, sample_rate: int) -> str: - prompt_file = tempfile.NamedTemporaryFile(delete=False, suffix=".wav") - prompt_file.close() - - wav = np.asarray(waveform, dtype=np.float32).reshape(-1) - wav = np.clip(wav, -1.0, 1.0) - pcm16 = (wav * 32767.0).astype(np.int16) - with wave.open(prompt_file.name, "wb") as wav_file: - wav_file.setnchannels(1) - wav_file.setsampwidth(2) - wav_file.setframerate(int(sample_rate)) - wav_file.writeframes(pcm16.tobytes()) - - return prompt_file.name - - @classmethod - def _resolve_prompt_inputs(cls, info: dict[str, Any]) -> tuple[str | None, str | None, str | None]: - prompt_text = cls._extract_val(info, "prompt_text", None) - prompt_wav_path = cls._extract_val(info, "prompt_wav_path", None) - if prompt_wav_path: - if prompt_text is None: - prompt_text = cls._extract_val(info, "ref_text", None) - return prompt_wav_path, prompt_text, None - - ref_audio = cls._extract_val(info, "ref_audio", None) - ref_text = cls._extract_val(info, "ref_text", None) - if ref_audio is None or ref_text is None: - return None, None, None - if isinstance(ref_audio, str): - return ref_audio, ref_text, None - - waveform, sample_rate = cls._normalize_ref_audio(ref_audio) - temp_prompt_wav = cls._write_temp_prompt_wav(waveform, sample_rate) - return temp_prompt_wav, ref_text, temp_prompt_wav - - def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: - if input_ids.numel() == 0: - return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32) - return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32) - - def _get_vocab_size(self) -> int: - model_config = getattr(self.vllm_config, "model_config", None) - if model_config is not None: - getter = getattr(model_config, "get_vocab_size", None) - if callable(getter): - try: - return int(getter()) - except Exception: - pass - hf_config = getattr(model_config, "hf_text_config", None) - if hf_config is not None and hasattr(hf_config, "vocab_size"): - return int(hf_config.vocab_size) - return 32000 - - def _make_empty_output( - self, - *, - output_key: str, - payload_factory: Callable[[], torch.Tensor], - infos: list[dict[str, Any]], - sample_rate: int, - out_device: torch.device, - out_dtype: torch.dtype, - hidden_rows: int | None = None, - ) -> OmniOutput: - if hidden_rows is None: - hidden_rows = len(infos) - return OmniOutput( - text_hidden_states=torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype), - multimodal_outputs={ - output_key: [payload_factory() for _ in infos], - "sr": [torch.tensor(sample_rate, dtype=torch.int32) for _ in infos], - }, - ) - - def _finalize_stage_output( - self, - *, - output_key: str, - outputs: list[torch.Tensor], - sample_rates: list[torch.Tensor], - out_device: torch.device, - out_dtype: torch.dtype, - hidden_rows: int | None = None, - ) -> OmniOutput: - multimodal_outputs: dict[str, Any] = {output_key: outputs, "sr": sample_rates} - if hidden_rows is not None: - text_hidden_states = torch.zeros((hidden_rows, 1), device=out_device, dtype=out_dtype) - elif outputs: - outputs_tensor = torch.stack(outputs) - text_hidden_states = ( - outputs_tensor.unsqueeze(-1) - if outputs_tensor.ndim == 1 - else outputs_tensor.reshape(-1, outputs_tensor.shape[-1]) - ) - else: - text_hidden_states = torch.zeros((0, 1), device=out_device, dtype=out_dtype) - text_hidden_states = text_hidden_states.to(device=out_device, dtype=out_dtype) - return OmniOutput( - text_hidden_states=text_hidden_states, - multimodal_outputs=multimodal_outputs, - ) - - def _forward_vae_stage( - self, - infos: list[dict[str, Any]], - *, - sample_rate: int, - async_chunk: bool, - out_device: torch.device, - out_dtype: torch.dtype, - ) -> OmniOutput: - if all(self._extract_val(info, "latent_audio_feat", None) is None for info in infos): - self._ar_emit_stop_token = True - return self._make_empty_output( - output_key="model_outputs", - payload_factory=lambda: torch.zeros((0,), dtype=torch.float32), - infos=infos, - sample_rate=sample_rate, - out_device=out_device, - out_dtype=out_dtype, - ) - - outputs: list[torch.Tensor] = [] - sample_rates: list[torch.Tensor] = [] - for info in infos: - latent_audio_feat = self._extract_val(info, "latent_audio_feat", None) - audio_tensor = self._pipeline.decode(latent_audio_feat, trim_streaming_patch=async_chunk) - outputs.append(audio_tensor.float().cpu()) - sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32)) - - self._ar_emit_stop_token = True - return self._finalize_stage_output( - output_key="model_outputs", - outputs=outputs, - sample_rates=sample_rates, - out_device=out_device, - out_dtype=out_dtype, - ) - - def _forward_latent_stage( - self, - infos: list[dict[str, Any]], - *, - sample_rate: int, - async_chunk: bool, - out_device: torch.device, - out_dtype: torch.dtype, - hidden_rows: int, - ) -> OmniOutput: - texts = [self._extract_val(info, "text", "") for info in infos] - if all(not text for text in texts): - self._ar_emit_stop_token = True - return self._make_empty_output( - output_key="latent_audio_feat", - payload_factory=lambda: torch.zeros((0,), dtype=torch.float32), - infos=infos, - sample_rate=sample_rate, - out_device=out_device, - out_dtype=out_dtype, - hidden_rows=hidden_rows, - ) - - outputs: list[torch.Tensor] = [] - sample_rates: list[torch.Tensor] = [] - last_chunk_flags: list[bool] | None = [] if async_chunk else None - payload_finished_flags: list[bool] | None = [] if async_chunk else None - for info in infos: - text = self._extract_val(info, "text", "") - cfg_value = float(self._extract_val(info, "cfg_value", 2.0)) - inference_timesteps = int(self._extract_val(info, "inference_timesteps", 10)) - min_len = int(self._extract_val(info, "min_len", 2)) - max_len = int(self._extract_val(info, "max_len", self._extract_val(info, "max_new_tokens", 4096))) - retry_badcase = bool(self._extract_val(info, "retry_badcase", True)) - retry_badcase_max_times = int(self._extract_val(info, "retry_badcase_max_times", 3)) - retry_badcase_ratio_threshold = float(self._extract_val(info, "retry_badcase_ratio_threshold", 6.0)) - streaming_prefix_len = int(self._extract_val(info, "streaming_prefix_len", 3)) - - request_key = self._resolve_stream_request_key(info) - created_temp: str | None = None - - if async_chunk: - terminal_pending = self._latent_stream_terminal_pending.get(request_key, 0) - if terminal_pending > 0: - outputs.append(torch.zeros((0,), dtype=torch.float32)) - assert last_chunk_flags is not None - last_chunk_flags.append(True) - assert payload_finished_flags is not None - payload_finished_flags.append(terminal_pending == 1) - if terminal_pending == 1: - self._latent_stream_terminal_pending.pop(request_key, None) - else: - self._latent_stream_terminal_pending[request_key] = terminal_pending - 1 - sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32)) - continue - - if request_key in self._latent_stream_completed: - outputs.append(torch.zeros((0,), dtype=torch.float32)) - assert last_chunk_flags is not None - last_chunk_flags.append(True) - assert payload_finished_flags is not None - payload_finished_flags.append(False) - sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32)) - continue - - if request_key not in self._latent_stream_gens: - prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info) - created_temp = temp_prompt_wav - self._latent_stream_gens[request_key] = self._pipeline.iter_latent_chunks_streaming( - text=text, - prompt_wav_path=prompt_wav_path, - prompt_text=prompt_text, - cfg_value=cfg_value, - inference_timesteps=inference_timesteps, - min_len=min_len, - max_len=max_len, - streaming_prefix_len=streaming_prefix_len, - retry_badcase=False, - retry_badcase_max_times=retry_badcase_max_times, - retry_badcase_ratio_threshold=retry_badcase_ratio_threshold, - ) - generator = self._latent_stream_gens[request_key] - try: - chunk_latent, is_last = next(generator) - except StopIteration: - self._latent_stream_gens.pop(request_key, None) - self._latent_stream_terminal_pending[request_key] = 1 - self._latent_stream_completed.add(request_key) - outputs.append(torch.zeros((0,), dtype=torch.float32)) - assert last_chunk_flags is not None - last_chunk_flags.append(True) - assert payload_finished_flags is not None - payload_finished_flags.append(True) - else: - if is_last: - self._latent_stream_gens.pop(request_key, None) - self._latent_stream_terminal_pending[request_key] = 1 - self._latent_stream_completed.add(request_key) - outputs.append(chunk_latent.detach().float().cpu()) - assert last_chunk_flags is not None - last_chunk_flags.append(bool(is_last)) - assert payload_finished_flags is not None - payload_finished_flags.append(False) - finally: - if created_temp is not None and os.path.exists(created_temp): - os.unlink(created_temp) - sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32)) - continue - - prompt_wav_path, prompt_text, temp_prompt_wav = self._resolve_prompt_inputs(info) - try: - latent_audio_feat = self._pipeline.generate_latents( - text=text, - prompt_wav_path=prompt_wav_path, - prompt_text=prompt_text, - cfg_value=cfg_value, - inference_timesteps=inference_timesteps, - min_len=min_len, - max_len=max_len, - retry_badcase=retry_badcase, - retry_badcase_max_times=retry_badcase_max_times, - retry_badcase_ratio_threshold=retry_badcase_ratio_threshold, - ) - outputs.append(latent_audio_feat.float().cpu()) - finally: - if temp_prompt_wav is not None and os.path.exists(temp_prompt_wav): - os.unlink(temp_prompt_wav) - - sample_rates.append(torch.tensor(sample_rate, dtype=torch.int32)) - - self._ar_emit_stop_token = all(last_chunk_flags) if async_chunk and last_chunk_flags else True - output = self._finalize_stage_output( - output_key="latent_audio_feat", - outputs=outputs, - sample_rates=sample_rates, - out_device=out_device, - out_dtype=out_dtype, - hidden_rows=hidden_rows, - ) - if async_chunk and payload_finished_flags is not None: - output.multimodal_outputs["finished"] = [ - torch.tensor(flag, dtype=torch.bool) for flag in payload_finished_flags - ] - return output - - def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> torch.Tensor: - del sampling_metadata - if isinstance(hidden_states, OmniOutput): - hidden_states = hidden_states.text_hidden_states - if hidden_states is None: - device, dtype = self._runner_hidden_device_dtype() - hidden_states = torch.zeros((0, 1), device=device, dtype=dtype) - if hidden_states.ndim == 1: - hidden_states = hidden_states.unsqueeze(-1) - elif hidden_states.ndim > 2: - hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1]) - - vocab_size = self._get_vocab_size() - num_rows = int(hidden_states.shape[0]) - logits = torch.zeros((num_rows, vocab_size), dtype=torch.float32, device=hidden_states.device) - eos_id = 2 if vocab_size > 2 else 0 - safe_id = 1 if vocab_size > 1 and 1 != eos_id else 0 - emit_stop = getattr(self, "_ar_emit_stop_token", True) - if num_rows > 0: - if emit_stop: - logits[:, eos_id] = 1.0e6 - else: - logits[:, eos_id] = -1.0e9 - logits[:, safe_id] = 1.0e6 - return logits - - @torch.no_grad() - def forward( - self, - input_ids: torch.Tensor | None = None, - positions: torch.Tensor | None = None, - intermediate_tensors: Any = None, - inputs_embeds: torch.Tensor | None = None, - runtime_additional_information: list[dict[str, Any]] | None = None, - model_intermediate_buffer: list[dict[str, Any]] | None = None, - **kwargs: Any, - ) -> OmniOutput: - del positions, intermediate_tensors, inputs_embeds, kwargs - self._ensure_model_loaded() - out_device, out_dtype = self._runner_hidden_device_dtype() - if input_ids is not None and input_ids.device.type == out_device.type: - out_device = input_ids.device - - infos = model_intermediate_buffer or runtime_additional_information or [{}] - hidden_rows = len(infos) - if input_ids is not None and len(input_ids.shape) > 0: - hidden_rows = max(hidden_rows, int(input_ids.shape[0])) - sample_rate = int(getattr(self._pipeline, "sample_rate", 24000)) - async_chunk = bool(getattr(self.vllm_config.model_config, "async_chunk", False)) - if self.model_stage in self._VAE_STAGES: - infos = self._maybe_recover_vae_infos(infos, input_ids, async_chunk=async_chunk) - return self._forward_vae_stage( - infos, - sample_rate=sample_rate, - async_chunk=async_chunk, - out_device=out_device, - out_dtype=out_dtype, - ) - if self.model_stage in self._LATENT_STAGES: - return self._forward_latent_stage( - infos, - sample_rate=sample_rate, - async_chunk=async_chunk, - out_device=out_device, - out_dtype=out_dtype, - hidden_rows=hidden_rows, - ) - raise ValueError(f"Unsupported VoxCPM model_stage at runtime: {self.model_stage}") - - def make_empty_intermediate_tensors( - self, batch_size: int, dtype: torch.dtype, device: torch.device - ) -> IntermediateTensors: - del batch_size, dtype, device - return {} - - -__all__ = ["VoxCPMForConditionalGeneration"] diff --git a/vllm_omni/model_executor/models/voxcpm/voxcpm_loader.py b/vllm_omni/model_executor/models/voxcpm/voxcpm_loader.py deleted file mode 100644 index dac7117ca..000000000 --- a/vllm_omni/model_executor/models/voxcpm/voxcpm_loader.py +++ /dev/null @@ -1,247 +0,0 @@ -from __future__ import annotations - -import importlib -import json -import os -import shutil -import sys -import tempfile -from contextlib import contextmanager -from hashlib import sha256 -from pathlib import Path -from typing import Any -from unittest.mock import patch - -import numpy as np -import torch -from vllm.config import VllmConfig -from vllm.logger import init_logger - -logger = init_logger(__name__) - - -def _iter_voxcpm_src_candidates() -> list[Path]: - candidates: list[Path] = [] - env_path = os.environ.get("VLLM_OMNI_VOXCPM_CODE_PATH") - if env_path: - candidates.append(Path(env_path).expanduser()) - - repo_root = Path(__file__).resolve().parents[4] - candidates.append(repo_root.parent / "VoxCPM" / "src") - - unique_candidates: list[Path] = [] - seen: set[str] = set() - for candidate in candidates: - candidate_key = str(candidate) - if candidate_key in seen: - continue - seen.add(candidate_key) - unique_candidates.append(candidate) - return unique_candidates - - -def _prepend_voxcpm_src(candidate: Path) -> None: - candidate_str = str(candidate) - if candidate_str not in sys.path: - sys.path.insert(0, candidate_str) - - -def _import_voxcpm_attrs(module_name: str, *attr_names: str) -> tuple[Any, ...]: - last_exc: ImportError | None = None - for candidate in _iter_voxcpm_src_candidates(): - if not candidate.exists(): - continue - _prepend_voxcpm_src(candidate) - try: - module = importlib.import_module(module_name) - return tuple(getattr(module, attr_name) for attr_name in attr_names) - except ImportError as exc: - last_exc = exc - - try: - module = importlib.import_module(module_name) - return tuple(getattr(module, attr_name) for attr_name in attr_names) - except ImportError as exc: - last_exc = exc - - raise ImportError(f"Failed to import {module_name}.") from last_exc - - -def _import_voxcpm_base_model_class(): - """Import upstream ``VoxCPMModel`` from ``VoxCPM/src/voxcpm`` (env, sibling tree, or pip).""" - try: - (VoxCPMModel,) = _import_voxcpm_attrs("voxcpm.model.voxcpm", "VoxCPMModel") - return VoxCPMModel - except ImportError as exc: - raise ImportError( - "Failed to import VoxCPMModel. Install the `voxcpm` package or set " - "`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory " - "(the parent of the `voxcpm` package that contains `model/` and `modules/`)." - ) from exc - - -def _import_voxcpm_audio_vae_classes(): - try: - return _import_voxcpm_attrs("voxcpm.modules.audiovae", "AudioVAE", "AudioVAEConfig") - except ImportError as exc: - raise ImportError( - "Failed to import VoxCPM AudioVAE. Install the `voxcpm` package or set " - "`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory." - ) from exc - - -def _device_to_string(device: torch.device) -> str: - if device.index is None: - return device.type - return f"{device.type}:{device.index}" - - -def _normalize_dtype_name(dtype: Any) -> str | None: - if dtype is None: - return None - if isinstance(dtype, torch.dtype): - mapping = { - torch.bfloat16: "bfloat16", - torch.float16: "float16", - torch.float32: "float32", - } - return mapping.get(dtype, str(dtype).removeprefix("torch.")) - dtype_str = str(dtype) - return dtype_str.removeprefix("torch.") - - -def _resolve_runtime_device(vllm_config: VllmConfig) -> torch.device: - try: - from vllm_omni.platforms import current_omni_platform - - return current_omni_platform.get_torch_device() - except Exception: - pass - - device = getattr(getattr(vllm_config, "device_config", None), "device", None) - if isinstance(device, torch.device): - return device - if device: - return torch.device(device) - return torch.device("cpu") - - -def _prepare_runtime_model_dir( - model_path: str | Path, - *, - target_device: torch.device, - target_dtype: str | None, -) -> str: - source_dir = Path(model_path) - config_path = source_dir / "config.json" - if not config_path.exists(): - return str(source_dir) - - config_text = config_path.read_text() - config_dict = json.loads(config_text) - desired_device = target_device.type - desired_dtype = target_dtype or config_dict.get("dtype") - - if config_dict.get("device") == desired_device and config_dict.get("dtype") == desired_dtype: - return str(source_dir) - - digest = sha256(f"{source_dir.resolve()}:{config_text}:{desired_device}:{desired_dtype}".encode()).hexdigest()[:16] - runtime_dir = Path(tempfile.gettempdir()) / "vllm_omni_voxcpm_runtime" / digest - runtime_dir.mkdir(parents=True, exist_ok=True) - - for entry in source_dir.iterdir(): - target = runtime_dir / entry.name - if entry.name == "config.json" or target.exists(): - continue - try: - target.symlink_to(entry, target_is_directory=entry.is_dir()) - except OSError as exc: - logger.warning( - "Falling back to copying VoxCPM runtime artifact %s into %s because symlink creation failed: %s", - entry, - runtime_dir, - exc, - ) - if entry.is_dir(): - shutil.copytree(entry, target, dirs_exist_ok=True) - else: - shutil.copy2(entry, target) - - patched_config = dict(config_dict) - patched_config["device"] = desired_device - if desired_dtype is not None: - patched_config["dtype"] = desired_dtype - (runtime_dir / "config.json").write_text(json.dumps(patched_config, indent=2, sort_keys=True)) - return str(runtime_dir) - - -@contextmanager -def _force_cuda_available_for_npu(device: torch.device): - if device.type != "npu": - yield - return - - with patch("torch.cuda.is_available", return_value=True): - yield - - -def _is_torchcodec_load_error(exc: BaseException) -> bool: - message = str(exc).lower() - return "torchcodec" in message or "load_with_torchcodec" in message - - -def _load_audio_with_soundfile( - prompt_wav_path: str, - *, - sample_rate: int, -) -> torch.Tensor: - try: - import soundfile as sf - except ImportError: - raise - - audio_np, source_sr = sf.read(prompt_wav_path, dtype="float32", always_2d=True) - audio = torch.from_numpy(np.ascontiguousarray(audio_np.T)) - - if audio.size(0) > 1: - audio = audio.mean(dim=0, keepdim=True) - - if int(source_sr) != int(sample_rate): - try: - import torchaudio - except ImportError as exc: - raise ImportError("torchaudio is required for resampling prompt audio.") from exc - audio = torchaudio.functional.resample(audio, int(source_sr), int(sample_rate)) - - return audio - - -def _build_prompt_cache_with_soundfile(model: Any, *args: Any, **kwargs: Any) -> dict[str, Any]: - if args: - prompt_text = args[0] - prompt_wav_path = args[1] if len(args) > 1 else kwargs.get("prompt_wav_path") - else: - prompt_text = kwargs.get("prompt_text") - prompt_wav_path = kwargs.get("prompt_wav_path") - - if not prompt_text or not prompt_wav_path: - raise ValueError("prompt_text and prompt_wav_path are required") - - audio = _load_audio_with_soundfile(prompt_wav_path, sample_rate=int(model.sample_rate)) - - patch_len = model.patch_size * model.chunk_size - if audio.size(1) % patch_len != 0: - padding_size = patch_len - audio.size(1) % patch_len - audio = torch.nn.functional.pad(audio, (padding_size, 0)) - - audio_feat = model.audio_vae.encode(audio.to(model.device), model.sample_rate).cpu() - audio_feat = audio_feat.view( - model.audio_vae.latent_dim, - -1, - model.patch_size, - ).permute(1, 2, 0) - - return { - "prompt_text": prompt_text, - "audio_feat": audio_feat, - } diff --git a/vllm_omni/model_executor/models/voxcpm/voxcpm_runtime_utils.py b/vllm_omni/model_executor/models/voxcpm/voxcpm_runtime_utils.py deleted file mode 100644 index 36b4282c2..000000000 --- a/vllm_omni/model_executor/models/voxcpm/voxcpm_runtime_utils.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import json -import shutil -from pathlib import Path - - -def resolve_voxcpm_model_dir(model: str) -> Path: - model_path = Path(model).expanduser() - if model_path.exists(): - return model_path - - from huggingface_hub import snapshot_download - - return Path(snapshot_download(repo_id=model)) - - -def prepare_voxcpm_hf_config_dir(model_dir: str | Path, hf_config_dir: str | Path) -> Path: - model_dir = Path(model_dir).expanduser() - hf_config_dir = Path(hf_config_dir).expanduser() - hf_config_dir.mkdir(parents=True, exist_ok=True) - - source_config_path = model_dir / "config.json" - if not source_config_path.exists(): - raise FileNotFoundError(f"VoxCPM config.json not found under {model_dir}") - - config_path = hf_config_dir / "config.json" - shutil.copy2(source_config_path, config_path) - - source_generation_config_path = model_dir / "generation_config.json" - if source_generation_config_path.exists(): - shutil.copy2(source_generation_config_path, hf_config_dir / "generation_config.json") - - config_dict = json.loads(config_path.read_text(encoding="utf-8")) - config_dict["model_type"] = "voxcpm" - config_dict.setdefault("architectures", ["VoxCPMForConditionalGeneration"]) - config_path.write_text(json.dumps(config_dict, indent=2, ensure_ascii=False), encoding="utf-8") - return hf_config_dir - - -__all__ = [ - "prepare_voxcpm_hf_config_dir", - "resolve_voxcpm_model_dir", -] diff --git a/vllm_omni/model_executor/models/voxcpm/voxcpm_stage_wrappers.py b/vllm_omni/model_executor/models/voxcpm/voxcpm_stage_wrappers.py deleted file mode 100644 index f4446c796..000000000 --- a/vllm_omni/model_executor/models/voxcpm/voxcpm_stage_wrappers.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -import os -from collections.abc import Generator -from typing import Any - -import torch -import torch.nn as nn -from einops import rearrange - - -class _DirectVoxCPMLatentGenerator: - def __init__(self, tts_model: Any): - self.tts_model = tts_model - self.sample_rate = int(getattr(tts_model, "sample_rate", 24000)) - - def generate_latents( - self, - *, - text: str, - prompt_wav_path: str | None = None, - prompt_text: str | None = None, - cfg_value: float = 2.0, - inference_timesteps: int = 10, - min_len: int = 2, - max_len: int = 4096, - retry_badcase: bool = True, - retry_badcase_max_times: int = 3, - retry_badcase_ratio_threshold: float = 6.0, - ) -> torch.Tensor: - if not isinstance(text, str) or not text.strip(): - raise ValueError("target text must be a non-empty string") - if (prompt_wav_path is None) != (prompt_text is None): - raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None") - if prompt_wav_path is not None and not os.path.exists(prompt_wav_path): - raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}") - - prompt_cache = None - if prompt_wav_path is not None and prompt_text is not None: - prompt_cache = self.tts_model.build_prompt_cache( - prompt_text=prompt_text, - prompt_wav_path=prompt_wav_path, - ) - - gen_kw = dict( - target_text=" ".join(text.split()), - prompt_cache=prompt_cache, - min_len=min_len, - max_len=max_len, - inference_timesteps=inference_timesteps, - cfg_value=cfg_value, - retry_badcase=retry_badcase, - retry_badcase_max_times=retry_badcase_max_times, - retry_badcase_ratio_threshold=retry_badcase_ratio_threshold, - ) - latent_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache", None) - if latent_entry is not None: - _, _, pred_audio_feat = latent_entry(**gen_kw) - else: - try: - _, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache( - **gen_kw, - latents_only=True, - ) - except TypeError: - _, _, pred_audio_feat = self.tts_model.generate_with_prompt_cache(**gen_kw) - return pred_audio_feat.detach().cpu().to(torch.float32) - - def iter_latent_chunks_streaming( - self, - *, - text: str, - prompt_wav_path: str | None = None, - prompt_text: str | None = None, - cfg_value: float = 2.0, - inference_timesteps: int = 10, - min_len: int = 2, - max_len: int = 4096, - streaming_prefix_len: int = 3, - retry_badcase: bool = False, - retry_badcase_max_times: int = 3, - retry_badcase_ratio_threshold: float = 6.0, - ) -> Generator[tuple[torch.Tensor, bool], None, None]: - """Yield ``(latent_window, is_last_chunk)`` for Omni async_chunk latent to VAE.""" - if not isinstance(text, str) or not text.strip(): - raise ValueError("target text must be a non-empty string") - if (prompt_wav_path is None) != (prompt_text is None): - raise ValueError("prompt_wav_path and prompt_text must both be provided or both be None") - if prompt_wav_path is not None and not os.path.exists(prompt_wav_path): - raise FileNotFoundError(f"prompt_wav_path does not exist: {prompt_wav_path}") - - prompt_cache = None - if prompt_wav_path is not None and prompt_text is not None: - prompt_cache = self.tts_model.build_prompt_cache( - prompt_text=prompt_text, - prompt_wav_path=prompt_wav_path, - ) - - gen_kw = dict( - target_text=" ".join(text.split()), - prompt_cache=prompt_cache, - min_len=min_len, - max_len=max_len, - inference_timesteps=inference_timesteps, - cfg_value=cfg_value, - retry_badcase=retry_badcase, - retry_badcase_max_times=retry_badcase_max_times, - retry_badcase_ratio_threshold=retry_badcase_ratio_threshold, - streaming_prefix_len=streaming_prefix_len, - ) - stream_entry = getattr(self.tts_model, "generate_latents_with_prompt_cache_streaming", None) - if stream_entry is not None: - gen = stream_entry(**gen_kw) - else: - fallback_stream_entry = getattr(self.tts_model, "generate_with_prompt_cache_streaming", None) - if fallback_stream_entry is not None: - gen = fallback_stream_entry(**gen_kw, latents_only=True) - else: - gen = self.tts_model._generate_with_prompt_cache(streaming=True, latents_only=True, **gen_kw) - - iterator = iter(gen) - previous = next(iterator, None) - while previous is not None: - current = next(iterator, None) - _, _target_tok, chunk_latent = previous - if not isinstance(chunk_latent, torch.Tensor): - chunk_latent = torch.as_tensor(chunk_latent) - yield chunk_latent, current is None - previous = current - - -class _DirectVoxCPMAudioVAE: - def __init__(self, audio_vae: nn.Module, *, patch_size: int = 2): - self.audio_vae = audio_vae - self.sample_rate = int(getattr(audio_vae, "sample_rate", 24000)) - self.latent_dim = int(getattr(audio_vae, "latent_dim", 64)) - self.patch_size = int(patch_size) - self._chunk_size = int(getattr(audio_vae, "chunk_size", 1)) - self._stream_audio_patch_samples = max(1, self.patch_size * self._chunk_size) - - def _prepare_latents_for_decode(self, latent_audio_feat: Any) -> torch.Tensor: - latents = latent_audio_feat - if not isinstance(latents, torch.Tensor): - latents = torch.tensor(latents, dtype=torch.float32) - latents = latents.detach().to(torch.float32) - - if latents.ndim == 3: - if latents.shape[-1] == self.latent_dim: - latents = rearrange(latents, "t p d -> 1 d (t p)") - elif latents.shape[1] == self.latent_dim: - latents = latents.contiguous() - else: - raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}") - elif latents.ndim == 2: - if latents.shape[0] == self.latent_dim: - latents = latents.unsqueeze(0) - elif latents.shape[1] == self.latent_dim: - latents = rearrange(latents, "t d -> 1 d t") - else: - raise ValueError(f"Unsupported latent_audio_feat shape: {tuple(latents.shape)}") - else: - raise ValueError(f"Unsupported latent_audio_feat ndim: {latents.ndim}") - - return latents - - @torch.no_grad() - def decode(self, latent_audio_feat: Any, *, trim_streaming_patch: bool = False) -> torch.Tensor: - latents = self._prepare_latents_for_decode(latent_audio_feat) - device = next(self.audio_vae.parameters()).device - raw = self.audio_vae.decode(latents.to(device=device, dtype=torch.float32)) - if isinstance(raw, dict): - audio = raw.get("audio") - if audio is None: - audio = next(v for v in raw.values() if isinstance(v, torch.Tensor)) - else: - audio = raw - if audio.dim() == 3: - stream = audio.squeeze(1) - elif audio.dim() == 2: - stream = audio - else: - stream = audio.reshape(audio.shape[0], -1) - if trim_streaming_patch: - stream = stream[..., -self._stream_audio_patch_samples :] - return stream.reshape(-1).detach().cpu().to(torch.float32) diff --git a/vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml b/vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml deleted file mode 100644 index c370646a1..000000000 --- a/vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml +++ /dev/null @@ -1,95 +0,0 @@ -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - engine_args: - model_stage: qwen3_tts - max_num_seqs: 10 - model_arch: Qwen3TTSTalkerForConditionalGeneration - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: false - trust_remote_code: true - async_scheduling: true - enable_prefix_caching: false - engine_output_type: latent - gpu_memory_utilization: 0.3 - max_num_batched_tokens: 512 - max_model_len: 4096 - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk - # Use named connector to apply runtime.connectors.extra. - output_connectors: - to_stage_1: connector_of_shared_memory - default_sampling_params: - temperature: 0.9 - top_k: 50 - max_tokens: 4096 - seed: 42 - detokenize: false - repetition_penalty: 1.05 - stop_token_ids: [2150] - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - engine_args: - model_stage: code2wav - max_num_seqs: 10 - model_arch: Qwen3TTSCode2Wav - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: true - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.3 - # Must be divisible by num_code_groups and cover (left_context + chunk). - # Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep headroom past 32k. - max_num_batched_tokens: 65536 - # async_chunk appends windows per step; max_model_len must cover accumulated flat codec stream. - max_model_len: 65536 - engine_input_source: [0] - final_output: true - final_output_type: audio - # Distributed connector configuration - input_connectors: - from_stage_0: connector_of_shared_memory - tts_args: - max_instructions_length: 500 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 65536 - seed: 42 - detokenize: true - repetition_penalty: 1.0 - -runtime: - enabled: true - - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 - # Frame-aligned codec streaming transport. - codec_streaming: true - # Connector polling / timeout (unit: loop count, sleep interval in seconds). - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - # Match the decoder sliding attention window to avoid chunk-boundary noise. - codec_chunk_frames: 25 - codec_left_context_frames: 72 - # Emit only the first audio chunk early, then return to codec_chunk_frames. - initial_codec_chunk_frames: 1 - - edges: - - from: 0 - to: 1 diff --git a/vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml deleted file mode 100644 index c6fd177a3..000000000 --- a/vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml +++ /dev/null @@ -1,98 +0,0 @@ -# VoxCPM two-stage streaming (align with qwen3_tts.yaml async_chunk pattern). -# Stage0 (latent_generator) emits latent in time chunks; Stage1 (VAE) decodes as chunks arrive. -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: bfloat16 - model_stage: latent_generator - model_arch: VoxCPMForConditionalGeneration - # Optional persistent HF-compatible config dir for native VoxCPM models. - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: true - enable_prefix_caching: false - engine_output_type: latent - gpu_memory_utilization: 0.7 - distributed_executor_backend: "mp" - max_num_batched_tokens: 4096 - max_model_len: 4096 - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae_async_chunk - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 4096 - stop_token_ids: [2] - seed: 42 - detokenize: false - repetition_penalty: 1.0 - final_output: false - output_connectors: - to_stage_1: voxcpm_shm - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: float32 - model_stage: vae - model_arch: VoxCPMForConditionalGeneration - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.15 - distributed_executor_backend: "mp" - max_num_batched_tokens: 8192 - max_model_len: 4096 - engine_input_source: [0] - final_output: true - final_output_type: audio - input_connectors: - from_stage_0: voxcpm_shm - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 128 - seed: 42 - detokenize: true - repetition_penalty: 1.0 - - -runtime: - enabled: true - - connectors: - voxcpm_shm: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 - # Frame-aligned codec streaming transport. - codec_streaming: true - # Connector polling / timeout (unit: loop count, sleep interval in seconds). - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - # Align with Omni: small chunks with sufficient context overlap. - codec_chunk_frames: 1 - codec_left_context_frames: 1 - - edges: - - from: 0 - to: 1 diff --git a/vllm_omni/model_executor/stage_input_processors/voxcpm.py b/vllm_omni/model_executor/stage_input_processors/voxcpm.py deleted file mode 100644 index 125335cd2..000000000 --- a/vllm_omni/model_executor/stage_input_processors/voxcpm.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import torch -from vllm.inputs import TextPrompt - -from vllm_omni.data_entry_keys import ( - CodesStruct, - MetaStruct, - OmniPayloadStruct, -) -from vllm_omni.inputs.data import OmniTokensPrompt - -_VOXCPM_LATENT_MAGIC = 131071 - - -def _serialize_latent_to_codes(latent: Any) -> list[int]: - latent_tensor = latent if isinstance(latent, torch.Tensor) else torch.as_tensor(latent) - latent_tensor = latent_tensor.detach().cpu().contiguous() - if latent_tensor.ndim == 3: - if latent_tensor.shape[0] != 1: - raise ValueError(f"Expected batch=1 latent tensor, got shape={tuple(latent_tensor.shape)}") - latent_tensor = latent_tensor.squeeze(0) - if latent_tensor.ndim != 2: - raise ValueError(f"Unsupported latent_audio_feat shape for async chunk: {tuple(latent_tensor.shape)}") - latent_dim, time_dim = int(latent_tensor.shape[0]), int(latent_tensor.shape[1]) - packed = latent_tensor.to(torch.bfloat16).contiguous().view(torch.uint16).reshape(-1).to(torch.int32) - return [_VOXCPM_LATENT_MAGIC, latent_dim, time_dim, *packed.tolist()] - - -def _coerce_finished_flag(value: Any) -> bool: - """Normalize VoxCPM async-chunk finished markers to a Python bool.""" - if value is None: - return False - if isinstance(value, torch.Tensor): - if value.numel() != 1: - raise ValueError(f"finished tensor must be scalar, got shape={tuple(value.shape)}") - return bool(value.detach().cpu().item()) - if isinstance(value, (list, tuple)): - if not value: - return False - if len(value) != 1: - raise ValueError(f"finished container must have one element, got len={len(value)}") - return _coerce_finished_flag(value[0]) - return bool(value) - - -def latent2vae( - source_outputs: list[Any], - _prompt: OmniTokensPrompt | TextPrompt | None = None, - _requires_multimodal_data: bool = False, -) -> list[OmniTokensPrompt]: - vae_inputs: list[OmniTokensPrompt] = [] - for source_output in source_outputs: - output = source_output.outputs[0] - multimodal_output = getattr(output, "multimodal_output", None) - if not isinstance(multimodal_output, dict) or "latent_audio_feat" not in multimodal_output: - raise ValueError( - "VoxCPM latent stage output missing 'latent_audio_feat'. " - f"request_id={getattr(source_output, 'request_id', None)}" - ) - - additional_information = { - "latent_audio_feat": multimodal_output["latent_audio_feat"], - } - if "sr" in multimodal_output: - additional_information["sample_rate"] = [int(multimodal_output["sr"])] - - vae_inputs.append( - OmniTokensPrompt( - prompt_token_ids=[0], - additional_information=additional_information, - multi_modal_data=None, - mm_processor_kwargs=None, - ) - ) - - return vae_inputs - - -def _eof_payload() -> OmniPayloadStruct: - return OmniPayloadStruct( - codes=CodesStruct(audio=torch.empty(0, dtype=torch.long)), - meta=MetaStruct(finished=torch.tensor(True, dtype=torch.bool)), - ) - - -def latent2vae_async_chunk( - transfer_manager: Any, - pooling_output: dict[str, Any] | None, - request: Any, - is_finished: bool = False, -) -> OmniPayloadStruct | None: - """Stage-0 latent → stage-1 VAE under ``async_chunk`` (connector payload).""" - # Kept for callback signature compatibility with OmniChunkTransferAdapter. - _ = transfer_manager - finished_request = _coerce_finished_flag(is_finished) - if callable(getattr(request, "is_finished", None)): - finished_request = finished_request or _coerce_finished_flag(request.is_finished()) - if not isinstance(pooling_output, dict): - return _eof_payload() if finished_request else None - - latent = pooling_output.get("latent_audio_feat") - if isinstance(latent, torch.Tensor) and latent.numel() == 0: - latent = None - - if latent is None: - return _eof_payload() if finished_request else None - - serialized_codes = _serialize_latent_to_codes(latent) - return OmniPayloadStruct( - codes=CodesStruct(audio=torch.tensor(serialized_codes, dtype=torch.long)), - meta=MetaStruct(finished=torch.tensor(finished_request, dtype=torch.bool)), - ) diff --git a/vllm_omni/platforms/npu/stage_configs/voxcpm.yaml b/vllm_omni/platforms/npu/stage_configs/voxcpm.yaml deleted file mode 100644 index dcd1f4051..000000000 --- a/vllm_omni/platforms/npu/stage_configs/voxcpm.yaml +++ /dev/null @@ -1,67 +0,0 @@ -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: bfloat16 - model_stage: latent_generator - model_arch: VoxCPMForConditionalGeneration - # Optional persistent HF-compatible config dir for native VoxCPM models. - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: latent - gpu_memory_utilization: 0.75 - distributed_executor_backend: "mp" - max_num_batched_tokens: 4096 - max_model_len: 4096 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 4096 - seed: 42 - detokenize: false - repetition_penalty: 1.0 - final_output: false - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: float32 - model_stage: vae - model_arch: VoxCPMForConditionalGeneration - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.1 - distributed_executor_backend: "mp" - max_num_batched_tokens: 8192 - max_model_len: 4096 - engine_input_source: [0] - custom_process_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: true - repetition_penalty: 1.0 diff --git a/vllm_omni/platforms/npu/stage_configs/voxcpm_async_chunk.yaml b/vllm_omni/platforms/npu/stage_configs/voxcpm_async_chunk.yaml deleted file mode 100644 index 87843634c..000000000 --- a/vllm_omni/platforms/npu/stage_configs/voxcpm_async_chunk.yaml +++ /dev/null @@ -1,89 +0,0 @@ -async_chunk: true -stage_args: - - stage_id: 0 - stage_type: llm - is_comprehension: true - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: bfloat16 - model_stage: latent_generator - model_arch: VoxCPMForConditionalGeneration - # Optional persistent HF-compatible config dir for native VoxCPM models. - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: ar - scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: latent - gpu_memory_utilization: 0.75 - distributed_executor_backend: "mp" - max_num_batched_tokens: 4096 - max_model_len: 4096 - custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxcpm.latent2vae_async_chunk - output_connectors: - to_stage_1: connector_of_shared_memory - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 4096 - seed: 42 - detokenize: false - repetition_penalty: 1.0 - final_output: false - - - stage_id: 1 - stage_type: llm - runtime: - devices: "0" - max_batch_size: 1 - engine_args: - dtype: float32 - model_stage: vae - model_arch: VoxCPMForConditionalGeneration - hf_config_path: ${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,} - worker_type: generation - scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler - enforce_eager: true - trust_remote_code: true - async_scheduling: false - enable_prefix_caching: false - engine_output_type: audio - gpu_memory_utilization: 0.1 - distributed_executor_backend: "mp" - max_num_batched_tokens: 8192 - max_model_len: 4096 - engine_input_source: [0] - input_connectors: - from_stage_0: connector_of_shared_memory - final_output: true - final_output_type: audio - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 1 - seed: 42 - detokenize: true - repetition_penalty: 1.0 - -runtime: - enabled: true - - connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - shm_threshold_bytes: 65536 - codec_streaming: false - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - - edges: - - from: 0 - to: 1 diff --git a/vllm_omni/transformers_utils/configs/__init__.py b/vllm_omni/transformers_utils/configs/__init__.py index 4f258bbc0..708d9c2b6 100644 --- a/vllm_omni/transformers_utils/configs/__init__.py +++ b/vllm_omni/transformers_utils/configs/__init__.py @@ -17,7 +17,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "FishSpeechConfig": "vllm_omni.transformers_utils.configs.fish_speech", "FishSpeechSlowARConfig": "vllm_omni.transformers_utils.configs.fish_speech", "FishSpeechFastARConfig": "vllm_omni.transformers_utils.configs.fish_speech", - "VoxCPMConfig": "vllm_omni.transformers_utils.configs.voxcpm", "VoxCPM2Config": "vllm_omni.transformers_utils.configs.voxcpm2", "VoxtralTTSConfig": "vllm_omni.transformers_utils.configs.voxtral_tts", "CosyVoice3Config": "vllm_omni.transformers_utils.configs.cosyvoice3", @@ -37,7 +36,6 @@ __all__ = [ "FishSpeechConfig", "FishSpeechSlowARConfig", "FishSpeechFastARConfig", - "VoxCPMConfig", "VoxCPM2Config", "VoxtralTTSConfig", "CosyVoice3Config", @@ -70,6 +68,5 @@ from vllm_omni.transformers_utils.configs import fish_speech as _fish_speech # from vllm_omni.transformers_utils.configs import mammoth_moda2 as _mammoth_moda2 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import ming_flash_omni as _ming_flash_omni # noqa: F401, E402 from vllm_omni.transformers_utils.configs import omnivoice as _omnivoice # noqa: F401, E402 -from vllm_omni.transformers_utils.configs import voxcpm as _voxcpm # noqa: F401, E402 from vllm_omni.transformers_utils.configs import voxcpm2 as _voxcpm2 # noqa: F401, E402 from vllm_omni.transformers_utils.configs import voxtral_tts as _voxtral_tts # noqa: F401, E402 diff --git a/vllm_omni/transformers_utils/configs/voxcpm.py b/vllm_omni/transformers_utils/configs/voxcpm.py deleted file mode 100644 index 026783891..000000000 --- a/vllm_omni/transformers_utils/configs/voxcpm.py +++ /dev/null @@ -1,68 +0,0 @@ -from transformers import AutoConfig -from transformers.configuration_utils import PretrainedConfig - - -class VoxCPMConfig(PretrainedConfig): - model_type = "voxcpm" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - bos_token_id: int = 1, - eos_token_id: int = 2, - vocab_size: int = 32000, - hidden_size: int = 1024, - intermediate_size: int = 4096, - max_position_embeddings: int = 4096, - num_attention_heads: int = 16, - num_hidden_layers: int = 24, - num_key_value_heads: int = 16, - rms_norm_eps: float = 1e-6, - rope_theta: float = 10000.0, - rope_scaling: dict | None = None, - lm_config: dict | None = None, - encoder_config: dict | None = None, - dit_config: dict | None = None, - audio_vae_config: dict | None = None, - patch_size: int = 2, - feat_dim: int = 64, - residual_lm_num_layers: int = 6, - scalar_quantization_latent_dim: int = 256, - scalar_quantization_scale: int = 9, - max_length: int = 4096, - device: str = "cuda", - dtype: str = "bfloat16", - dit_mean_mode: bool = False, - **kwargs, - ): - super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.max_position_embeddings = max_position_embeddings - self.num_attention_heads = num_attention_heads - self.num_hidden_layers = num_hidden_layers - self.num_key_value_heads = num_key_value_heads - self.rms_norm_eps = rms_norm_eps - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - - self.lm_config = lm_config or {} - self.encoder_config = encoder_config or {} - self.dit_config = dit_config or {} - self.audio_vae_config = audio_vae_config - - self.patch_size = patch_size - self.feat_dim = feat_dim - self.residual_lm_num_layers = residual_lm_num_layers - self.scalar_quantization_latent_dim = scalar_quantization_latent_dim - self.scalar_quantization_scale = scalar_quantization_scale - self.max_length = max_length - self.device = device - self.dtype = dtype - self.dit_mean_mode = dit_mean_mode - - -AutoConfig.register("voxcpm", VoxCPMConfig) - -__all__ = ["VoxCPMConfig"]