diff --git a/docs/user_guide/examples/offline_inference/text_to_speech.md b/docs/user_guide/examples/offline_inference/text_to_speech.md index 62e6b96d2..5d16ef349 100644 --- a/docs/user_guide/examples/offline_inference/text_to_speech.md +++ b/docs/user_guide/examples/offline_inference/text_to_speech.md @@ -130,7 +130,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \ ### Notes - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform. -- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config ` to override. +- H200 automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml`; other devices use `cosyvoice3.yaml`. See the [serving defaults and tradeoffs](../online_serving/text_to_speech.md#cosyvoice3). Pass `--deploy-config cosyvoice3.yaml` for the previous profile or `--deploy-config ` to override. The example uses the resolved profile's sampling parameters. - `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path. --- diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index acd7076d4..334e94dad 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -155,16 +155,20 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot ./cosyvoice3/run_server.sh ``` -Streaming is on by default via `async_chunk: true` in `vllm_omni/deploy/cosyvoice3.yaml`. Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path. +Streaming is on by default. On Hopper GPUs with at least 140 GiB of device memory (H200), serving automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml`: AR32 / codec32, standard sampling, packed Flow, cached ISTFT, HiFT decode graphs, and local CUDA MPS. No `--hf-overrides` is needed. Other devices use `cosyvoice3.yaml` with RAS sampling and AR8 / codec8. -Cross-request Stage-1 flow batching is opt-in. Enable it in the server environment when concurrent requests should share a flow-estimator call: +The H200 default targets streaming throughput at C32/C64. It changes sampling from RAS to standard (temperature 0.7, top-p 0.8, top-k 20, repetition penalty 1.21); output lengths and quality can differ. HiFT captures a recurring exact shape on its third use, which adds latency to that request, and its graph pool needs memory headroom beyond the engine reservations. This profile is not a claim of optimal first-audio latency at low concurrency. Pass `--deploy-config cosyvoice3.yaml` to retain the previous configuration, or `--deploy-config ` to select another profile. + +Pass `--no-async-chunk` (or `NO_ASYNC_CHUNK=1 ./cosyvoice3/run_server.sh`) for the legacy synchronous path; use `--deploy-config cosyvoice3.yaml` alongside it when selecting that path on H200. + +Cross-request Stage-1 flow batching is enabled in the H200 profile. For the generic configuration, enable it in the server environment when concurrent requests should share a flow-estimator call: ```bash export COSYVOICE3_BATCH_FLOW=1 vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remote-code ``` -Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead. +Batching preserves output lengths and streaming cache alignment, but the different GEMM shapes can produce small waveform differences compared with processing each request separately. Use `--deploy-config cosyvoice3.yaml` and leave `COSYVOICE3_BATCH_FLOW` unset (or set it to `0`) when request-independent numerical behavior is required. Set `COSYVOICE3_BATCH_FLOW_DEBUG=1` to log the observed group-size distribution and enable detailed Stage-1 profiler scopes; diagnostics are disabled by default to avoid per-step profiling overhead. ### CLI client @@ -194,7 +198,7 @@ The client supports `--api-base`, `--model`, `--text`, `--ref-audio`, `--ref-tex ### Notes - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform. -- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config ` to override. +- Deploy config auto-loads the device-specific CosyVoice3 default described above. Pass `--deploy-config ` to override. - For offline inference and the end-to-end script, see the [offline CosyVoice3 section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3). --- diff --git a/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py b/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py index 40e194120..3d1f6bd7e 100644 --- a/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py +++ b/examples/offline_inference/text_to_speech/cosyvoice3/end2end.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project import argparse +import copy import functools import json import os @@ -10,13 +11,11 @@ from typing import Any import numpy as np import soundfile as sf -from vllm import SamplingParams from vllm.multimodal.media.audio import load_audio from vllm_omni.entrypoints.omni import Omni from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token -from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config # Upstream zero-shot reference clip ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav" @@ -59,8 +58,7 @@ def run_e2e(): "--deploy-config", type=str, default=None, - help="Override the deploy config path. If unset, auto-loads " - "vllm_omni/deploy/cosyvoice3.yaml based on the HF model_type.", + help="Override the deploy config path. If unset, selects the bundled CosyVoice3 profile for the device.", ) parser.add_argument("--text", type=str, default="Hello, this is a test of the CosyVoice system capability.") parser.add_argument( @@ -105,8 +103,6 @@ def run_e2e(): profiler_config=args.profiler_config, ) - sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1} - print("Model initialized. Preparing inputs...") ref_audio_path = args.ref_audio or _default_ref_audio() if not os.path.exists(ref_audio_path): @@ -138,7 +134,7 @@ def run_e2e(): print(f"Generating for prompt: {args.text}") - config = CosyVoice3Config() + config = omni.engine.stage_vllm_configs[0].model_config.hf_config tokenizer = get_qwen_tokenizer( token_path=args.tokenizer, skip_special_tokens=config.skip_special_tokens, @@ -149,29 +145,10 @@ def run_e2e(): min_len = int(base_len * config.min_token_text_ratio) max_len = int(base_len * config.max_token_text_ratio) - # Build SamplingParams for each stage (GPT, S2Mel, Vocoder) - gpt_sampling = SamplingParams( - temperature=1.0, - top_p=sampling_cfg["top_p"], - top_k=sampling_cfg["top_k"], - repetition_penalty=2.0, - min_tokens=min_len, - max_tokens=max_len, - stop_token_ids=[sampling_cfg["eos_token_id"]], - # allowed_token_ids=list(range(6561+3)), - detokenize=False, - ) - # Not used - s2mel_sampling = SamplingParams( - temperature=1.0, - top_p=1.0, - top_k=-1, - repetition_penalty=2.0, - max_tokens=256, - detokenize=False, - ) - - sampling_params_list = [gpt_sampling, s2mel_sampling] + # Keep the deploy profile's sampling policy, penalties and required stops. + sampling_params_list = copy.deepcopy(omni.default_sampling_params_list) + sampling_params_list[0].min_tokens = max(1, min_len) + sampling_params_list[0].max_tokens = min(2048, max_len) profiler_enabled = args.profiler_config is not None if profiler_enabled: diff --git a/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh b/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh index e40b1bb13..76ec73d89 100755 --- a/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh +++ b/examples/online_serving/text_to_speech/cosyvoice3/run_server.sh @@ -1,11 +1,15 @@ #!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + # Launch vLLM-Omni server for CosyVoice3 TTS # # Usage: # ./run_server.sh # CUDA_VISIBLE_DEVICES=0 ./run_server.sh # -# Streaming (async-chunk) is on by default via vllm_omni/deploy/cosyvoice3.yaml. +# Streaming (async-chunk) is on by default. H200 selects the optimized profile; +# other devices use vllm_omni/deploy/cosyvoice3.yaml. # Set NO_ASYNC_CHUNK=1 to use the legacy synchronous path. set -e diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index db97f990d..ef43b85ab 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -571,6 +571,46 @@ class TestPipelineDiscovery: assert "qwen3_omni_moe_thinker_only" in OMNI_PIPELINES assert "qwen3_tts" in OMNI_PIPELINES + @pytest.mark.parametrize( + "cuda,major,memory_gib,optimized", + [ + (False, None, 0, False), + (True, None, 141, False), + (True, 8, 141, False), + (True, 9, 80, False), + (True, 10, 180, False), + (True, 9, 141, True), + ], + ) + def test_cosyvoice3_device_default_and_explicit_overrides(self, monkeypatch, cuda, major, memory_gib, optimized): + from vllm.platforms import current_platform + from vllm.platforms.interface import DeviceCapability + + monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda) + capability = DeviceCapability(major, 0) if major is not None else None + monkeypatch.setattr(current_platform, "get_device_capability", lambda: capability) + monkeypatch.setattr(current_platform, "get_device_total_memory", lambda: memory_gib * 1024**3) + pipeline = resolve_pipeline_config("cosyvoice3") + assert pipeline is not None + config = VllmOmniConfig.from_pipeline_config(pipeline) + talker, codec = config.stage_configs + assert talker.scheduler_config.max_num_seqs == codec.scheduler_config.max_num_seqs == (32 if optimized else 8) + assert talker.runtime_config.cuda_mps == codec.runtime_config.cuda_mps == optimized + assert talker.model_config.hf_overrides == ({"cosyvoice3_sampling_mode": "standard"} if optimized else None) + assert pipeline.stages[0].sampling_constraints["stop_token_ids"] == list(range(6561, 6761)) + if optimized: + assert codec.runtime_config.env["COSYVOICE3_CACHED_ISTFT"] == "1" + assert codec.runtime_config.env["COSYVOICE3_HIFT_GRAPH"] == "1" + override = VllmOmniConfig.from_pipeline_config( + pipeline, cli_overrides={"hf_overrides": {"cosyvoice3_sampling_mode": "ras", "seed": 7}} + ) + assert override.stage_by_id(0).model_config.hf_overrides == {"cosyvoice3_sampling_mode": "ras", "seed": 7} + explicit = VllmOmniConfig.from_pipeline_config( + pipeline, deploy_config_path=get_deploy_config_path("cosyvoice3.yaml") + ) + assert explicit.stage_by_id(0).scheduler_config.max_num_seqs == 8 + assert explicit.stage_by_id(0).model_config.hf_overrides is None + def test_registry_resolver_qwen3_omni_all_stages(self): """Test that providing the HF config for qwen3 omni with audio enabled uses all stages.""" pipeline = resolve_pipeline_config( diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index cbcb94cdb..1aa80ed92 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -54,7 +54,7 @@ from vllm_omni.model_executor.models.cosmos3.pipeline import ( COSMOS3_OMNI_DEPLOY_PIPELINE, COSMOS3_POLICY_PIPELINE, ) -from vllm_omni.model_executor.models.cosyvoice3.pipeline import COSYVOICE3_PIPELINE +from vllm_omni.model_executor.models.cosyvoice3.pipeline import resolve_cosyvoice3_pipeline from vllm_omni.model_executor.models.covo_audio.pipeline import COVO_AUDIO_PIPELINE from vllm_omni.model_executor.models.dreamzero.pipeline import DREAMZERO_PIPELINE from vllm_omni.model_executor.models.fish_speech.pipeline import FISH_SPEECH_PIPELINE @@ -150,7 +150,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { # key (see deploy/cosmos3_policy_droid.yaml and deploy/cosmos3_omni.yaml). "cosmos3_omni_deploy": COSMOS3_OMNI_DEPLOY_PIPELINE, "cosmos3_policy": COSMOS3_POLICY_PIPELINE, - "cosyvoice3": COSYVOICE3_PIPELINE, + "cosyvoice3": resolve_cosyvoice3_pipeline, "covo_audio": COVO_AUDIO_PIPELINE, "dreamzero": DREAMZERO_PIPELINE, "fish_qwen3_omni": FISH_SPEECH_PIPELINE, diff --git a/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml b/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml index 0d738a0b1..c0ea4174f 100644 --- a/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml +++ b/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml @@ -1,5 +1,6 @@ -# Opt-in single-Hopper streaming throughput profile: AR32 / codec32, Flow10. -# Use --hf-overrides '{"cosyvoice3_sampling_mode": "standard"}' and stream=true. +# Single-Hopper streaming throughput profile: AR32 / codec32, Flow10. +# Automatically selected on Hopper devices with at least 140 GiB (H200). +# Standard sampling is configured below; request stream=true for first audio. # F0 and HiFT stay FP32; only deterministic HiFT decode is graphed. # HiFT graphs capture on the third use of each exact shape, adding latency then. # The codec graph pool is not capped by gpu_memory_utilization; leave memory headroom. @@ -7,6 +8,8 @@ base_config: cosyvoice3_packed_streaming.yaml stages: - stage_id: 0 max_num_seqs: 32 + hf_overrides: + cosyvoice3_sampling_mode: standard default_sampling_params: max_tokens: 2048 repetition_penalty: 1.21 diff --git a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py index d6bb0e5bb..e156debcf 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py +++ b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """CosyVoice3 pipeline topology (frozen). Stage 0: Talker — text prompt → speech tokens (LLM autoregressive). @@ -12,11 +12,16 @@ Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform. through the shared-memory connector. """ +from dataclasses import replace + +from transformers import PretrainedConfig + from vllm_omni.config.stage_config import ( PipelineConfig, StageExecutionType, StagePipelineConfig, ) +from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config _PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3" @@ -35,8 +40,9 @@ COSYVOICE3_PIPELINE = PipelineConfig( async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"), custom_process_next_stage_input_func=f"{_PROC}.text2flow_full_payload", sampling_constraints={ - # merged speech stop token (logsumexp of all 200 stop logits) - "stop_token_ids": [6562], + # Standard sampling can emit any of the 200 control tokens. + # RAS merges their logits into 6562, which is also in this range. + "stop_token_ids": list(range(6561, 6761)), }, ), StagePipelineConfig( @@ -52,3 +58,27 @@ COSYVOICE3_PIPELINE = PipelineConfig( ), ), ) + + +def resolve_cosyvoice3_pipeline(hf_config: PretrainedConfig | None = None) -> PipelineConfig | None: + """Use the validated throughput profile on H200-class Hopper devices. + + Query the platform through NVML without initializing CUDA in the parent. + Explicit deploy configs still take precedence over this pipeline default. + """ + from vllm.platforms import current_platform + + if hf_config is not None and not isinstance(hf_config, CosyVoice3Config): + return None + if current_platform.is_cuda(): + capability = current_platform.get_device_capability() + if ( + capability is not None + and capability.major == 9 + and current_platform.get_device_total_memory() >= 140 * 1024**3 + ): + return replace( + COSYVOICE3_PIPELINE, + default_deploy_config_name="cosyvoice3_packed_streaming_optimized_standard.yaml", + ) + return COSYVOICE3_PIPELINE