diff --git a/docs/user_guide/examples/offline_inference/text_to_speech.md b/docs/user_guide/examples/offline_inference/text_to_speech.md index 5d16ef349..350f344f1 100644 --- a/docs/user_guide/examples/offline_inference/text_to_speech.md +++ b/docs/user_guide/examples/offline_inference/text_to_speech.md @@ -130,7 +130,7 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \ ### Notes - Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform. -- H200 automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml`; other devices use `cosyvoice3.yaml`. See the [serving defaults and tradeoffs](../online_serving/text_to_speech.md#cosyvoice3). Pass `--deploy-config cosyvoice3.yaml` for the previous profile or `--deploy-config ` to override. The example uses the resolved profile's sampling parameters. +- H200 automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when the CUDA MPS control tool is available; other devices and environments use `cosyvoice3.yaml`. See the [serving defaults and tradeoffs](../online_serving/text_to_speech.md#cosyvoice3). Pass `--deploy-config cosyvoice3.yaml` for the previous profile or `--deploy-config ` to override. The example uses the resolved profile's sampling parameters. - `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path. --- diff --git a/docs/user_guide/examples/online_serving/text_to_speech.md b/docs/user_guide/examples/online_serving/text_to_speech.md index 334e94dad..69aed26e4 100644 --- a/docs/user_guide/examples/online_serving/text_to_speech.md +++ b/docs/user_guide/examples/online_serving/text_to_speech.md @@ -155,7 +155,9 @@ vllm serve FunAudioLLM/Fun-CosyVoice3-0.5B-2512 --omni --port 8091 --trust-remot ./cosyvoice3/run_server.sh ``` -Streaming is on by default. On Hopper GPUs with at least 140 GiB of device memory (H200), serving automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml`: AR32 / codec32, standard sampling, packed Flow, cached ISTFT, HiFT decode graphs, and local CUDA MPS. No `--hf-overrides` is needed. Other devices use `cosyvoice3.yaml` with RAS sampling and AR8 / codec8. +Streaming is on by default. On CUDA Hopper GPUs with at least 140 GiB of device memory (H200), serving automatically selects `cosyvoice3_packed_streaming_optimized_standard.yaml` when `nvidia-cuda-mps-control` is available on `PATH`: AR32 / codec32, standard sampling, packed Flow, cached ISTFT, HiFT decode graphs, and local CUDA MPS. No `--hf-overrides` is needed. Other devices and environments without the MPS control tool use `cosyvoice3.yaml` with RAS sampling and AR8 / codec8. + +The optimized profile is not a universal hardware configuration. Its packed kernels are Hopper-only; the talker's Model Runner V2 is unsupported on NPU/XPU, and CUDA MPS requires a local NVIDIA GPU and its control tool. Larger batches, 25-frame chunks, and graph allocations also require per-device validation on smaller or slower GPUs. Automatic selection retains the generic profile outside the validated device class; an explicit deploy configuration takes precedence. The H200 default targets streaming throughput at C32/C64. It changes sampling from RAS to standard (temperature 0.7, top-p 0.8, top-k 20, repetition penalty 1.21); output lengths and quality can differ. HiFT captures a recurring exact shape on its third use, which adds latency to that request, and its graph pool needs memory headroom beyond the engine reservations. This profile is not a claim of optimal first-audio latency at low concurrency. Pass `--deploy-config cosyvoice3.yaml` to retain the previous configuration, or `--deploy-config ` to select another profile. diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index ef43b85ab..fe870a7c1 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -572,17 +572,20 @@ class TestPipelineDiscovery: assert "qwen3_tts" in OMNI_PIPELINES @pytest.mark.parametrize( - "cuda,major,memory_gib,optimized", + "cuda,major,memory_gib,mps,optimized", [ - (False, None, 0, False), - (True, None, 141, False), - (True, 8, 141, False), - (True, 9, 80, False), - (True, 10, 180, False), - (True, 9, 141, True), + (False, None, 0, True, False), + (True, None, 141, True, False), + (True, 8, 141, True, False), + (True, 9, 80, True, False), + (True, 10, 180, True, False), + (True, 9, 141, False, False), + (True, 9, 141, True, True), ], ) - def test_cosyvoice3_device_default_and_explicit_overrides(self, monkeypatch, cuda, major, memory_gib, optimized): + def test_cosyvoice3_device_default_and_explicit_overrides( + self, monkeypatch, cuda, major, memory_gib, mps, optimized + ): from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability @@ -590,6 +593,10 @@ class TestPipelineDiscovery: capability = DeviceCapability(major, 0) if major is not None else None monkeypatch.setattr(current_platform, "get_device_capability", lambda: capability) monkeypatch.setattr(current_platform, "get_device_total_memory", lambda: memory_gib * 1024**3) + monkeypatch.setattr( + "vllm_omni.model_executor.models.cosyvoice3.pipeline.shutil.which", + lambda command: "/usr/bin/nvidia-cuda-mps-control" if mps else None, + ) pipeline = resolve_pipeline_config("cosyvoice3") assert pipeline is not None config = VllmOmniConfig.from_pipeline_config(pipeline) diff --git a/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml b/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml index c0ea4174f..000751af3 100644 --- a/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml +++ b/vllm_omni/deploy/cosyvoice3_packed_streaming_optimized_standard.yaml @@ -1,5 +1,6 @@ # Single-Hopper streaming throughput profile: AR32 / codec32, Flow10. -# Automatically selected on Hopper devices with at least 140 GiB (H200). +# Automatically selected on Hopper devices with at least 140 GiB (H200) +# when nvidia-cuda-mps-control is available on PATH. # Standard sampling is configured below; request stream=true for first audio. # F0 and HiFT stay FP32; only deterministic HiFT decode is graphed. # HiFT graphs capture on the third use of each exact shape, adding latency then. diff --git a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py index e156debcf..e40bd5670 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/pipeline.py +++ b/vllm_omni/model_executor/models/cosyvoice3/pipeline.py @@ -12,6 +12,7 @@ Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform. through the shared-memory connector. """ +import shutil from dataclasses import replace from transformers import PretrainedConfig @@ -61,7 +62,7 @@ COSYVOICE3_PIPELINE = PipelineConfig( def resolve_cosyvoice3_pipeline(hf_config: PretrainedConfig | None = None) -> PipelineConfig | None: - """Use the validated throughput profile on H200-class Hopper devices. + """Select a streaming default compatible with the device and runtime. Query the platform through NVML without initializing CUDA in the parent. Explicit deploy configs still take precedence over this pipeline default. @@ -76,6 +77,7 @@ def resolve_cosyvoice3_pipeline(hf_config: PretrainedConfig | None = None) -> Pi capability is not None and capability.major == 9 and current_platform.get_device_total_memory() >= 140 * 1024**3 + and shutil.which("nvidia-cuda-mps-control") is not None ): return replace( COSYVOICE3_PIPELINE,