From b6b868f3debfbeb761a0d3582aaf0eec85f94f45 Mon Sep 17 00:00:00 2001 From: yulun <100981785+Big2Wheel@users.noreply.github.com> Date: Sun, 20 Sep 2026 11:18:12 +0800 Subject: [PATCH] [Hardware][Ascend] Enable VoxCPM2 eager optimizations (#7207) Signed-off-by: Big2Wheel <1404207590@qq.com> --- recipes/OpenBMB/VoxCPM2.md | 70 ++++++++++++-- recipes/README.md | 2 +- .../e2e/offline_inference/test_voxcpm2_tts.py | 8 +- tests/e2e/online_serving/test_voxcpm2_tts.py | 6 +- .../test_voxcpm2_tts_expansion.py | 6 +- .../voxcpm2/test_talker_state_eviction.py | 96 ++++++++++++++++++- .../models/voxcpm2/voxcpm2_talker.py | 82 ++++++++++++---- 7 files changed, 234 insertions(+), 36 deletions(-) diff --git a/recipes/OpenBMB/VoxCPM2.md b/recipes/OpenBMB/VoxCPM2.md index 0fd8e210e..42f770a00 100644 --- a/recipes/OpenBMB/VoxCPM2.md +++ b/recipes/OpenBMB/VoxCPM2.md @@ -43,9 +43,65 @@ sufficient — no tensor parallelism is required. ## Hardware Support -This recipe documents one tested 24 GB consumer-GPU configuration. -Larger-VRAM (H20 / H100 / A100) and other vendor sections (ROCm, NPU) are -welcome as community validation lands. +This recipe documents tested single-card configurations for a 24 GB NVIDIA +GPU and an Ascend Atlas A2. + +## Ascend NPU + +### 1 x Ascend 910B3 64GB (Atlas A2) + +VoxCPM2 runs on one Atlas A2 card with the standard deploy config. The model's +LocDiT path uses eager PyTorch on NPU; keep `enforce_eager: true`. The default +config enables fused QKV, fused gate/up projection, and the zero-delta-time +cache for this path. + +#### Environment + +- Hardware: 1 x Ascend 910B3 64GB (Atlas A2) +- Container: `quay.nju.edu.cn/ascend/vllm-omni:v0.28.0` +- vLLM: 0.28.0 +- vLLM-Omni: current `main` +- Model: `openbmb/VoxCPM2`, bfloat16 + +#### Command + +Expose one physical NPU to the container, then start the server normally. The +visible card is addressed as logical device `0` by `voxcpm2.yaml`. + +```bash +export ASCEND_RT_VISIBLE_DEVICES=5 +export VLLM_WORKER_MULTIPROC_METHOD=spawn + +vllm serve /path/to/VoxCPM2 --omni \ + --host 0.0.0.0 --port 8000 +``` + +#### Performance validation + +The component timer was measured after model warmup with a single Chinese TTS +request. Each decode step executes nine LocDiT estimator calls, so per-step +measurements compare the same amount of model work even when generated audio +duration differs. + +| path | CFM estimator / call | decode / step | forward / step | +| ---- | -------------------- | ------------- | -------------- | +| Before (optimization flags skipped on NPU) | 18.84 ms | 193.86 ms | 265.35 ms | +| Fused QKV + fused MLP + zero-dt cache | 17.50 ms | 182.22 ms | 254.07 ms | +| Above + experimental reduced-precision RoPE | 14.93 ms | 157.36 ms | 227.33 ms | + +The default fused path reduces CFM estimator latency by about **7.1%** and +complete decode-step latency by **6.0%**. Its generated WAV was byte-identical +to the pre-optimization baseline. Setting +`enable_loc_dit_fast_rope: true` can increase the reductions to about **20.8%** +and **18.8%**, respectively, but changes the recurrent acoustic state and is +therefore an opt-in quality/performance tradeoff. Its test audio was valid 48 +kHz mono PCM with finite samples and no clipping; evaluate voice quality on a +representative corpus before enabling it in production. + +Do not set `enforce_eager: false` for VoxCPM2 on Ascend yet. ACL Graph captures +the language-model runner, but the request-owned VoxCPM2 state has dynamic +prefill/decode shapes; replay can therefore feed an empty captured row into a +non-empty residual state and fail at `aclnnCat`. ## GPU @@ -88,8 +144,8 @@ Pass `--deploy-config ` to override. #### Verification **Server cold-start**: ~60 s from `vllm serve` to `Application startup -complete` (subprocess fork + vLLM 0.21 init + model load + flashinfer JIT -+ torch.compile of LocDiT / feat_encoder / AudioVAE + CUDA-Graph warmup). +complete` (subprocess fork, vLLM 0.21 init, model load, flashinfer JIT, +torch.compile of LocDiT / feat_encoder / AudioVAE, and CUDA-Graph warmup). The first request after startup pays a small additional cost; steady-state requests are much faster. @@ -199,7 +255,7 @@ after a one-off ~28 s engine init: | ---- | -------- | --------- | ----- | ------------------------------------------------ | | #1 | 6.72 s | 11.97 s | 1.782 | cold: torch.compile + CUDA-Graph capture | | #2 | 6.24 s | 11.43 s | 1.831 | still runtime warmup | -| #3 | 6.88 s | 0.82 s | 0.120 | ⚡ steady-state | +| #3 | 6.88 s | 0.82 s | 0.120 | steady-state | | #4 | 6.56 s | 0.78 s | 0.119 | steady-state | | #5 | 5.76 s | 0.70 s | 0.121 | steady-state | @@ -231,7 +287,7 @@ Same 5-call methodology as the zero-shot table, this time with a | ---- | -------- | --------- | ----- | ------------------------------------------------ | | #1 | 5.44 s | 12.38 s | 2.276 | cold: compile + CUDA-Graph capture + ref encode | | #2 | 5.12 s | 2.47 s | 0.482 | most warmup done | -| #3 | 5.44 s | 0.76 s | 0.139 | ⚡ steady-state | +| #3 | 5.44 s | 0.76 s | 0.139 | steady-state | | #4 | 4.96 s | 0.68 s | 0.137 | steady-state | | #5 | 5.28 s | 0.71 s | 0.134 | steady-state | diff --git a/recipes/README.md b/recipes/README.md index c84223510..22174a9bb 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -76,7 +76,7 @@ recipes/ | [`cosmos3/Cosmos3-Edge.md`](./cosmos3/Cosmos3-Edge.md) | T2I / T2V / I2V + action (Physical-AI) generation on the Nemotron-based Edge transformer | 1x GPU (~8 GiB) | | [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V / V2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 | | [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online full-duplex serving and offline inference for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB | -| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB | +| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB / 1x Ascend 910B3 64GB | | [`OpenMOSS/MOSS-TTS.md`](./OpenMOSS/MOSS-TTS.md) | Online + offline multilingual TTS (MOSS-TTS family, 8B) | 1x H100 80GB | | [`OpenMOSS/MOSS-TTS-Nano-NPU.md`](./OpenMOSS/MOSS-TTS-Nano-NPU.md) | Online multilingual voice-cloned TTS | 1x Ascend 910B3 64GB | | [`NVIDIA/Nemotron-Labs-Audex.md`](./NVIDIA/Nemotron-Labs-Audex.md) | TTS / text-to-audio / audio understanding / cascaded S2S (Audex 2B + 30B-A3B) | 1x H100 80GB | diff --git a/tests/e2e/offline_inference/test_voxcpm2_tts.py b/tests/e2e/offline_inference/test_voxcpm2_tts.py index 0b6ff9940..67f36254b 100644 --- a/tests/e2e/offline_inference/test_voxcpm2_tts.py +++ b/tests/e2e/offline_inference/test_voxcpm2_tts.py @@ -13,7 +13,7 @@ from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniRunner from tests.helpers.stage_config import get_deploy_config_path -VOXCPM2_MODEL = "openbmb/VoxCPM2" +VOXCPM2_MODEL = os.environ.get("VOXCPM2_MODEL", "openbmb/VoxCPM2") DEPLOY_CONFIG = get_deploy_config_path("voxcpm2.yaml") SAMPLE_RATE = 48000 @@ -44,7 +44,7 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor: @pytest.mark.slow @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: """Test zero-shot TTS produces valid audio output.""" outputs = omni_runner.omni.generate([{"prompt": "Hello, this is a test."}]) @@ -57,7 +57,7 @@ def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None: @pytest.mark.slow @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: """Test voice cloning with a reference audio file. @@ -98,7 +98,7 @@ def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None: @pytest.mark.slow @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) def test_voxcpm2_prefill_decode_mixed_batch_003(omni_runner: OmniRunner) -> None: """Regression: prefill+decode mixed batch must not crash (PR #2903).""" long_prompt = ( diff --git a/tests/e2e/online_serving/test_voxcpm2_tts.py b/tests/e2e/online_serving/test_voxcpm2_tts.py index 0d0364138..1f27d5eb0 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts.py @@ -18,7 +18,7 @@ from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import get_deploy_config_path -MODEL = "openbmb/VoxCPM2" +MODEL = os.environ.get("VOXCPM2_MODEL", "openbmb/VoxCPM2") DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0 MAX_CONCURRENT = 4 @@ -48,7 +48,7 @@ tts_server_params = [ @pytest.mark.slow @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_text_to_audio_001(omni_server, online_client) -> None: """ @@ -73,7 +73,7 @@ def test_text_to_audio_001(omni_server, online_client) -> None: @pytest.mark.slow @pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_text_to_audio_002(omni_server, online_client) -> None: """ diff --git a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py index d630c8627..ad01a9fcb 100644 --- a/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py +++ b/tests/e2e/online_serving/test_voxcpm2_tts_expansion.py @@ -20,7 +20,7 @@ pytestmark = [pytest.mark.slow, pytest.mark.tts] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -MODEL = "openbmb/VoxCPM2" +MODEL = os.environ.get("VOXCPM2_MODEL", "openbmb/VoxCPM2") DEFAULT_AUDIO_SPEECH_TIMEOUT_S = 300.0 _MIN_AUDIO_BYTES = 40_000 MAX_CONCURRENT = 4 @@ -47,7 +47,7 @@ tts_server_params = [ ] -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_voice_clone_streaming_001(omni_server, online_client) -> None: """ @@ -72,7 +72,7 @@ def test_voice_clone_streaming_001(omni_server, online_client) -> None: online_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT) -@hardware_test(res={"cuda": "L4"}, num_cards=1) +@hardware_test(res={"cuda": "L4", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) def test_response_format_001(omni_server, online_client) -> None: """ diff --git a/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py b/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py index e0b52d1aa..0ad8b5df9 100644 --- a/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py +++ b/tests/model_executor/models/voxcpm2/test_talker_state_eviction.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Regression tests for VoxCPM2 talker per-request state lifecycle.""" from __future__ import annotations @@ -72,6 +72,99 @@ def _seed_cached_decode(talker, req_id: str): class TestStateEvictionContract: + def test_eager_optimizations_do_not_require_torch_inductor(self, monkeypatch) -> None: + from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_talker as tk + + class FakeEstimator: + pass + + class FakeFeatDecoder: + def __init__(self, estimator) -> None: + self.estimator = estimator + self.mean_mode = False + + class FakeTTS: + def __init__(self, estimator) -> None: + self.feat_decoder = FakeFeatDecoder(estimator) + + talker = _make_bare_talker() + estimator = FakeEstimator() + talker._tts = FakeTTS(estimator) + _, _, RuntimeConfig = _voxcpm2_talker_mod() + talker._runtime_config = RuntimeConfig() + talker._enable_torch_compile = False + + applied: list[str] = [] + + def install_qkv(*_args, **_kwargs) -> int: + applied.append("qkv") + return 12 + + def install_mlp(*_args, **_kwargs) -> int: + applied.append("mlp") + return 12 + + def install_zero_dt(*_args, **_kwargs) -> bool: + applied.append("zero_dt") + return True + + monkeypatch.setattr( + tk, + "_install_locdit_fused_qkv", + install_qkv, + ) + monkeypatch.setattr( + tk, + "_install_locdit_fused_mlp", + install_mlp, + ) + monkeypatch.setattr( + tk, + "_install_locdit_zero_dt_cache", + install_zero_dt, + ) + + talker._setup_execution_optimizations() + + assert applied == ["qkv", "mlp", "zero_dt"] + assert talker._eager_optimizations_applied is True + assert not hasattr(estimator, "_compiled") + + def test_profiler_drops_invalid_accelerator_graph_events(self, monkeypatch) -> None: + from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_talker as tk + + timer = tk._PerfTimer(enabled=False) + timer._enabled = True + + class InvalidEvent: + def elapsed_time(self, _end) -> float: + raise RuntimeError("event recorder null") + + timer._pairs.append(("captured", InvalidEvent(), object())) + monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None) + + assert timer.breakdown() == "" + assert timer._pairs == [] + + def test_profiler_disables_device_without_timing_events(self, monkeypatch) -> None: + from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_talker as tk + + class DeviceWithoutEvents: + pass + + monkeypatch.setattr( + torch, + "get_device_module", + lambda _device: DeviceWithoutEvents(), + ) + + timer = tk._PerfTimer(enabled=True) + timer.start("cpu") + + assert timer._enabled is False + assert timer._device_module is None + assert timer._starts == {} + def test_runtime_config_normalizes_mutually_exclusive_paths(self) -> None: _, _, RuntimeConfig = _voxcpm2_talker_mod() @@ -295,6 +388,7 @@ class TestChunkedPrefillContract: talker._get_multichar_zh_split = lambda: {} talker._perf = _NoopPerf() talker._enable_torch_compile = False + talker._setup_execution_optimizations = lambda: None talker._setup_cfm_buffers = lambda: None talker._run_cfm_for_state = lambda *_args: torch.ones(1, 1, 4) diff --git a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py index 78c0b26ea..f20e86a89 100644 --- a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py +++ b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py @@ -352,20 +352,44 @@ class _PendingAudioCopy: # =================================================================== +class _TimingEvent(Protocol): + def record(self) -> None: ... + + def elapsed_time(self, end_event: _TimingEvent) -> float: ... + + class _PerfTimer: - __slots__ = ("_enabled", "_timers", "_counts", "_starts", "_pairs") + __slots__ = ( + "_enabled", + "_device_module", + "_timers", + "_counts", + "_starts", + "_pairs", + ) def __init__(self, enabled: bool = False): self._enabled = enabled + self._device_module = None + if enabled: + device_module = torch.get_device_module(current_omni_platform.get_torch_device()) + if not hasattr(device_module, "Event"): + logger.warning_once( + "VoxCPM2 profiler disabled: the current device does not provide accelerator timing events" + ) + self._enabled = False + else: + self._device_module = device_module self._timers: dict[str, float] = {} self._counts: dict[str, int] = {} - self._starts: dict[str, torch.cuda.Event] = {} - self._pairs: list[tuple[str, torch.cuda.Event, torch.cuda.Event]] = [] + self._starts: dict[str, _TimingEvent] = {} + self._pairs: list[tuple[str, _TimingEvent, _TimingEvent]] = [] def start(self, name: str) -> None: if not self._enabled: return - evt = torch.cuda.Event(enable_timing=True) + assert self._device_module is not None + evt = self._device_module.Event(enable_timing=True) evt.record() self._starts[name] = evt @@ -373,7 +397,8 @@ class _PerfTimer: if not self._enabled or name not in self._starts: return start_evt = self._starts.pop(name) - end_evt = torch.cuda.Event(enable_timing=True) + assert self._device_module is not None + end_evt = self._device_module.Event(enable_timing=True) end_evt.record() self._pairs.append((name, start_evt, end_evt)) @@ -382,7 +407,18 @@ class _PerfTimer: return torch.accelerator.synchronize() for name, s, e in self._pairs: - self._timers[name] = self._timers.get(name, 0.0) + s.elapsed_time(e) + try: + elapsed_ms = s.elapsed_time(e) + except RuntimeError as error: + # An event recorded during accelerator graph capture may not + # own a runtime recorder. Profiling must not fail inference. + logger.warning_once( + "VoxCPM2 profiler discarded invalid event pair %r: %s", + name, + error, + ) + continue + self._timers[name] = self._timers.get(name, 0.0) + elapsed_ms self._counts[name] = self._counts.get(name, 0) + 1 self._pairs.clear() @@ -1198,14 +1234,11 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): self._feat_encoder_for_unified_capture = compiled return compiled - def _setup_torch_compile(self) -> None: - if not self._enable_torch_compile: + def _setup_execution_optimizations(self) -> None: + if getattr(self, "_eager_optimizations_applied", False): return tts = self.tts estimator = tts.feat_decoder.estimator - if hasattr(estimator, "_compiled"): - return - targets: list[str] = [] cfg = self._runtime_config @@ -1225,6 +1258,24 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): if _install_locdit_zero_dt_cache(estimator): targets.append("LocDiT zero-dt embedding cache") + # The rewrites above are regular eager PyTorch operations. Ascend does + # not support TorchInductor, but still benefits from fused projections + # and the cached delta-time embedding. + self._eager_optimizations_applied = True + if targets: + logger.info("VoxCPM2: eager optimizations applied to: %s", ", ".join(targets)) + if self._enable_torch_compile: + self._setup_torch_compile() + + def _setup_torch_compile(self) -> None: + tts = self.tts + estimator = tts.feat_decoder.estimator + if hasattr(estimator, "_compiled"): + return + + targets: list[str] = [] + cfg = self._runtime_config + external_cfm_capture = self._enable_cfm_cuda_graph or self._enable_unified_decode_graph if cfg.enable_loc_dit_layer_nvtx: patched = _install_locdit_layer_nvtx(estimator) @@ -1684,8 +1735,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): dtype = self._side_dtype self._setup_cfm_buffers() - if self._enable_torch_compile: - self._setup_torch_compile() + self._setup_execution_optimizations() self.model.precompute_fused_qkv() self.residual_model.precompute_fused_qkv() @@ -2391,8 +2441,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): self._perf.start("prefill_tail_batch") tts = self.tts self._setup_cfm_buffers() - if self._enable_torch_compile: - self._setup_torch_compile() + self._setup_execution_optimizations() dit_hs: list[torch.Tensor] = [] conds: list[torch.Tensor] = [] @@ -2477,8 +2526,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): self._perf.stop("prefill.dit_proj") self._setup_cfm_buffers() - if self._enable_torch_compile: - self._setup_torch_compile() + self._setup_execution_optimizations() pred_feat = self._run_cfm_for_state(state, dit_h, prefix_feat_cond.transpose(1, 2).contiguous())