[MiniCPM-o][NPU] Enable MiniCPM-o Code2Wav NPUGraph replay (#5604)

Signed-off-by: Zhou248 <62542847+Zhou248@users.noreply.github.com>
This commit is contained in:
Zhou248 authored and GitHub committed 2026-08-17 15:09:51 +08:00
1 parent 170d498771
commit 5d09cf27a9
15 files changed
+1129 -30

No files matched your search

+333
View File
@@ -0,0 +1,333 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.platforms.npu.graph_tools import (
CapturedDeviceGraph,
NPUExactGraphRunner,
)
from vllm_omni.platforms.npu.models import minicpmo_4_5_code2wav as code2wav_patch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_captured_graph_replay_clones_persistent_outputs():
static_input = torch.zeros(1)
static_output = torch.zeros(1)
class _Graph:
def replay(self):
static_output.copy_(static_input * 2)
graph = CapturedDeviceGraph(
graph=_Graph(),
static_inputs=(static_input,),
static_outputs=(static_output,),
)
first = graph.replay((torch.tensor([2.0]),))[0]
second = graph.replay((torch.tensor([3.0]),))[0]
torch.testing.assert_close(first, torch.tensor([4.0]))
torch.testing.assert_close(second, torch.tensor([6.0]))
assert first.data_ptr() != second.data_ptr()
def test_capture_uses_vllm_global_graph_pool(monkeypatch):
runner = NPUExactGraphRunner()
global_pool = object()
seen_pools = []
synchronizations = 0
class _Graph:
def replay(self):
pass
@contextmanager
def graph_context(graph, *, pool):
del graph
seen_pools.append(pool)
yield
def synchronize():
nonlocal synchronizations
synchronizations += 1
fake_npu = SimpleNamespace(
NPUGraph=_Graph,
graph=graph_context,
synchronize=synchronize,
)
monkeypatch.setattr(torch, "npu", fake_npu, raising=False)
import vllm.platforms
monkeypatch.setattr(
vllm.platforms,
"current_platform",
SimpleNamespace(get_global_graph_pool=lambda: global_pool),
)
captured = runner.capture(
(torch.tensor([2.0]),),
lambda value: (value + 1,),
)
assert seen_pools == [global_pool]
assert synchronizations == 2
torch.testing.assert_close(captured.static_outputs[0], torch.tensor([3.0]))
def test_exact_signature_dispatch_captures_then_replays(monkeypatch):
runner = NPUExactGraphRunner(max_graphs=2)
replay_count = 0
class _FunctionalGraph:
def __init__(self, compute):
self.compute = compute
def replay(self, inputs):
nonlocal replay_count
replay_count += 1
return tuple(output.detach().clone() for output in self.compute(*inputs))
monkeypatch.setattr(runner, "_eligible", lambda inputs: True)
monkeypatch.setattr(
runner,
"capture",
lambda inputs, compute: _FunctionalGraph(compute),
)
def compute(value):
return (value * 2,)
first = runner.run("unit", (torch.tensor([2.0]),), (False,), compute)
second = runner.run("unit", (torch.tensor([3.0]),), (False,), compute)
torch.testing.assert_close(first[0], torch.tensor([4.0]))
torch.testing.assert_close(second[0], torch.tensor([6.0]))
assert runner.stats == {"captures": 1, "failed": 0, "hits": 1}
assert replay_count == 1
def test_capture_failure_is_fatal_for_stage_process(monkeypatch):
runner = NPUExactGraphRunner(
component_name="test component",
disable_config_hint="disable the test graph",
)
captures = 0
computes = 0
monkeypatch.setattr(runner, "_eligible", lambda inputs: True)
def fail_capture(inputs, compute):
nonlocal captures
captures += 1
raise RuntimeError("unsupported op")
monkeypatch.setattr(runner, "capture", fail_capture)
def compute(value):
nonlocal computes
computes += 1
return (value + 1,)
with pytest.raises(RuntimeError, match="restart the stage process"):
runner.run("unit", (torch.tensor([1.0]),), (False,), compute)
with pytest.raises(RuntimeError, match="cannot continue"):
runner.run(
"different-shape",
(torch.tensor([1.0, 2.0]),),
(True,),
compute,
)
assert captures == 1
assert computes == 1
assert runner.stats == {"captures": 0, "failed": 1, "hits": 0}
def test_code2wav_runtime_disables_internal_format_and_jit(monkeypatch):
monkeypatch.delenv("ASCEND_LAUNCH_BLOCKING", raising=False)
compile_modes = []
fake_npu = SimpleNamespace(
config=SimpleNamespace(allow_internal_format=True),
set_compile_mode=lambda **kwargs: compile_modes.append(kwargs),
)
monkeypatch.setattr(torch, "npu", fake_npu, raising=False)
code2wav_patch.prepare_code2wav_graph_runtime()
assert fake_npu.config.allow_internal_format is False
assert compile_modes == [{"jit_compile": False}]
def test_code2wav_runtime_rejects_launch_blocking(monkeypatch):
monkeypatch.setenv("ASCEND_LAUNCH_BLOCKING", "1")
with pytest.raises(RuntimeError, match="ASCEND_LAUNCH_BLOCKING=1"):
code2wav_patch.prepare_code2wav_graph_runtime()
@pytest.mark.parametrize(
("enabled", "max_graphs", "expected_prepared"),
[
(True, 7, 1),
(False, 7, 0),
(True, 0, 0),
],
)
def test_code2wav_patch_reads_stage_additional_config(
monkeypatch,
enabled,
max_graphs,
expected_prepared,
):
prepared = 0
class _Backend:
speech_window = torch.zeros(1)
flow = SimpleNamespace(training=False)
model = SimpleNamespace(
backend=None,
vllm_config=SimpleNamespace(
additional_config={
"code2wav_enable_npu_graph": enabled,
"code2wav_max_npu_graphs": max_graphs,
}
),
)
def build_backend(instance):
instance.backend = _Backend()
def prepare_runtime():
nonlocal prepared
prepared += 1
monkeypatch.setattr(code2wav_patch, "_original_build_backend", build_backend)
monkeypatch.setattr(code2wav_patch, "prepare_code2wav_graph_runtime", prepare_runtime)
code2wav_patch._patched_build_backend(model)
assert prepared == expected_prepared
if expected_prepared:
graph_runner = code2wav_patch._backend_graph_runners[model.backend]
assert isinstance(graph_runner, NPUExactGraphRunner)
assert graph_runner.max_graphs == max_graphs
else:
assert model.backend not in code2wav_patch._backend_graph_runners
@pytest.mark.parametrize("with_cache", [False, True])
def test_code2wav_estimator_graph_dispatch_is_platform_owned(monkeypatch, with_cache):
calls = []
class _Backend:
_trt_stepper = None
_cfm_graph_wrapper = None
class _Runner:
def run(self, operation, inputs, constants, compute):
calls.append((operation, inputs, constants))
return compute(*inputs)
backend = _Backend()
runner = _Runner()
code2wav_patch._backend_graph_runners[backend] = runner
def graphable_estimator_step(
instance,
estimator,
*,
x,
mu,
time_embedding,
speakers,
cond,
cnn_cache,
att_cache,
):
del instance, estimator, time_embedding, speakers, cond
cache_marker = x.new_tensor(0 if cnn_cache is None else 1)
return x + mu, cache_marker, cache_marker.clone()
monkeypatch.setattr(code2wav_patch, "_original_estimator_step", lambda *args, **kwargs: None)
monkeypatch.setattr(code2wav_patch, "_graphable_estimator_step", graphable_estimator_step)
value = torch.tensor([2.0])
cache = torch.tensor([1.0]) if with_cache else None
estimator = SimpleNamespace(t_embedder=lambda time: time + 1)
outputs = code2wav_patch._patched_estimator_step(
backend,
estimator,
x=value,
mu=value,
time=value,
speakers=value,
cond=value,
cnn_cache=cache,
att_cache=cache,
)
assert calls[0][0] == "cfm_estimator"
assert calls[0][2] == (with_cache,)
assert len(calls[0][1]) == (7 if with_cache else 5)
torch.testing.assert_close(calls[0][1][2], torch.tensor([[3.0]]))
torch.testing.assert_close(outputs[0], torch.tensor([4.0]))
torch.testing.assert_close(outputs[1], torch.tensor(1.0 if with_cache else 0.0))
def test_code2wav_platform_wraps_flow_execution_context(monkeypatch):
entered = []
class _Backend:
pass
class _Context:
def __init__(self, device, require_math):
self.device = device
self.require_math = require_math
def __enter__(self):
entered.append((self.device, self.require_math))
def __exit__(self, exc_type, exc, traceback):
return False
backend = _Backend()
code2wav_patch._backend_graph_runners[backend] = NPUExactGraphRunner()
monkeypatch.setattr(
code2wav_patch,
"_flow_execution_context",
lambda device, *, require_math: _Context(device, require_math),
)
monkeypatch.setattr(
code2wav_patch,
"_original_setup_batch",
lambda instance, features, batch_size: ("setup", batch_size),
)
monkeypatch.setattr(
code2wav_patch,
"_original_decode_batch",
lambda instance, tokens, features, states, *, last_chunk, flush_encoder: (
"decode",
last_chunk,
flush_encoder,
),
)
features = SimpleNamespace(speech_tokens=torch.zeros(1))
assert code2wav_patch._patched_setup_batch(backend, features, 3) == ("setup", 3)
assert code2wav_patch._patched_decode_batch(
backend,
torch.zeros(1),
features,
[],
last_chunk=True,
) == ("decode", True, False)
assert entered == [(torch.device("cpu"), True), (torch.device("cpu"), True)]
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Ascend hardware validation for MiniCPM-o Code2Wav NPUGraph replay."""
from math import prod
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from tests.helpers.mark import hardware_marks
from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import (
BatchedToken2Wav,
)
from vllm_omni.platforms.npu.graph_tools import NPUExactGraphRunner
from vllm_omni.platforms.npu.models.minicpmo_4_5_code2wav import (
_graphable_estimator_step,
prepare_code2wav_graph_runtime,
)
from vllm_omni.platforms.npu.models.step_audio2_token2wav import (
npu_token2wav_sdpa_context,
)
pytestmark = [
pytest.mark.core_model,
pytest.mark.omni,
*hardware_marks(res={"npu": "A3"}, num_cards=1),
]
class _Flow(nn.Module):
def __init__(self, estimator: nn.Module):
super().__init__()
self.decoder = SimpleNamespace(estimator=estimator)
class _Token2Wav:
def __init__(self, estimator: nn.Module):
self.flow = _Flow(estimator).eval()
self.hift = nn.Identity().eval()
self.float16 = False
self.n_timesteps = 2
self.mel_cache_len = 1
self.source_cache_len = 2
self.speech_window = torch.ones(4, device="npu")
def test_minicpmo_cfm_estimator_npugraph_matches_eager():
decoder_dit = pytest.importorskip("cosyvoice2.flow.decoder_dit")
prepare_code2wav_graph_runtime()
estimator = (
decoder_dit.DiT(
in_channels=6,
out_channels=2,
depth=2,
num_heads=2,
head_dim=8,
hidden_size=16,
mlp_ratio=2.0,
)
.npu()
.eval()
)
graph_runner = NPUExactGraphRunner(max_graphs=4)
adapter = BatchedToken2Wav(_Token2Wav(estimator))
def inputs(seed: int):
def make(shape: tuple[int, ...], offset: float):
values = torch.arange(
prod(shape),
device="npu",
dtype=torch.float32,
)
return torch.sin(values * 0.17 + seed + offset).reshape(shape)
return (
make((2, 2, 6), 0.0),
make((2, 2, 6), 0.5),
make((2, 1, 16), 1.0),
make((2, 1), 1.5),
make((2, 1, 6), 2.0),
)
def compute(*values, caches=None):
cnn_cache, att_cache = (None, None) if caches is None else caches
return _graphable_estimator_step(
adapter,
estimator,
x=values[0],
mu=values[1],
time_embedding=values[2],
speakers=values[3],
cond=values[4],
cnn_cache=cnn_cache,
att_cache=att_cache,
)
with torch.inference_mode(), npu_token2wav_sdpa_context(require_math=True):
warm_inputs = inputs(11)
warm_outputs = graph_runner.run(
"cfm_estimator",
warm_inputs,
(False,),
lambda *values: compute(*values),
)
replay_inputs = inputs(13)
expected = compute(*replay_inputs)
actual = graph_runner.run(
"cfm_estimator",
replay_inputs,
(False,),
lambda *values: compute(*values),
)
for eager, replayed in zip(expected, actual, strict=True):
torch.testing.assert_close(replayed, eager, rtol=1e-3, atol=1e-3)
cached_inputs = (*inputs(17), warm_outputs[1], warm_outputs[2])
graph_runner.run(
"cfm_estimator",
cached_inputs,
(True,),
lambda *values: compute(*values[:5], caches=(values[5], values[6])),
)
replay_cached_inputs = (
*inputs(19),
warm_outputs[1] + 0.1,
warm_outputs[2] + 0.1,
)
expected_cached = compute(
*replay_cached_inputs[:5],
caches=(replay_cached_inputs[5], replay_cached_inputs[6]),
)
actual_cached = graph_runner.run(
"cfm_estimator",
replay_cached_inputs,
(True,),
lambda *values: compute(*values[:5], caches=(values[5], values[6])),
)
for eager, replayed in zip(expected_cached, actual_cached, strict=True):
torch.testing.assert_close(replayed, eager, rtol=1e-3, atol=1e-3)
assert graph_runner.stats == {"captures": 2, "failed": 0, "hits": 2}
@@ -7,6 +7,7 @@ from __future__ import annotations
import importlib.util
import sys
import types
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
@@ -194,3 +195,71 @@ def test_hift_patch_reports_incompatible_layout(monkeypatch: pytest.MonkeyPatch)
with pytest.raises(TypeError, match=r"m_source\.l_sin_gen\._f02sine"):
module.patch_step_audio2_hift_for_npu(SimpleNamespace())
def test_sdpa_context_preserves_exceptions_from_token2wav_body(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_patch_module(monkeypatch)
fake_attention_patch = types.ModuleType("vllm_omni.platforms.npu.models.cosyvoice2_dit_attn")
fake_attention_patch.apply_cosyvoice2_dit_attn_npu_patch = lambda: None
@contextmanager
def fake_math_context(*, require_available=False):
del require_available
yield
fake_attention_patch.npu_math_sdpa_context = fake_math_context
monkeypatch.setitem(
sys.modules,
"vllm_omni.platforms.npu.models.cosyvoice2_dit_attn",
fake_attention_patch,
)
with pytest.raises(RuntimeError, match="original graph failure"):
with module.npu_token2wav_sdpa_context():
raise RuntimeError("original graph failure")
def test_sdpa_context_requires_math_for_npu_graph(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_patch_module(monkeypatch)
required = []
fake_attention_patch = types.ModuleType("vllm_omni.platforms.npu.models.cosyvoice2_dit_attn")
fake_attention_patch.apply_cosyvoice2_dit_attn_npu_patch = lambda: None
@contextmanager
def fake_math_context(*, require_available=False):
required.append(require_available)
yield
fake_attention_patch.npu_math_sdpa_context = fake_math_context
monkeypatch.setitem(
sys.modules,
"vllm_omni.platforms.npu.models.cosyvoice2_dit_attn",
fake_attention_patch,
)
with module.npu_token2wav_sdpa_context(require_math=True):
pass
assert required == [True]
def test_sdpa_context_does_not_hide_patch_setup_failures(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_patch_module(monkeypatch)
fake_attention_patch = types.ModuleType("vllm_omni.platforms.npu.models.cosyvoice2_dit_attn")
def fail_patch():
raise RuntimeError("attention patch failed")
fake_attention_patch.apply_cosyvoice2_dit_attn_npu_patch = fail_patch
fake_attention_patch.npu_math_sdpa_context = contextmanager(lambda: iter((None,)))
monkeypatch.setitem(
sys.modules,
"vllm_omni.platforms.npu.models.cosyvoice2_dit_attn",
fake_attention_patch,
)
with pytest.raises(RuntimeError, match="attention patch failed"):
with module.npu_token2wav_sdpa_context():
pass