[Bugfix][MiniCPM-o] Stabilize code2wav CFM CUDA graph caching with reference audio normalization and mel-frame bucketing (#7416)

Signed-off-by: y_null <y_null@qq.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
y-nullandamy-why-3459 authored and GitHub committed 2026-09-16 01:24:31 -07:00
1 parent 507cb1d834
commit 5f7f62e13c
13 files changed
+838 -25

No files matched your search

@@ -46,6 +46,80 @@ def relpos_encode_token_budget(
return max(lookahead + 1, min(int(cap), room))
def _cfm_pad_frames(
*,
mel_frames: int,
offset: int,
noise_capacity: int,
bucket_frames: int,
disabled: bool,
) -> int:
"""Frame padding that aligns one CFM call onto the capture-shape grid.
Chunk lengths vary per request: the steady-state chunk is 50 mel frames,
but the first/last chunk and the ``plan_token2wav_encode_slices`` splits
land on arbitrary lengths, and every distinct length is its own CUDA-graph
capture shape. Padding the frame axis up to ``bucket_frames`` collapses
them onto one grid (the caller trims the output back). Measured on the
shipped config, 128 requests / concurrency 8: 61 captures and 1 cache
flush without bucketing versus 15 captures and 0 flushes with it, audio
RTF 1.317 -> 1.191 under otherwise identical conditions.
The attention and CNN caches also advance by the padded width so their
shapes stay on the same grid; in practice that is bounded by the trim in
``_decode_batch_once`` (the cache width saturates at ``prompt_len +
100``) rather than by the decoder's ``rand_noise`` capacity.
Returns 0 when bucketing does not apply: the ragged valid-lengths path,
graphs unavailable or already disabled by ``CFMGraphWrapper._disable``,
or padding that would overflow the decoder's noise buffer.
"""
if disabled or bucket_frames <= 1:
return 0
pad = (bucket_frames - mel_frames % bucket_frames) % bucket_frames
if offset + mel_frames + pad > noise_capacity:
return 0
return pad
def _zero_padded_frames(tensor: torch.Tensor, valid_frames: int | None) -> None:
"""Keep the padded columns of ``tensor`` at zero, in place.
Bucketing pads the frame axis so the capture shape stays fixed. The padded
columns carry no valid content: they are excluded from the attention by the
``attn_mask`` built in ``_decode_batch_once``, and zeroing keeps them from
leaking back in. The integration step ``x = x + dt * velocity`` would
otherwise make the padded region non-zero again after the initial zeroing,
and the cross-chunk caches would read those values back.
"""
if valid_frames is None or valid_frames >= int(tensor.shape[-1]):
return
tensor[..., valid_frames:] = 0.0
def _zero_padded_cnn_cache(
cnn_cache: torch.Tensor,
estimator: nn.Module,
pad_frames: int,
) -> None:
"""Clear the cache positions that come from padded frames, in place.
Each block's CNN cache holds the tail of its convolution output
(``new_cnn_cache = x[..., -causal_padding[0]:]``, inside ``stepaudio2``),
so when the padding sits at the chunk tail those positions are
padding-derived and would otherwise become the next chunk's left context.
Padding is at most ``pad_frames`` wide, so only the trailing positions that
can come from it are cleared; the valid part of the window is kept.
"""
for index, block in enumerate(estimator.blocks):
width = int(block.conv.block[1].causal_padding[0])
if width <= 0:
continue
zero_from = max(0, width - pad_frames)
if zero_from < width:
cnn_cache[index][..., zero_from:] = 0.0
def plan_token2wav_encode_slices(
num_frames: int,
*,
@@ -249,6 +323,17 @@ class BatchedToken2Wav(nn.Module):
"CFM CUDA Graph is disabled on device type %s",
flow_parameter.device.type if flow_parameter is not None else "unknown",
)
# mel-frame bucket size for the CFM CUDA Graph path. Pad each decode
# chunk up to a multiple of this many frames so the graph cache key
# space stays small (0 disables bucketing, e.g. when graphs are off).
self._cfm_graph_bucket_frames = (
int(cfm_graph_cfg.get("bucket_frames", 0)) if self._cfm_graph_wrapper is not None else 0
)
if self._cfm_graph_bucket_frames > 1:
logger.info(
"CFM CUDA Graph bucketing enabled (bucket_frames=%d)",
self._cfm_graph_bucket_frames,
)
self._prompt_features: dict[tuple[str, str], PromptFeatures] = {}
def _hift_inference(
@@ -398,6 +483,7 @@ class BatchedToken2Wav(nn.Module):
att_cache: torch.Tensor | None,
attn_mask: torch.Tensor | None = None,
valid_lengths: list[int] | None = None,
valid_frames: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
if self._trt_stepper is not None and valid_lengths is None:
out, new_cnn, new_att = self._trt_stepper.step(
@@ -413,6 +499,13 @@ class BatchedToken2Wav(nn.Module):
time_embedding = estimator.t_embedder(time).unsqueeze(1)
width = int(x.shape[-1])
speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width)
if valid_frames is not None and valid_frames < width:
# Every frame gets the speaker vector, so the padded columns would
# stay non-zero even with x/mu/cond zeroed -- and the estimator
# would then emit non-zero features there, leaving the
# chunk-boundary attention/CNN caches padding-derived. Zero them.
speaker_features = speaker_features.clone()
_zero_padded_frames(speaker_features, valid_frames)
estimator_input = torch.cat((x, mu, speaker_features, cond), dim=1)
cnn_out, att_out = self._estimator_buffers(estimator, estimator_input, att_cache)
old_cnn: Any = cnn_cache if cnn_cache is not None else [None] * len(estimator.blocks)
@@ -433,6 +526,7 @@ class BatchedToken2Wav(nn.Module):
graph_att,
cnn_out,
att_out,
attn_mask,
)
if valid_lengths is not None:
if not hasattr(estimator, "in_proj"):
@@ -566,6 +660,26 @@ class BatchedToken2Wav(nn.Module):
estimator = decoder.estimator
batch_size = int(mu.shape[0])
offset = int(att_cache.shape[4]) if att_cache is not None else 0
mel_frames = int(mu.shape[2])
pad_frames = _cfm_pad_frames(
mel_frames=mel_frames,
offset=offset,
noise_capacity=int(decoder.rand_noise.shape[2]),
bucket_frames=self._cfm_graph_bucket_frames,
disabled=(
valid_lengths is not None
or self._cfm_graph_wrapper is None
# `_disable` keeps the wrapper object alive, so check the flag
# too: padding under a disabled wrapper is pure overhead.
or not self._cfm_graph_wrapper.enabled
),
)
if pad_frames:
# Replicate rather than zero: a repeated last frame is a closer
# continuation of the chunk than silence, so the padded positions
# carry less of a step change into the attention and CNN caches.
mu = torch.nn.functional.pad(mu, (0, pad_frames), mode="replicate")
cond = torch.nn.functional.pad(cond, (0, pad_frames), mode="replicate")
end = offset + int(mu.shape[2])
if end > int(decoder.rand_noise.shape[2]):
raise RuntimeError(
@@ -574,6 +688,13 @@ class BatchedToken2Wav(nn.Module):
f'"available":{int(decoder.rand_noise.shape[2])}}}'
)
x = decoder.rand_noise[:, :, offset:end].expand(batch_size, -1, -1).clone()
if pad_frames:
# The padded columns would otherwise carry real noise values. They
# are excluded from this chunk's attention by ``attn_mask`` below
# and held at zero here, so the padded region stays inert. Masking
# is what removes them; zeroing alone would not, since a zero row
# still occupies part of the softmax denominator.
x[:, :, mel_frames:] = 0.0
timeline = torch.linspace(
0,
1,
@@ -603,6 +724,20 @@ class BatchedToken2Wav(nn.Module):
device=mu.device,
)
attn_mask = valid_queries.unsqueeze(2) & torch.cat((current_keys, old_keys), dim=2)
elif pad_frames:
# Mask the padded keys instead of only zeroing their content: a
# zero-valued key/value pair still takes probability mass out of the
# softmax denominator, so the real frames keep attending to the
# padding unless it is explicitly excluded.
kv_len = int(mu.shape[2]) + offset
attn_mask = torch.ones(
2 * batch_size,
int(mu.shape[2]),
kv_len,
dtype=torch.bool,
device=mu.device,
)
attn_mask[:, :, mel_frames : mel_frames + pad_frames] = False
next_cnn: list[torch.Tensor] = []
next_att_cache: torch.Tensor | None = None
ragged_att_cache: list[torch.Tensor] | None = None
@@ -622,10 +757,14 @@ class BatchedToken2Wav(nn.Module):
att_cache=old_att,
attn_mask=attn_mask,
valid_lengths=valid_lengths,
valid_frames=mel_frames if pad_frames else None,
)
if pad_frames:
_zero_padded_cnn_cache(step_cnn, estimator, pad_frames)
conditional, unconditional = estimate.split(batch_size, dim=0)
velocity = (1.0 + decoder.inference_cfg_rate) * conditional - decoder.inference_cfg_rate * unconditional
x = x + dt * velocity
_zero_padded_frames(x, mel_frames if pad_frames else None)
time = time + dt
if step + 1 < self.n_timesteps:
dt = timeline[step + 2] - time[0]
@@ -666,7 +805,15 @@ class BatchedToken2Wav(nn.Module):
dtype=self._estimator_att_cache_dtype,
)
next_att_cache[step].copy_(step_att)
if pad_frames:
# The padded steps ran but hold no valid content, and each
# cache entry becomes the next chunk's keys. Clear the
# padded columns so the boundary the next chunk reads is
# the valid one, in step with the mask built above.
next_att_cache[step][..., mel_frames : mel_frames + pad_frames, :] = 0.0
del step_att
if pad_frames:
x = x[:, :, :mel_frames]
if ragged_att_cache is not None:
return x, torch.stack(next_cnn), ragged_att_cache
assert next_att_cache is not None
@@ -135,11 +135,13 @@ class HiFTGraphWrapper:
return speech, cache_source
def _tensor_signature(value: torch.Tensor) -> tuple:
def _tensor_signature(value: torch.Tensor | None) -> tuple:
if value is None:
return (None,)
return tuple(value.shape), str(value.dtype), str(value.device)
_DTYPE_MAP = {str(dtype): dtype for dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64)}
_DTYPE_MAP = {str(dtype): dtype for dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64, torch.bool)}
def _memory_snapshot(device: torch.device) -> tuple[int, int] | None:
@@ -231,7 +233,7 @@ class CFMGraphWrapper:
return {**self._stats, "cache_size": len(self._cache)}
def _call_graph_fn(self, args: tuple[torch.Tensor, ...]) -> torch.Tensor:
return self.graph_fn(args[0], args[1], None, args[2], args[3], args[4], args[5])
return self.graph_fn(args[0], args[1], args[6], args[2], args[3], args[4], args[5])
def _eager(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
self._stats["eager"] += 1
@@ -261,10 +263,21 @@ class CFMGraphWrapper:
self.enabled = False
self._flush()
def _capture(self, key: tuple) -> tuple | None:
"""Capture a CUDA graph for the given key. Returns None on failure."""
def _capture(self, key: tuple, inputs: tuple | None = None) -> tuple | None:
"""Capture a CUDA graph for the given key. Returns None on failure.
``inputs`` should be the real tensors for this key whenever they are
available: the static buffers must be built from the values the graph
will actually run with. Zero-filled placeholders are only a fallback --
capturing with them bakes the placeholder branch into the graph (e.g. an
attention mask that looks like "nothing is masked").
"""
try:
static_inputs = _tensors_from_key(key)
static_inputs = (
tuple(None if tensor is None else tensor.detach().clone() for tensor in inputs)
if inputs is not None
else _tensors_from_key(key)
)
except KeyError:
# An unsupported dtype is a property of this shape alone, so run it
# eager and keep the other shapes on graphs.
@@ -315,8 +328,9 @@ class CFMGraphWrapper:
att_cache: torch.Tensor,
cnn_out: torch.Tensor,
att_out: torch.Tensor,
attn_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
inputs = (estimator_input, time_emb, cnn_cache, att_cache, cnn_out, att_out)
inputs = (estimator_input, time_emb, cnn_cache, att_cache, cnn_out, att_out, attn_mask)
self._stats["calls"] += 1
if not self.enabled or torch.cuda.is_current_stream_capturing() or estimator_input.device.type != "cuda":
@@ -330,7 +344,7 @@ class CFMGraphWrapper:
if entry is None:
if len(self._cache) >= self.max_graphs:
self._flush()
entry = self._capture(key)
entry = self._capture(key, inputs)
if entry is None:
return self._eager(inputs)
self._cache[key] = entry
@@ -339,7 +353,8 @@ class CFMGraphWrapper:
static_inputs, static_output, graph = entry
for static, current in zip(static_inputs, inputs, strict=True):
static.copy_(current)
if static is not None:
static.copy_(current)
graph.replay()
return (
static_output.detach().clone(),
@@ -9,6 +9,7 @@ import os
import tempfile
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from functools import lru_cache
from hashlib import sha256
from pathlib import Path
from typing import Any
@@ -57,6 +58,92 @@ def _scalar(value: Any, default: Any = None) -> Any:
return default if value is None else value
_REF_TARGET_SAMPLE_RATE = 24000
# Overridable per deployment via ``ref_audio_max_seconds``.
_REF_MAX_SECONDS = 6.0
# Channel ceiling for the optional (channels, samples) downmix guard.
_REF_MAX_CHANNELS = 8
@lru_cache(maxsize=8)
def _get_resampler(orig_freq: int, new_freq: int):
"""Cached anti-aliased resampler matching token2wav's own loader.
``torchaudio`` is imported lazily (it is not a hard runtime requirement of
this model on every platform) and the transform is built once per rate pair
instead of once per request, keeping it off the first-packet path.
"""
import torchaudio
return torchaudio.transforms.Resample(orig_freq=orig_freq, new_freq=new_freq)
def _read_reference_wav(path: str) -> tuple[Any, int]:
"""Read a WAV file for :func:`_normalize_reference`.
``soundfile`` returns ``(samples, channels)`` for multi-channel files
while the normalizer expects ``(channels, samples)``. Without the
transpose a stereo prompt looks like a huge channel count, the downmix
guard rejects it, and the caller silently falls back to the raw file --
a second L0 value for the same content.
"""
waveform, sample_rate_hz = sf.read(path, dtype="float32", always_2d=False)
if getattr(waveform, "ndim", 1) > 1:
waveform = waveform.T
return waveform, int(sample_rate_hz)
def _normalize_reference(
ref_audio: Any,
sample_rate_hz: int,
*,
max_seconds: float = _REF_MAX_SECONDS,
) -> tuple[torch.Tensor, int]:
"""Normalize a per-request reference waveform onto a shared grid.
MiniCPM-o streaming uses the reference-audio length as the CFM attention
cache origin (L0). Unnormalized references give every request a distinct
L0, which explodes the CFM CUDA-graph cache key space (see #6628). Fold
every reference onto the same sample rate and a fixed length (truncate
long ones, zero-pad short ones) so L0 becomes one constant value.
References longer than ``max_seconds`` are truncated with a warning;
the window is configurable via ``ref_audio_max_seconds``.
"""
tensor = torch.as_tensor(ref_audio, dtype=torch.float32)
if tensor.dim() > 1:
# (channels, samples) -> mono; plain reshape(-1) would interleave.
# Upstream flattens request references to 1-D, so anything else is a
# non-standard caller: reject layouts we cannot downmix unambiguously
# instead of silently averaging the waveform away.
if tensor.shape[0] > _REF_MAX_CHANNELS:
raise ValueError(f"reference audio must be 1-D or (channels, samples); got shape {tuple(tensor.shape)}")
tensor = tensor.mean(dim=0)
waveform = tensor.reshape(-1).cpu().contiguous()
if waveform.numel() == 0:
# Keep the caller's "empty_ref_audio" error path intact: an empty
# waveform must not be zero-padded into a valid-length reference.
return waveform, _REF_TARGET_SAMPLE_RATE
if sample_rate_hz != _REF_TARGET_SAMPLE_RATE:
# Match token2wav's own loader (torchaudio, anti-aliased): it only
# resamples when the stored rate differs, so this output is what the
# model consumes.
waveform = _get_resampler(sample_rate_hz, _REF_TARGET_SAMPLE_RATE)(waveform.view(1, -1)).view(-1).contiguous()
max_samples = max(1, int(max_seconds * _REF_TARGET_SAMPLE_RATE))
if waveform.numel() > max_samples:
logger.warning(
"Reference audio is %.2fs, truncating to the %.2fs window (set ``ref_audio_max_seconds`` to keep more).",
waveform.numel() / _REF_TARGET_SAMPLE_RATE,
max_seconds,
)
waveform = waveform[:max_samples].contiguous()
elif waveform.numel() < max_samples:
# Zero-pad short references so every request shares one L0; trailing
# silence has minimal style impact (verified via E3 WER/SIM).
waveform = torch.nn.functional.pad(waveform, (0, max_samples - waveform.numel()))
return waveform, _REF_TARGET_SAMPLE_RATE
def _codec_tensor(value: Any, fallback: torch.Tensor) -> torch.Tensor:
if isinstance(value, torch.Tensor):
return value.reshape(-1).to(device=fallback.device, dtype=torch.long)
@@ -181,7 +268,11 @@ class MiniCPMO45Code2Wav(nn.Module):
self._cfm_graph_config = {
"enabled": bool(extra.get("enable_cfm_graph", False)),
"max_graphs": int(extra.get("cfm_max_graphs", 32)),
"bucket_frames": int(extra.get("cfm_graph_bucket_frames", 0)),
}
self._ref_max_seconds = float(extra.get("ref_audio_max_seconds", _REF_MAX_SECONDS))
if self._ref_max_seconds <= 0:
raise ValueError("MiniCPM-o Code2Wav ref_audio_max_seconds must be > 0")
self._min_batch_size = int(extra.get("code2wav_min_batch_size", 1))
if self._min_batch_size < 1:
raise ValueError("MiniCPM-o Code2Wav code2wav_min_batch_size must be >= 1")
@@ -192,6 +283,7 @@ class MiniCPMO45Code2Wav(nn.Module):
raise ValueError("MiniCPM-o Code2Wav code2wav_initial_batch_size must be 0 or >= code2wav_min_batch_size")
self._default_prompt_id = str(extra.get("prompt_cache_id", "HT_ref_audio"))
self._prompt_wav_override = extra.get("prompt_wav")
self._default_prompt_normalized: tuple[str, str] | None = None
@property
def _default_prompt_wav(self) -> str:
@@ -199,6 +291,46 @@ class MiniCPMO45Code2Wav(nn.Module):
return str(self._prompt_wav_override)
return str(Path(self.model_path) / "assets" / "HT_ref_audio.wav")
def _normalized_default_prompt(self) -> tuple[str, str]:
"""Fold the shipped default prompt onto the request-reference grid.
The shipped asset is 6.016 s and is loaded directly by token2wav, so a
request without a reference would otherwise keep a second L0 value a
couple of frames away from the normalized request references -- two
graph-key families instead of one (both use ``ref_audio_max_seconds``).
Returns ``(prompt_wav,
prompt_cache_id)``; if the asset cannot be read, the shipped path is
returned unchanged.
"""
if self._default_prompt_normalized is not None:
return self._default_prompt_normalized
source = self._default_prompt_wav
fallback = (source, self._default_prompt_id)
try:
waveform, sample_rate_hz = _read_reference_wav(source)
normalized, target_sr = _normalize_reference(
waveform,
sample_rate_hz,
max_seconds=self._ref_max_seconds,
)
except Exception:
logger.warning("Could not normalize the default prompt %s; using it as-is", source)
self._default_prompt_normalized = fallback
return self._default_prompt_normalized
if normalized.numel() == 0:
self._default_prompt_normalized = fallback
return self._default_prompt_normalized
digest = sha256()
digest.update(normalized.numpy().tobytes())
digest.update(str(target_sr).encode())
cache_id = f"default-ref-{digest.hexdigest()[:24]}-{target_sr}"
path = Path(self._runtime_prompt_dir.name) / f"{cache_id}.wav"
if not path.is_file():
sf.write(path, normalized.numpy(), target_sr, format="WAV")
logger.info("Default prompt normalized onto the %.2fs window: %s", self._ref_max_seconds, path)
self._default_prompt_normalized = (str(path), cache_id)
return self._default_prompt_normalized
def _extra_config(self) -> dict[str, Any]:
model_config = getattr(self.vllm_config, "model_config", None)
connector = getattr(model_config, "stage_connector_config", None)
@@ -220,9 +352,13 @@ class MiniCPMO45Code2Wav(nn.Module):
sample_rate: Any,
) -> tuple[str, _RuntimePrompt]:
sample_rate_hz = int(_scalar(sample_rate, 0))
waveform = torch.as_tensor(ref_audio, dtype=torch.float32).reshape(-1).cpu().contiguous()
if sample_rate_hz <= 0:
raise _batch_error("invalid_ref_audio_sample_rate", sample_rate=sample_rate_hz)
waveform, sample_rate_hz = _normalize_reference(
ref_audio,
sample_rate_hz,
max_seconds=self._ref_max_seconds,
)
if waveform.numel() == 0:
raise _batch_error("empty_ref_audio")
if not bool(torch.isfinite(waveform).all().item()):
@@ -288,9 +424,10 @@ class MiniCPMO45Code2Wav(nn.Module):
if entry is not None:
return entry.cache_id, entry.path, cache_key
default_wav, default_cache_id = self._normalized_default_prompt()
return (
str(_scalar(meta.get("prompt_cache_id"), self._default_prompt_id)),
str(_scalar(meta.get("prompt_wav"), self._default_prompt_wav)),
str(_scalar(meta.get("prompt_cache_id"), default_cache_id)),
str(_scalar(meta.get("prompt_wav"), default_wav)),
None,
)