[Bugfix][MiniCPM-o] Stabilize code2wav CFM CUDA graph caching with reference audio normalization and mel-frame bucketing (#7416)
Signed-off-by: y_null <y_null@qq.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
507cb1d834
commit
5f7f62e13c
13 files changed
+838
-25
No files matched your search
@@ -46,6 +46,80 @@ def relpos_encode_token_budget(
|
||||
return max(lookahead + 1, min(int(cap), room))
|
||||
|
||||
|
||||
def _cfm_pad_frames(
|
||||
*,
|
||||
mel_frames: int,
|
||||
offset: int,
|
||||
noise_capacity: int,
|
||||
bucket_frames: int,
|
||||
disabled: bool,
|
||||
) -> int:
|
||||
"""Frame padding that aligns one CFM call onto the capture-shape grid.
|
||||
|
||||
Chunk lengths vary per request: the steady-state chunk is 50 mel frames,
|
||||
but the first/last chunk and the ``plan_token2wav_encode_slices`` splits
|
||||
land on arbitrary lengths, and every distinct length is its own CUDA-graph
|
||||
capture shape. Padding the frame axis up to ``bucket_frames`` collapses
|
||||
them onto one grid (the caller trims the output back). Measured on the
|
||||
shipped config, 128 requests / concurrency 8: 61 captures and 1 cache
|
||||
flush without bucketing versus 15 captures and 0 flushes with it, audio
|
||||
RTF 1.317 -> 1.191 under otherwise identical conditions.
|
||||
|
||||
The attention and CNN caches also advance by the padded width so their
|
||||
shapes stay on the same grid; in practice that is bounded by the trim in
|
||||
``_decode_batch_once`` (the cache width saturates at ``prompt_len +
|
||||
100``) rather than by the decoder's ``rand_noise`` capacity.
|
||||
|
||||
Returns 0 when bucketing does not apply: the ragged valid-lengths path,
|
||||
graphs unavailable or already disabled by ``CFMGraphWrapper._disable``,
|
||||
or padding that would overflow the decoder's noise buffer.
|
||||
"""
|
||||
if disabled or bucket_frames <= 1:
|
||||
return 0
|
||||
pad = (bucket_frames - mel_frames % bucket_frames) % bucket_frames
|
||||
if offset + mel_frames + pad > noise_capacity:
|
||||
return 0
|
||||
return pad
|
||||
|
||||
|
||||
def _zero_padded_frames(tensor: torch.Tensor, valid_frames: int | None) -> None:
|
||||
"""Keep the padded columns of ``tensor`` at zero, in place.
|
||||
|
||||
Bucketing pads the frame axis so the capture shape stays fixed. The padded
|
||||
columns carry no valid content: they are excluded from the attention by the
|
||||
``attn_mask`` built in ``_decode_batch_once``, and zeroing keeps them from
|
||||
leaking back in. The integration step ``x = x + dt * velocity`` would
|
||||
otherwise make the padded region non-zero again after the initial zeroing,
|
||||
and the cross-chunk caches would read those values back.
|
||||
"""
|
||||
if valid_frames is None or valid_frames >= int(tensor.shape[-1]):
|
||||
return
|
||||
tensor[..., valid_frames:] = 0.0
|
||||
|
||||
|
||||
def _zero_padded_cnn_cache(
|
||||
cnn_cache: torch.Tensor,
|
||||
estimator: nn.Module,
|
||||
pad_frames: int,
|
||||
) -> None:
|
||||
"""Clear the cache positions that come from padded frames, in place.
|
||||
|
||||
Each block's CNN cache holds the tail of its convolution output
|
||||
(``new_cnn_cache = x[..., -causal_padding[0]:]``, inside ``stepaudio2``),
|
||||
so when the padding sits at the chunk tail those positions are
|
||||
padding-derived and would otherwise become the next chunk's left context.
|
||||
Padding is at most ``pad_frames`` wide, so only the trailing positions that
|
||||
can come from it are cleared; the valid part of the window is kept.
|
||||
"""
|
||||
for index, block in enumerate(estimator.blocks):
|
||||
width = int(block.conv.block[1].causal_padding[0])
|
||||
if width <= 0:
|
||||
continue
|
||||
zero_from = max(0, width - pad_frames)
|
||||
if zero_from < width:
|
||||
cnn_cache[index][..., zero_from:] = 0.0
|
||||
|
||||
|
||||
def plan_token2wav_encode_slices(
|
||||
num_frames: int,
|
||||
*,
|
||||
@@ -249,6 +323,17 @@ class BatchedToken2Wav(nn.Module):
|
||||
"CFM CUDA Graph is disabled on device type %s",
|
||||
flow_parameter.device.type if flow_parameter is not None else "unknown",
|
||||
)
|
||||
# mel-frame bucket size for the CFM CUDA Graph path. Pad each decode
|
||||
# chunk up to a multiple of this many frames so the graph cache key
|
||||
# space stays small (0 disables bucketing, e.g. when graphs are off).
|
||||
self._cfm_graph_bucket_frames = (
|
||||
int(cfm_graph_cfg.get("bucket_frames", 0)) if self._cfm_graph_wrapper is not None else 0
|
||||
)
|
||||
if self._cfm_graph_bucket_frames > 1:
|
||||
logger.info(
|
||||
"CFM CUDA Graph bucketing enabled (bucket_frames=%d)",
|
||||
self._cfm_graph_bucket_frames,
|
||||
)
|
||||
self._prompt_features: dict[tuple[str, str], PromptFeatures] = {}
|
||||
|
||||
def _hift_inference(
|
||||
@@ -398,6 +483,7 @@ class BatchedToken2Wav(nn.Module):
|
||||
att_cache: torch.Tensor | None,
|
||||
attn_mask: torch.Tensor | None = None,
|
||||
valid_lengths: list[int] | None = None,
|
||||
valid_frames: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if self._trt_stepper is not None and valid_lengths is None:
|
||||
out, new_cnn, new_att = self._trt_stepper.step(
|
||||
@@ -413,6 +499,13 @@ class BatchedToken2Wav(nn.Module):
|
||||
time_embedding = estimator.t_embedder(time).unsqueeze(1)
|
||||
width = int(x.shape[-1])
|
||||
speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width)
|
||||
if valid_frames is not None and valid_frames < width:
|
||||
# Every frame gets the speaker vector, so the padded columns would
|
||||
# stay non-zero even with x/mu/cond zeroed -- and the estimator
|
||||
# would then emit non-zero features there, leaving the
|
||||
# chunk-boundary attention/CNN caches padding-derived. Zero them.
|
||||
speaker_features = speaker_features.clone()
|
||||
_zero_padded_frames(speaker_features, valid_frames)
|
||||
estimator_input = torch.cat((x, mu, speaker_features, cond), dim=1)
|
||||
cnn_out, att_out = self._estimator_buffers(estimator, estimator_input, att_cache)
|
||||
old_cnn: Any = cnn_cache if cnn_cache is not None else [None] * len(estimator.blocks)
|
||||
@@ -433,6 +526,7 @@ class BatchedToken2Wav(nn.Module):
|
||||
graph_att,
|
||||
cnn_out,
|
||||
att_out,
|
||||
attn_mask,
|
||||
)
|
||||
if valid_lengths is not None:
|
||||
if not hasattr(estimator, "in_proj"):
|
||||
@@ -566,6 +660,26 @@ class BatchedToken2Wav(nn.Module):
|
||||
estimator = decoder.estimator
|
||||
batch_size = int(mu.shape[0])
|
||||
offset = int(att_cache.shape[4]) if att_cache is not None else 0
|
||||
mel_frames = int(mu.shape[2])
|
||||
pad_frames = _cfm_pad_frames(
|
||||
mel_frames=mel_frames,
|
||||
offset=offset,
|
||||
noise_capacity=int(decoder.rand_noise.shape[2]),
|
||||
bucket_frames=self._cfm_graph_bucket_frames,
|
||||
disabled=(
|
||||
valid_lengths is not None
|
||||
or self._cfm_graph_wrapper is None
|
||||
# `_disable` keeps the wrapper object alive, so check the flag
|
||||
# too: padding under a disabled wrapper is pure overhead.
|
||||
or not self._cfm_graph_wrapper.enabled
|
||||
),
|
||||
)
|
||||
if pad_frames:
|
||||
# Replicate rather than zero: a repeated last frame is a closer
|
||||
# continuation of the chunk than silence, so the padded positions
|
||||
# carry less of a step change into the attention and CNN caches.
|
||||
mu = torch.nn.functional.pad(mu, (0, pad_frames), mode="replicate")
|
||||
cond = torch.nn.functional.pad(cond, (0, pad_frames), mode="replicate")
|
||||
end = offset + int(mu.shape[2])
|
||||
if end > int(decoder.rand_noise.shape[2]):
|
||||
raise RuntimeError(
|
||||
@@ -574,6 +688,13 @@ class BatchedToken2Wav(nn.Module):
|
||||
f'"available":{int(decoder.rand_noise.shape[2])}}}'
|
||||
)
|
||||
x = decoder.rand_noise[:, :, offset:end].expand(batch_size, -1, -1).clone()
|
||||
if pad_frames:
|
||||
# The padded columns would otherwise carry real noise values. They
|
||||
# are excluded from this chunk's attention by ``attn_mask`` below
|
||||
# and held at zero here, so the padded region stays inert. Masking
|
||||
# is what removes them; zeroing alone would not, since a zero row
|
||||
# still occupies part of the softmax denominator.
|
||||
x[:, :, mel_frames:] = 0.0
|
||||
timeline = torch.linspace(
|
||||
0,
|
||||
1,
|
||||
@@ -603,6 +724,20 @@ class BatchedToken2Wav(nn.Module):
|
||||
device=mu.device,
|
||||
)
|
||||
attn_mask = valid_queries.unsqueeze(2) & torch.cat((current_keys, old_keys), dim=2)
|
||||
elif pad_frames:
|
||||
# Mask the padded keys instead of only zeroing their content: a
|
||||
# zero-valued key/value pair still takes probability mass out of the
|
||||
# softmax denominator, so the real frames keep attending to the
|
||||
# padding unless it is explicitly excluded.
|
||||
kv_len = int(mu.shape[2]) + offset
|
||||
attn_mask = torch.ones(
|
||||
2 * batch_size,
|
||||
int(mu.shape[2]),
|
||||
kv_len,
|
||||
dtype=torch.bool,
|
||||
device=mu.device,
|
||||
)
|
||||
attn_mask[:, :, mel_frames : mel_frames + pad_frames] = False
|
||||
next_cnn: list[torch.Tensor] = []
|
||||
next_att_cache: torch.Tensor | None = None
|
||||
ragged_att_cache: list[torch.Tensor] | None = None
|
||||
@@ -622,10 +757,14 @@ class BatchedToken2Wav(nn.Module):
|
||||
att_cache=old_att,
|
||||
attn_mask=attn_mask,
|
||||
valid_lengths=valid_lengths,
|
||||
valid_frames=mel_frames if pad_frames else None,
|
||||
)
|
||||
if pad_frames:
|
||||
_zero_padded_cnn_cache(step_cnn, estimator, pad_frames)
|
||||
conditional, unconditional = estimate.split(batch_size, dim=0)
|
||||
velocity = (1.0 + decoder.inference_cfg_rate) * conditional - decoder.inference_cfg_rate * unconditional
|
||||
x = x + dt * velocity
|
||||
_zero_padded_frames(x, mel_frames if pad_frames else None)
|
||||
time = time + dt
|
||||
if step + 1 < self.n_timesteps:
|
||||
dt = timeline[step + 2] - time[0]
|
||||
@@ -666,7 +805,15 @@ class BatchedToken2Wav(nn.Module):
|
||||
dtype=self._estimator_att_cache_dtype,
|
||||
)
|
||||
next_att_cache[step].copy_(step_att)
|
||||
if pad_frames:
|
||||
# The padded steps ran but hold no valid content, and each
|
||||
# cache entry becomes the next chunk's keys. Clear the
|
||||
# padded columns so the boundary the next chunk reads is
|
||||
# the valid one, in step with the mask built above.
|
||||
next_att_cache[step][..., mel_frames : mel_frames + pad_frames, :] = 0.0
|
||||
del step_att
|
||||
if pad_frames:
|
||||
x = x[:, :, :mel_frames]
|
||||
if ragged_att_cache is not None:
|
||||
return x, torch.stack(next_cnn), ragged_att_cache
|
||||
assert next_att_cache is not None
|
||||
|
||||
@@ -135,11 +135,13 @@ class HiFTGraphWrapper:
|
||||
return speech, cache_source
|
||||
|
||||
|
||||
def _tensor_signature(value: torch.Tensor) -> tuple:
|
||||
def _tensor_signature(value: torch.Tensor | None) -> tuple:
|
||||
if value is None:
|
||||
return (None,)
|
||||
return tuple(value.shape), str(value.dtype), str(value.device)
|
||||
|
||||
|
||||
_DTYPE_MAP = {str(dtype): dtype for dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64)}
|
||||
_DTYPE_MAP = {str(dtype): dtype for dtype in (torch.float32, torch.float16, torch.bfloat16, torch.float64, torch.bool)}
|
||||
|
||||
|
||||
def _memory_snapshot(device: torch.device) -> tuple[int, int] | None:
|
||||
@@ -231,7 +233,7 @@ class CFMGraphWrapper:
|
||||
return {**self._stats, "cache_size": len(self._cache)}
|
||||
|
||||
def _call_graph_fn(self, args: tuple[torch.Tensor, ...]) -> torch.Tensor:
|
||||
return self.graph_fn(args[0], args[1], None, args[2], args[3], args[4], args[5])
|
||||
return self.graph_fn(args[0], args[1], args[6], args[2], args[3], args[4], args[5])
|
||||
|
||||
def _eager(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
self._stats["eager"] += 1
|
||||
@@ -261,10 +263,21 @@ class CFMGraphWrapper:
|
||||
self.enabled = False
|
||||
self._flush()
|
||||
|
||||
def _capture(self, key: tuple) -> tuple | None:
|
||||
"""Capture a CUDA graph for the given key. Returns None on failure."""
|
||||
def _capture(self, key: tuple, inputs: tuple | None = None) -> tuple | None:
|
||||
"""Capture a CUDA graph for the given key. Returns None on failure.
|
||||
|
||||
``inputs`` should be the real tensors for this key whenever they are
|
||||
available: the static buffers must be built from the values the graph
|
||||
will actually run with. Zero-filled placeholders are only a fallback --
|
||||
capturing with them bakes the placeholder branch into the graph (e.g. an
|
||||
attention mask that looks like "nothing is masked").
|
||||
"""
|
||||
try:
|
||||
static_inputs = _tensors_from_key(key)
|
||||
static_inputs = (
|
||||
tuple(None if tensor is None else tensor.detach().clone() for tensor in inputs)
|
||||
if inputs is not None
|
||||
else _tensors_from_key(key)
|
||||
)
|
||||
except KeyError:
|
||||
# An unsupported dtype is a property of this shape alone, so run it
|
||||
# eager and keep the other shapes on graphs.
|
||||
@@ -315,8 +328,9 @@ class CFMGraphWrapper:
|
||||
att_cache: torch.Tensor,
|
||||
cnn_out: torch.Tensor,
|
||||
att_out: torch.Tensor,
|
||||
attn_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
inputs = (estimator_input, time_emb, cnn_cache, att_cache, cnn_out, att_out)
|
||||
inputs = (estimator_input, time_emb, cnn_cache, att_cache, cnn_out, att_out, attn_mask)
|
||||
self._stats["calls"] += 1
|
||||
|
||||
if not self.enabled or torch.cuda.is_current_stream_capturing() or estimator_input.device.type != "cuda":
|
||||
@@ -330,7 +344,7 @@ class CFMGraphWrapper:
|
||||
if entry is None:
|
||||
if len(self._cache) >= self.max_graphs:
|
||||
self._flush()
|
||||
entry = self._capture(key)
|
||||
entry = self._capture(key, inputs)
|
||||
if entry is None:
|
||||
return self._eager(inputs)
|
||||
self._cache[key] = entry
|
||||
@@ -339,7 +353,8 @@ class CFMGraphWrapper:
|
||||
|
||||
static_inputs, static_output, graph = entry
|
||||
for static, current in zip(static_inputs, inputs, strict=True):
|
||||
static.copy_(current)
|
||||
if static is not None:
|
||||
static.copy_(current)
|
||||
graph.replay()
|
||||
return (
|
||||
static_output.detach().clone(),
|
||||
|
||||
@@ -9,6 +9,7 @@ import os
|
||||
import tempfile
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -57,6 +58,92 @@ def _scalar(value: Any, default: Any = None) -> Any:
|
||||
return default if value is None else value
|
||||
|
||||
|
||||
_REF_TARGET_SAMPLE_RATE = 24000
|
||||
# Overridable per deployment via ``ref_audio_max_seconds``.
|
||||
_REF_MAX_SECONDS = 6.0
|
||||
# Channel ceiling for the optional (channels, samples) downmix guard.
|
||||
_REF_MAX_CHANNELS = 8
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _get_resampler(orig_freq: int, new_freq: int):
|
||||
"""Cached anti-aliased resampler matching token2wav's own loader.
|
||||
|
||||
``torchaudio`` is imported lazily (it is not a hard runtime requirement of
|
||||
this model on every platform) and the transform is built once per rate pair
|
||||
instead of once per request, keeping it off the first-packet path.
|
||||
"""
|
||||
import torchaudio
|
||||
|
||||
return torchaudio.transforms.Resample(orig_freq=orig_freq, new_freq=new_freq)
|
||||
|
||||
|
||||
def _read_reference_wav(path: str) -> tuple[Any, int]:
|
||||
"""Read a WAV file for :func:`_normalize_reference`.
|
||||
|
||||
``soundfile`` returns ``(samples, channels)`` for multi-channel files
|
||||
while the normalizer expects ``(channels, samples)``. Without the
|
||||
transpose a stereo prompt looks like a huge channel count, the downmix
|
||||
guard rejects it, and the caller silently falls back to the raw file --
|
||||
a second L0 value for the same content.
|
||||
"""
|
||||
waveform, sample_rate_hz = sf.read(path, dtype="float32", always_2d=False)
|
||||
if getattr(waveform, "ndim", 1) > 1:
|
||||
waveform = waveform.T
|
||||
return waveform, int(sample_rate_hz)
|
||||
|
||||
|
||||
def _normalize_reference(
|
||||
ref_audio: Any,
|
||||
sample_rate_hz: int,
|
||||
*,
|
||||
max_seconds: float = _REF_MAX_SECONDS,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Normalize a per-request reference waveform onto a shared grid.
|
||||
|
||||
MiniCPM-o streaming uses the reference-audio length as the CFM attention
|
||||
cache origin (L0). Unnormalized references give every request a distinct
|
||||
L0, which explodes the CFM CUDA-graph cache key space (see #6628). Fold
|
||||
every reference onto the same sample rate and a fixed length (truncate
|
||||
long ones, zero-pad short ones) so L0 becomes one constant value.
|
||||
|
||||
References longer than ``max_seconds`` are truncated with a warning;
|
||||
the window is configurable via ``ref_audio_max_seconds``.
|
||||
"""
|
||||
tensor = torch.as_tensor(ref_audio, dtype=torch.float32)
|
||||
if tensor.dim() > 1:
|
||||
# (channels, samples) -> mono; plain reshape(-1) would interleave.
|
||||
# Upstream flattens request references to 1-D, so anything else is a
|
||||
# non-standard caller: reject layouts we cannot downmix unambiguously
|
||||
# instead of silently averaging the waveform away.
|
||||
if tensor.shape[0] > _REF_MAX_CHANNELS:
|
||||
raise ValueError(f"reference audio must be 1-D or (channels, samples); got shape {tuple(tensor.shape)}")
|
||||
tensor = tensor.mean(dim=0)
|
||||
waveform = tensor.reshape(-1).cpu().contiguous()
|
||||
if waveform.numel() == 0:
|
||||
# Keep the caller's "empty_ref_audio" error path intact: an empty
|
||||
# waveform must not be zero-padded into a valid-length reference.
|
||||
return waveform, _REF_TARGET_SAMPLE_RATE
|
||||
if sample_rate_hz != _REF_TARGET_SAMPLE_RATE:
|
||||
# Match token2wav's own loader (torchaudio, anti-aliased): it only
|
||||
# resamples when the stored rate differs, so this output is what the
|
||||
# model consumes.
|
||||
waveform = _get_resampler(sample_rate_hz, _REF_TARGET_SAMPLE_RATE)(waveform.view(1, -1)).view(-1).contiguous()
|
||||
max_samples = max(1, int(max_seconds * _REF_TARGET_SAMPLE_RATE))
|
||||
if waveform.numel() > max_samples:
|
||||
logger.warning(
|
||||
"Reference audio is %.2fs, truncating to the %.2fs window (set ``ref_audio_max_seconds`` to keep more).",
|
||||
waveform.numel() / _REF_TARGET_SAMPLE_RATE,
|
||||
max_seconds,
|
||||
)
|
||||
waveform = waveform[:max_samples].contiguous()
|
||||
elif waveform.numel() < max_samples:
|
||||
# Zero-pad short references so every request shares one L0; trailing
|
||||
# silence has minimal style impact (verified via E3 WER/SIM).
|
||||
waveform = torch.nn.functional.pad(waveform, (0, max_samples - waveform.numel()))
|
||||
return waveform, _REF_TARGET_SAMPLE_RATE
|
||||
|
||||
|
||||
def _codec_tensor(value: Any, fallback: torch.Tensor) -> torch.Tensor:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.reshape(-1).to(device=fallback.device, dtype=torch.long)
|
||||
@@ -181,7 +268,11 @@ class MiniCPMO45Code2Wav(nn.Module):
|
||||
self._cfm_graph_config = {
|
||||
"enabled": bool(extra.get("enable_cfm_graph", False)),
|
||||
"max_graphs": int(extra.get("cfm_max_graphs", 32)),
|
||||
"bucket_frames": int(extra.get("cfm_graph_bucket_frames", 0)),
|
||||
}
|
||||
self._ref_max_seconds = float(extra.get("ref_audio_max_seconds", _REF_MAX_SECONDS))
|
||||
if self._ref_max_seconds <= 0:
|
||||
raise ValueError("MiniCPM-o Code2Wav ref_audio_max_seconds must be > 0")
|
||||
self._min_batch_size = int(extra.get("code2wav_min_batch_size", 1))
|
||||
if self._min_batch_size < 1:
|
||||
raise ValueError("MiniCPM-o Code2Wav code2wav_min_batch_size must be >= 1")
|
||||
@@ -192,6 +283,7 @@ class MiniCPMO45Code2Wav(nn.Module):
|
||||
raise ValueError("MiniCPM-o Code2Wav code2wav_initial_batch_size must be 0 or >= code2wav_min_batch_size")
|
||||
self._default_prompt_id = str(extra.get("prompt_cache_id", "HT_ref_audio"))
|
||||
self._prompt_wav_override = extra.get("prompt_wav")
|
||||
self._default_prompt_normalized: tuple[str, str] | None = None
|
||||
|
||||
@property
|
||||
def _default_prompt_wav(self) -> str:
|
||||
@@ -199,6 +291,46 @@ class MiniCPMO45Code2Wav(nn.Module):
|
||||
return str(self._prompt_wav_override)
|
||||
return str(Path(self.model_path) / "assets" / "HT_ref_audio.wav")
|
||||
|
||||
def _normalized_default_prompt(self) -> tuple[str, str]:
|
||||
"""Fold the shipped default prompt onto the request-reference grid.
|
||||
|
||||
The shipped asset is 6.016 s and is loaded directly by token2wav, so a
|
||||
request without a reference would otherwise keep a second L0 value a
|
||||
couple of frames away from the normalized request references -- two
|
||||
graph-key families instead of one (both use ``ref_audio_max_seconds``).
|
||||
Returns ``(prompt_wav,
|
||||
prompt_cache_id)``; if the asset cannot be read, the shipped path is
|
||||
returned unchanged.
|
||||
"""
|
||||
if self._default_prompt_normalized is not None:
|
||||
return self._default_prompt_normalized
|
||||
source = self._default_prompt_wav
|
||||
fallback = (source, self._default_prompt_id)
|
||||
try:
|
||||
waveform, sample_rate_hz = _read_reference_wav(source)
|
||||
normalized, target_sr = _normalize_reference(
|
||||
waveform,
|
||||
sample_rate_hz,
|
||||
max_seconds=self._ref_max_seconds,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Could not normalize the default prompt %s; using it as-is", source)
|
||||
self._default_prompt_normalized = fallback
|
||||
return self._default_prompt_normalized
|
||||
if normalized.numel() == 0:
|
||||
self._default_prompt_normalized = fallback
|
||||
return self._default_prompt_normalized
|
||||
digest = sha256()
|
||||
digest.update(normalized.numpy().tobytes())
|
||||
digest.update(str(target_sr).encode())
|
||||
cache_id = f"default-ref-{digest.hexdigest()[:24]}-{target_sr}"
|
||||
path = Path(self._runtime_prompt_dir.name) / f"{cache_id}.wav"
|
||||
if not path.is_file():
|
||||
sf.write(path, normalized.numpy(), target_sr, format="WAV")
|
||||
logger.info("Default prompt normalized onto the %.2fs window: %s", self._ref_max_seconds, path)
|
||||
self._default_prompt_normalized = (str(path), cache_id)
|
||||
return self._default_prompt_normalized
|
||||
|
||||
def _extra_config(self) -> dict[str, Any]:
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
connector = getattr(model_config, "stage_connector_config", None)
|
||||
@@ -220,9 +352,13 @@ class MiniCPMO45Code2Wav(nn.Module):
|
||||
sample_rate: Any,
|
||||
) -> tuple[str, _RuntimePrompt]:
|
||||
sample_rate_hz = int(_scalar(sample_rate, 0))
|
||||
waveform = torch.as_tensor(ref_audio, dtype=torch.float32).reshape(-1).cpu().contiguous()
|
||||
if sample_rate_hz <= 0:
|
||||
raise _batch_error("invalid_ref_audio_sample_rate", sample_rate=sample_rate_hz)
|
||||
waveform, sample_rate_hz = _normalize_reference(
|
||||
ref_audio,
|
||||
sample_rate_hz,
|
||||
max_seconds=self._ref_max_seconds,
|
||||
)
|
||||
if waveform.numel() == 0:
|
||||
raise _batch_error("empty_ref_audio")
|
||||
if not bool(torch.isfinite(waveform).all().item()):
|
||||
@@ -288,9 +424,10 @@ class MiniCPMO45Code2Wav(nn.Module):
|
||||
if entry is not None:
|
||||
return entry.cache_id, entry.path, cache_key
|
||||
|
||||
default_wav, default_cache_id = self._normalized_default_prompt()
|
||||
return (
|
||||
str(_scalar(meta.get("prompt_cache_id"), self._default_prompt_id)),
|
||||
str(_scalar(meta.get("prompt_wav"), self._default_prompt_wav)),
|
||||
str(_scalar(meta.get("prompt_cache_id"), default_cache_id)),
|
||||
str(_scalar(meta.get("prompt_wav"), default_wav)),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user