[Bugfix] Fix MiMo-Audio voice instability: stochastic local_sampler + codec streaming context (#3686)

Signed-off-by: Galleons2029 <Galleons777@gmail.com>
Signed-off-by: Jialong Liu <88185941+Galleons2029@users.noreply.github.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Jialong LiuandYueqian Lin authored and GitHub committed 2026-05-21 10:18:00 -04:00
1 parent bd1617ce2b
commit e949ccf087
5 files changed
+150 -17

No files matched your search

@@ -26,6 +26,13 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = logging.getLogger(__name__)
# Minimum safe values for codec streaming parameters. Mirrors the constants
# in stage_input_processors/mimo_audio.py — keep in sync.
_MIN_CODEC_CHUNK_FRAMES = 3
_MIN_CODEC_LEFT_CONTEXT_FRAMES = 40 # must cover vocoder_attn_window_size[0]
_DEFAULT_CODEC_CHUNK_FRAMES = 10
_DEFAULT_CODEC_LEFT_CONTEXT_FRAMES = 40
def flat_codec_group_element_count(group_size: int, audio_channels: int) -> int:
"""Flat token count for one MiMo talker codec group on the code2wav wire.
@@ -485,12 +492,37 @@ class MiMoAudioToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
if connector_cfg
else None
)
self._codec_chunk_frames = int(extra_cfg.get("codec_chunk_frames", 3)) if isinstance(extra_cfg, dict) else 3
if self._codec_chunk_frames <= 0:
raise ValueError(f"codec_chunk_frames must be positive, got {self._codec_chunk_frames}")
self._codec_left_context_frames = (
int(extra_cfg.get("codec_left_context_frames", 3)) if isinstance(extra_cfg, dict) else 3
raw_chunk = (
int(extra_cfg.get("codec_chunk_frames", _DEFAULT_CODEC_CHUNK_FRAMES))
if isinstance(extra_cfg, dict)
else _DEFAULT_CODEC_CHUNK_FRAMES
)
if raw_chunk < _MIN_CODEC_CHUNK_FRAMES:
logger.warning(
"codec_chunk_frames=%d is below minimum %d; falling back to %d.",
raw_chunk,
_MIN_CODEC_CHUNK_FRAMES,
_DEFAULT_CODEC_CHUNK_FRAMES,
)
raw_chunk = _DEFAULT_CODEC_CHUNK_FRAMES
self._codec_chunk_frames = raw_chunk
raw_left = (
int(extra_cfg.get("codec_left_context_frames", _DEFAULT_CODEC_LEFT_CONTEXT_FRAMES))
if isinstance(extra_cfg, dict)
else _DEFAULT_CODEC_LEFT_CONTEXT_FRAMES
)
if raw_left < _MIN_CODEC_LEFT_CONTEXT_FRAMES:
logger.warning(
"codec_left_context_frames=%d is below minimum %d (must cover vocoder attention "
"window %s); falling back to %d to prevent voice instability.",
raw_left,
_MIN_CODEC_LEFT_CONTEXT_FRAMES,
getattr(self.config, "vocoder_attn_window_size", [40, 10]),
_DEFAULT_CODEC_LEFT_CONTEXT_FRAMES,
)
raw_left = _DEFAULT_CODEC_LEFT_CONTEXT_FRAMES
self._codec_left_context_frames = raw_left
def load_weights(
self,
@@ -542,8 +542,20 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
)
self.device = current_omni_platform.get_torch_device()
# global_sampler MUST stay greedy (do_sample=False) so its token decision
# matches vLLM's external sampler (SamplingParams temperature=0.0). Both
# run argmax on the same logits, so they always agree on whether the next
# token is <|empty|> (audio step) or a real text token. Enabling
# do_sample=True here without also routing vLLM's sampled token back into
# this gate check would cause the two to diverge and corrupt KV-cache state.
self.global_sampler = MiMoSampler(do_sample=False, temperature=0.6, top_p=0.95)
self.local_sampler = MiMoSampler(do_sample=False, temperature=0.9, top_p=0.95)
# local_sampler drives audio-code generation inside local_forward. It is
# entirely internal and NOT subject to vLLM's SamplingParams, so stochastic
# sampling is safe here and is required to produce natural, varied speech.
# Setting do_sample=True also disables the CUDA-graph path (use_cg gate in
# local_forward checks `do_sample is False`), preventing MiMoLocalSamplerTensor
# from silently forcing argmax even when temperature > 0.
self.local_sampler = MiMoSampler(do_sample=True, temperature=0.9, top_p=0.95)
self.removed_tokens = None
self.speech_vocab_sizes = config.parsed_speech_vocab_sizes()
@@ -806,7 +818,7 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
device=tokens_device,
)
if local_sampler is None:
local_sampler = MiMoSampler(do_sample=False, temperature=0.6, top_p=0.9)
local_sampler = MiMoSampler(do_sample=True, temperature=0.9, top_p=0.95)
past_key_values = DynamicCache()
for t in range(delay_iters):
@@ -852,7 +864,7 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
local_sampler: MiMoSampler | None = None,
):
if local_sampler is None:
local_sampler = MiMoSampler(do_sample=False, temperature=0.6, top_p=0.9)
local_sampler = MiMoSampler(do_sample=True, temperature=0.9, top_p=0.95)
b = int(local_embeds.shape[0])
use_cg = (local_sampler.do_sample is None or local_sampler.do_sample is False) and bool(