[Bugfix][MiniCPM-o] Stop Talker mid-utterance truncation and load cod… (#6346)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
amy-why-3459andCursor authored and GitHub committed 2026-08-20 11:50:07 +08:00
1 parent 47b1893676
commit 9c61d5b67b
27 files changed
+1118 -678

No files matched your search

@@ -19,6 +19,65 @@ from .cuda_graph_wrapper import CFMGraphWrapper, HiFTGraphWrapper
logger = init_logger(__name__)
_SILENCE_TOKEN = 4218
# CosyVoice2 RelPos PE is built for max_len=5000 and `forward_chunk` never
# calls ``extend_pe``. After the 2x upsample a single Code2Wav prefill longer
# than this many codec tokens overflows the PE slice (matrix_ac vs matrix_bd).
_DEFAULT_RELPOS_MAX_POS = 5000
_MAX_ENCODE_TOKEN_CAP = 1024
def relpos_encode_token_budget(
*,
max_pos: int,
stride: int,
cache_offset: int,
lookahead: int,
cap: int = _MAX_ENCODE_TOKEN_CAP,
) -> int:
"""How many codec tokens ``forward_chunk`` can take before RelPos PE wraps.
``position_encoding(size)`` needs ``size <= max_pos``. After upsample,
``size = cache_offset * stride + ~stride * token_frames`` (plus last-chunk
lookahead pad).
"""
stride = max(1, int(stride))
lookahead = max(0, int(lookahead))
room = int(max_pos) // stride - max(0, int(cache_offset)) - lookahead - 1
return max(lookahead + 1, min(int(cap), room))
def plan_token2wav_encode_slices(
num_frames: int,
*,
max_frames: int,
min_nonfinal: int,
last_chunk: bool,
) -> list[tuple[int, int]]:
"""Split a Code2Wav token span so every non-final piece has a lookahead window."""
if num_frames <= 0:
return []
min_nonfinal = max(1, int(min_nonfinal))
max_frames = max(int(max_frames), min_nonfinal)
last_min = 1 if last_chunk else min_nonfinal
if num_frames <= max_frames:
return [(0, num_frames)]
slices: list[tuple[int, int]] = []
start = 0
while start < num_frames:
remaining = num_frames - start
if remaining <= max_frames:
slices.append((start, num_frames))
break
take = max_frames
tail = remaining - take
if tail <= max_frames and tail < last_min:
take = remaining - last_min
if take < min_nonfinal:
take = remaining
slices.append((start, start + take))
start += take
return slices
def _autocast_disabled(device: torch.device):
@@ -237,6 +296,41 @@ class BatchedToken2Wav(nn.Module):
width = getattr(layer, "pre_lookahead_len", None)
return int(width) if width is not None else None
def _relpos_max_pos(self) -> int:
embed = getattr(self.flow.encoder, "embed", None)
pe = getattr(embed, "pe", None)
if isinstance(pe, torch.Tensor) and pe.numel() > 0:
return max(1, int(pe.size(1) // 2))
return _DEFAULT_RELPOS_MAX_POS
def _upsample_stride(self) -> int:
stride = getattr(getattr(self.flow.encoder, "up_layer", None), "stride", None)
return max(1, int(stride)) if stride is not None else 2
def _max_encode_token_frames(self, states: list[BatchedToken2WavState]) -> int:
att = None
if states:
att = self._stack_flow_cache(states).get("conformer_att_cache")
offset1 = int(att.shape[3] // 2) if att is not None else 0
lookahead = self._pre_lookahead_len() or 0
return relpos_encode_token_budget(
max_pos=self._relpos_max_pos(),
stride=self._upsample_stride(),
cache_offset=offset1,
lookahead=lookahead,
)
def _ensure_relpos_pe(self, tokens: torch.Tensor, att_cache: torch.Tensor | None) -> None:
"""Grow CosyVoice RelPos PE before ``forward_chunk``, which never calls extend_pe."""
embed = getattr(self.flow.encoder, "embed", None)
extend_pe = getattr(embed, "extend_pe", None)
if not callable(extend_pe):
return
offset1 = int(att_cache.shape[3] // 2) if att_cache is not None else 0
lookahead = self._pre_lookahead_len() or 0
needed = offset1 * self._upsample_stride() + self._upsample_stride() * (int(tokens.shape[1]) + lookahead + 1)
extend_pe(tokens.new_zeros((1, max(needed, 1))))
def _encode_chunk(
self,
tokens: torch.Tensor,
@@ -245,6 +339,7 @@ class BatchedToken2Wav(nn.Module):
cnn_cache: torch.Tensor | None,
att_cache: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
self._ensure_relpos_pe(tokens, att_cache)
embedded = self.flow.input_embedding(tokens)
hidden, new_cnn, new_att = self.flow.encoder.forward_chunk(
xs=embedded,
@@ -508,14 +603,68 @@ class BatchedToken2Wav(nn.Module):
# must carry at least one full kernel. Only the final chunk is allowed
# to be shorter: ``forward_chunk`` zero-pads it by the lookahead width.
lookahead = self._pre_lookahead_len()
num_frames = int(tokens.shape[1])
if lookahead is not None and not last_chunk:
num_frames = int(tokens.shape[1])
if num_frames <= lookahead:
raise RuntimeError(
"MiniCPMO45Code2WavBatchError "
f'{{"reason":"chunk_below_lookahead_window","frames":{num_frames},'
f'"minimum":{lookahead + 1}}}'
)
# A non-async Talker dump can land thousands of codec tokens in one
# Code2Wav prefill. CosyVoice RelPos PE (max_len=5000) plus 2x upsample
# cannot score that in one ``forward_chunk`` (6968 vs 985 on NPU).
max_frames = self._max_encode_token_frames(states)
slices = plan_token2wav_encode_slices(
num_frames,
max_frames=max_frames,
min_nonfinal=(lookahead + 1) if lookahead is not None else 1,
last_chunk=last_chunk,
)
if len(slices) > 1:
logger.info(
"MiniCPM-o Code2Wav splitting %d codec tokens into %d encoder windows "
"(max_frames=%d) to stay inside RelPos PE.",
num_frames,
len(slices),
max_frames,
)
parts: list[list[torch.Tensor]] = [[] for _ in range(batch_size)]
current = states
for index, (start, end) in enumerate(slices):
is_last_piece = index == len(slices) - 1
audios, current = self._decode_batch_once(
tokens[:, start:end],
features,
current,
last_chunk=last_chunk and is_last_piece,
flush_encoder=flush_encoder and is_last_piece,
)
for row, audio in enumerate(audios):
parts[row].append(audio)
merged = [
torch.cat(row_parts) if row_parts else tokens.new_zeros((0,), dtype=torch.float32)
for row_parts in parts
]
return merged, current
return self._decode_batch_once(
tokens,
features,
states,
last_chunk=last_chunk,
flush_encoder=flush_encoder,
)
def _decode_batch_once(
self,
tokens: torch.Tensor,
features: PromptFeatures,
states: list[BatchedToken2WavState],
*,
last_chunk: bool,
flush_encoder: bool = False,
) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]:
batch_size = int(tokens.shape[0])
flow_cache = self._stack_flow_cache(states)
speakers = features.speaker_embedding.expand(batch_size, -1)
with self._autocast(tokens.device):
@@ -301,6 +301,10 @@ class ConditionalChatTTSConfig(PretrainedConfig):
self.top_p = top_p
self.top_k = top_k
self.repetition_penalty = repetition_penalty
# Talker stage is a single-vocab codec LM: vLLM's Sampler and
# embed_tokens both address this table, not the Thinker text vocab.
self.vocab_size = int(num_audio_tokens)
self.eos_token_id = int(num_audio_tokens) - 1
class MiniCPMOConfig(Qwen2Config):
@@ -8,11 +8,12 @@ Pipeline:
1. Receive thinker hidden_states + full token IDs via additional_information
2. Extract tts_bos..tts_eos region
3. Build condition: emb_text(tokens) + projector_semantic(hidden) (hidden_text_merge)
4. Continuously generate request-aligned discrete audio-code deltas
4. Project last hidden through head_code; vLLM Sampler picks the codec id
5. Next decode embeds that id with emb_code and emits it to Code2Wav
"""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from typing import Any
import torch
@@ -32,45 +33,34 @@ from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
_REPETITION_WINDOW = 16
_REPETITION_PENALTY_CHUNK_SIZE = 16
_MIN_AUDIO_TOKENS = 64
_MAX_AUDIO_TOKENS = 2048
_AUDIO_TOKENS_PER_TEXT_TOKEN = 10
# Codec-token sampling happens inside the model; vLLM sampling parameters
# only choose the Talker's binary continue/stop row.
_CODEC_SEED = 42
_CODEC_TEMPERATURE = 0.8
_CODEC_TOP_K = 25
_CODEC_TOP_P = 0.85
_CODEC_REPETITION_PENALTY = 1.05
_CODEC_MIN_TOKENS = 50
# Native duplex Talker must finish after one MiniCPMTTS.generate_chunk:
# 25 codec frames (``codec_chunk_frames``) plus the terminating sample.
# Without this, the single-vocab Sampler keeps the stage-1 request alive
# until codec EOS / 4096 and Thinker never starts the next model turn.
_DUPLEX_CODEC_TOKENS_PER_CHUNK = 26
@dataclass(slots=True)
class _PendingCodecSample:
output_index: int
hidden_row: torch.Tensor
codes: torch.Tensor
request_id: str
step: int
state: dict[str, Any]
info: dict[str, Any]
def _native_duplex_chunk_budget(meta: Mapping[str, Any] | None) -> tuple[int, int]:
"""Return ``(max_tokens, min_tokens)`` for one native-duplex Talker request."""
boundary = isinstance(meta, Mapping) and (bool(meta.get("turn_start")) or bool(meta.get("turn_end")))
ceiling = _DUPLEX_CODEC_TOKENS_PER_CHUNK
return ceiling, 0 if boundary else ceiling
def _max_audio_tokens(condition_tokens: int) -> int:
"""Bound codec generation with a conservative text-length estimate.
def blank_scheduler_prompt_for_penalties(
prompt_token_ids: torch.Tensor,
vocab_size: int,
) -> torch.Tensor:
"""Return a penalty prompt whose every position is the pad id (``vocab_size``).
EOS is masked for the first 50 steps, so a direct ``text_tokens * 10``
limit can terminate short responses before EOS is eligible. The 2048
ceiling matches the checkpoint's native generation default and keeps the
sequence within the Talker's 4096-position context.
No Talker prompt position is codec history: prefill conditioning arrives as
embeddings from ``preprocess`` and decode embeds sampled ids with
``emb_code``. The scheduler ids are placeholders (``llm2tts`` fills them
with ``0``, or with thinker token ids on the non-handoff path), so scoring
them would tax unrelated codec tokens.
"""
return max(
_MIN_AUDIO_TOKENS,
min(_MAX_AUDIO_TOKENS, condition_tokens * _AUDIO_TOKENS_PER_TEXT_TOKEN),
)
return torch.full_like(prompt_token_ids, int(vocab_size))
def _restore_weight_norm_weight(weight_g: torch.Tensor, weight_v: torch.Tensor) -> torch.Tensor:
@@ -122,32 +112,6 @@ def _apply_batched_repetition_penalty(
return penalized
def _apply_top_k_top_p(
logits: torch.Tensor,
*,
top_k: int | None,
top_p: float | None,
min_tokens_to_keep: int = 3,
) -> torch.Tensor:
"""Apply the same candidate floors as the upstream Transformers warpers."""
filtered = logits.clone()
vocab_size = filtered.shape[-1]
# MiniCPM-o's gen_logits() appends TopPLogitsWarper before
# TopKLogitsWarper. The order is observable for fixed-seed sampling.
if top_p is not None and 0.0 < top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(filtered, descending=False, dim=-1)
cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
remove = cumulative_probs <= (1.0 - float(top_p))
remove[..., -min_tokens_to_keep:] = False
remove = remove.scatter(-1, sorted_indices, remove)
filtered.masked_fill_(remove, float("-inf"))
if top_k is not None and top_k > 0:
keep = min(vocab_size, max(int(top_k), min_tokens_to_keep))
threshold = torch.topk(filtered, keep, dim=-1).values[..., -1, None]
filtered.masked_fill_(filtered < threshold, float("-inf"))
return filtered
class _MiniCPMTTSProjector(nn.Module):
"""Checkpoint-compatible hidden-state projector used by MiniCPMTTS."""
@@ -173,8 +137,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
config: MiniCPMOConfig = vllm_config.model_config.hf_config
self.config = config
self.vllm_config = vllm_config
self._batch_stop_logits: torch.Tensor | None = None
self._request_generators: dict[str, torch.Generator] = {}
self._force_eos_rows: list[bool] | None = None
self._mask_eos_rows: list[bool] | None = None
self._request_audio_states: dict[str, dict[str, Any]] = {}
self._deferred_cleanup_ids: set[str] = set()
@@ -186,23 +150,18 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687)
self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692)
self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562)
self._codec_eos_id = int(getattr(tts_config, "eos_token_id", self._num_audio_tokens - 1))
self._hidden_size = getattr(tts_config, "hidden_size", 768)
self._normalize = getattr(tts_config, "normalize_projected_hidden", True)
self._codec_seed = int(getattr(tts_config, "seed", _CODEC_SEED))
self._codec_temperature = float(getattr(tts_config, "temperature", _CODEC_TEMPERATURE))
self._codec_top_k = int(getattr(tts_config, "top_k", _CODEC_TOP_K))
self._codec_top_p = float(getattr(tts_config, "top_p", _CODEC_TOP_P))
self._codec_repetition_penalty = float(getattr(tts_config, "repetition_penalty", _CODEC_REPETITION_PENALTY))
self._codec_min_tokens = int(getattr(tts_config, "min_new_tokens", _CODEC_MIN_TOKENS))
else:
self._tts_config = None
self._codec_eos_id = 0
self.has_preprocess = True
self.has_postprocess = False
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {
("audio_codes", "current"),
("audio_codes", "accumulated"),
}
# Same-step codes travel through make_omni_output from the previous
# sampled id (decode preprocess embeds that id via emb_code).
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {("codes", "audio")}
self._init_native_talker(prefix)
def _init_native_talker(self, prefix: str) -> None:
@@ -326,6 +285,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
info_dict.get("request_id"),
)
native_duplex = bool(info_dict.get("native_duplex", False))
meta = info_dict.get("meta")
full_embeds = self._build_condition_embeddings(
token_ids,
hidden_states,
@@ -333,7 +293,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
)
offset = int(info_dict.get("_omni_num_computed_tokens", 0))
request_id = str(info_dict.get("request_id", "0"))
meta = info_dict.get("meta")
# The handoff rebuilds only the tail-aligned Talker condition.
# Materialize zero-token embeddings for any scheduler prompt
# prefix so chunked prefill can slice from a non-zero offset.
@@ -356,21 +315,11 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
f"tts_ids={token_ids.shape[0]} tts_hidden={hidden_states.shape[0]} "
f"prompt_len={info_dict.get('_omni_prompt_len')}"
)
duplex_boundary = isinstance(meta, dict) and (
bool(meta.get("turn_start", False)) or bool(meta.get("turn_end", False))
)
state: dict[str, Any] = {"finished": empty_condition, "step": 0}
if native_duplex:
max_tokens = _DUPLEX_CODEC_TOKENS_PER_CHUNK
min_tokens = 0 if duplex_boundary else _DUPLEX_CODEC_TOKENS_PER_CHUNK
else:
max_tokens = _max_audio_tokens(int(token_ids.numel()))
min_tokens = self._codec_min_tokens
state = {
"step": 0,
"max_tokens": max_tokens,
"min_tokens": min_tokens,
"finished": empty_condition,
}
max_tokens, min_tokens = _native_duplex_chunk_budget(meta if isinstance(meta, Mapping) else None)
state["max_tokens"] = max_tokens
state["min_tokens"] = min_tokens
request_states = getattr(self, "_request_audio_states", None)
if request_states is None:
request_states = {}
@@ -382,98 +331,38 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
embeds,
{
"audio_state": state,
"audio_codes": {
"current": empty_codes,
"accumulated": empty_codes,
},
# Prefill has no previous codec id. vLLM samples the first
# code after this forward; the next decode emits it.
"codes": {"audio": empty_codes},
},
)
current = (info_dict.get("audio_codes", {}) or {}).get("current")
if not isinstance(current, torch.Tensor) or current.numel() != 1:
if state.get("finished"):
# A request that finished before sampling any code can still be
# scheduled for decode steps while sampling min_tokens masks the
# stop token. make_omni_output ignores its hidden states, so any
# shape-correct embedding will do.
weight = self.emb_code[0].weight
return input_ids, weight.new_zeros((span_len, weight.shape[1])), {}
raise RuntimeError("MiniCPM-o Talker decode is missing the previous request-local audio code")
code = current.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(1)
request_id = str(info_dict.get("request_id", "0"))
stored = self._request_audio_states.get(request_id)
if isinstance(stored, dict):
state = stored
if isinstance(state, dict) and state.get("finished"):
# An empty speech segment can still be scheduled until EOS is
# eligible. The sampler is forced to EOS; any shape-correct
# embedding is enough for these leftover decode rows.
weight = self.emb_code[0].weight
empty_codes = torch.empty(0, dtype=torch.long, device=weight.device)
return input_ids, weight.new_zeros((span_len, weight.shape[1])), {"codes": {"audio": empty_codes}}
# Decode: vLLM's previous sampled codec id is this step's input.
# Embed it with the codec table (not the 32k Llama embed_tokens) and
# hand the same id to make_omni_output so Code2Wav sees it this step.
code = input_ids.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(-1)[-1:]
embeds = self.emb_code[0](code)
return input_ids, embeds, {}
def _request_generator(self, request_id: str, device: torch.device) -> torch.Generator:
generator = self._request_generators.get(request_id)
if generator is None:
generator = torch.Generator(device=device)
generator.manual_seed(self._codec_seed)
self._request_generators[request_id] = generator
return generator
def _sample_audio_codes(
self,
hidden_states: torch.Tensor,
histories: Sequence[torch.Tensor],
request_ids: Sequence[str],
steps: Sequence[int],
) -> torch.Tensor:
batch_size = int(hidden_states.shape[0])
if not (len(histories) == len(request_ids) == len(steps) == batch_size):
raise ValueError(
"MiniCPM-o batched codec sampling requires one history, request id, "
f"and step per hidden row, got batch={batch_size}, histories={len(histories)}, "
f"request_ids={len(request_ids)}, steps={len(steps)}"
)
if batch_size == 0:
return torch.empty(0, dtype=torch.long, device=hidden_states.device)
logits = self.head_code[0](hidden_states).float() / self._codec_temperature
eos_id = self._num_audio_tokens - 1
logits = _apply_batched_repetition_penalty(
logits,
histories,
penalty=self._codec_repetition_penalty,
window_size=_REPETITION_WINDOW,
)
request_states = getattr(self, "_request_audio_states", {})
mask_eos_values: list[bool] = []
for request_id, step in zip(request_ids, steps, strict=True):
state = request_states.get(request_id)
min_tokens = (
int(state.get("min_tokens", self._codec_min_tokens))
if isinstance(state, dict)
else self._codec_min_tokens
)
mask_eos_values.append(step < min_tokens)
mask_eos = torch.tensor(
mask_eos_values,
dtype=torch.bool,
device=logits.device,
)
logits[:, eos_id].masked_fill_(mask_eos, float("-inf"))
logits = _apply_top_k_top_p(
logits,
top_k=self._codec_top_k,
top_p=self._codec_top_p,
min_tokens_to_keep=3,
)
probabilities = torch.softmax(logits, dim=-1)
# torch.multinomial accepts one generator for the entire call. Keep it
# per row so compaction, request ordering, or another request finishing
# cannot perturb request-local RNG streams; the expensive projection
# and logit transforms above remain fully batched.
return torch.cat(
[
torch.multinomial(
probabilities[row : row + 1],
num_samples=1,
generator=self._request_generator(request_id, probabilities.device),
)
for row, request_id in enumerate(request_ids)
],
dim=0,
).reshape(-1)
if int(code.item()) == int(self._codec_eos_id):
if isinstance(state, dict):
state["finished"] = True
elif stored is None:
self._request_audio_states[request_id] = {"finished": True, "step": 0}
delta = torch.empty(0, dtype=torch.long, device=code.device)
else:
delta = code.reshape(1, 1)
return input_ids, embeds, {"codes": {"audio": delta}}
def make_omni_output(
self,
@@ -487,18 +376,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
spans = kwargs.get("request_token_spans")
if spans is None or len(spans) != len(infos):
raise RuntimeError("MiniCPM-o continuous Talker requires one request_token_span per request")
sample_eligible = kwargs.get("request_sample_eligible")
if sample_eligible is None:
sample_eligible = [True] * len(infos)
if len(sample_eligible) != len(infos):
raise RuntimeError(
f"MiniCPM-o continuous Talker received {len(sample_eligible)} sampling flags for {len(infos)} requests"
)
emit_duplex_metadata = any(isinstance(info, dict) and info.get("native_duplex") is True for info in infos)
# Rows default to continue. Only previously finished or newly terminal
# requests stop; prefill/ineligible rows stay aligned as False.
stop_flags = [False] * len(infos)
native_duplex_flags: list[torch.Tensor] = []
duplex_epochs: list[torch.Tensor] = []
duplex_turn_ids: list[torch.Tensor] = []
@@ -507,7 +386,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
empty_delta = hidden.new_empty((0, 1), dtype=torch.long)
codec_deltas = [empty_delta for _ in infos]
terminal_flags = [torch.tensor(False, dtype=torch.bool) for _ in infos]
pending_samples: list[_PendingCodecSample] = []
force_eos_rows = [False] * len(infos)
mask_eos_rows = [False] * len(infos)
for index, info in enumerate(infos):
info_dict = info if isinstance(info, dict) else {}
native_duplex = info_dict.get("native_duplex") is True
@@ -554,100 +434,39 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
if not isinstance(info, dict):
continue
start, end = spans[index]
end = min(int(end), int(hidden.shape[0]))
if int(start) >= end:
continue
request_id = str(info.get("request_id", index))
request_states = getattr(self, "_request_audio_states", None)
if request_states is None:
request_states = {}
self._request_audio_states = request_states
state = request_states.get(request_id)
state = self._request_audio_states.get(request_id)
if not isinstance(state, dict):
state = dict(info.get("audio_state", {}) or {})
request_states[request_id] = state
if state.get("finished"):
stop_flags[index] = True
continue
if not sample_eligible[index]:
# vLLM computes a logit row for incomplete chunked prefills but
# discards its sampled token. Advancing codec/RNG state here
# would make output depend on prefill chunking and compaction.
continue
codes = state.get("codes")
if not isinstance(codes, torch.Tensor):
codes = (info.get("audio_codes", {}) or {}).get("accumulated")
if not isinstance(codes, torch.Tensor):
codes = torch.empty(0, dtype=torch.long, device=hidden.device)
else:
codes = codes.to(device=hidden.device, dtype=torch.long).reshape(-1)
self._request_audio_states[request_id] = state
empty_speech = bool(state.get("finished"))
codes = info.get("codes", {})
audio = codes.get("audio") if isinstance(codes, Mapping) else None
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
codec_deltas[index] = audio.to(device=hidden.device, dtype=torch.long).reshape(-1, 1)
if state.get("max_tokens") is not None:
state["step"] = int(state.get("step", 0)) + 1
max_tokens = state.get("max_tokens")
min_tokens = state.get("min_tokens")
step = int(state.get("step", 0))
pending_samples.append(
_PendingCodecSample(
output_index=index,
hidden_row=hidden[end - 1 : end],
codes=codes,
request_id=request_id,
step=step,
state=state,
info=info,
)
)
# 26 samples include the terminating EOS, so force it after 25
# forwarded frames — the same cadence as generate_chunk.
hit_chunk_limit = max_tokens is not None and step >= int(max_tokens) - 1
chunk_done = empty_speech or hit_chunk_limit
if hit_chunk_limit:
# The next sampled id is forced EOS and will finish the
# request; mark the chunk done on this step so the data
# plane does not wait for a follow-up empty decode.
state["finished"] = True
force_eos_rows[index] = chunk_done
mask_eos_rows[index] = not force_eos_rows[index] and min_tokens is not None and step < int(min_tokens)
terminal_flags[index] = torch.tensor(chunk_done, dtype=torch.bool)
if pending_samples:
active_hidden_rows = [pending.hidden_row for pending in pending_samples]
active_hidden = (
active_hidden_rows[0] if len(active_hidden_rows) == 1 else torch.cat(active_hidden_rows, dim=0)
)
sampled_batch = self._sample_audio_codes(
active_hidden,
[pending.codes for pending in pending_samples],
[pending.request_id for pending in pending_samples],
[pending.step for pending in pending_samples],
)
# One batched device-to-host synchronization replaces one .item()
# synchronization per request.
sampled_ids = sampled_batch.detach().to(device="cpu").tolist()
else:
sampled_batch = hidden.new_empty((0,), dtype=torch.long)
sampled_ids = []
for row, pending in enumerate(pending_samples):
sampled = sampled_batch[row].reshape(())
sampled_id = int(sampled_ids[row])
codes = pending.codes
state = pending.state
info = pending.info
is_eos = sampled_id == self._num_audio_tokens - 1
state["step"] = int(state.get("step", 0)) + 1
reached_limit = int(state["step"]) >= int(state.get("max_tokens", 2048))
finished = is_eos or reached_limit
state["finished"] = finished
# MiniCPMTTS.generate_chunk consumes the boundary sample but
# returns only codes that were fed into the retained KV state.
if not is_eos and not reached_limit:
codes = torch.cat([codes[-(_REPETITION_WINDOW - 1) :], sampled.reshape(1)])
delta = sampled.reshape(1, 1)
else:
delta = empty_delta
state["codes"] = codes
info["audio_state"] = state
info["audio_codes"] = {
"current": sampled.reshape(1),
"accumulated": codes,
}
codec_deltas[pending.output_index] = delta
terminal_flags[pending.output_index] = torch.tensor(finished, dtype=torch.bool)
stop_flags[pending.output_index] = finished
self._batch_stop_logits = (
hidden.new_tensor([[float("-inf"), 0.0] if finished else [0.0, float("-inf")] for finished in stop_flags])
if stop_flags
else hidden.new_empty((0, 2))
)
# Lists are deliberate: the runner routes element i to request i,
# preserving compaction alignment while emitting only this step's code.
# Empty-speech and finished duplex chunks must sample codec EOS so
# the scheduler releases the request; mid-chunk rows mask EOS until
# the 26-token generate_chunk budget is met.
self._force_eos_rows = force_eos_rows
self._mask_eos_rows = mask_eos_rows
meta_outputs = {"finished": terminal_flags}
if emit_duplex_metadata:
meta_outputs.update(
@@ -659,13 +478,12 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
"turn_end": turn_end_flags,
}
)
multimodal_outputs: dict[str, Any] = {
"codes": {"audio": codec_deltas},
"meta": meta_outputs,
}
return OmniOutput(
text_hidden_states=hidden,
multimodal_outputs=multimodal_outputs,
multimodal_outputs={
"codes": {"audio": codec_deltas},
"meta": meta_outputs,
},
)
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
@@ -674,7 +492,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
def _flush_deferred_cleanup(self) -> None:
request_audio_states = getattr(self, "_request_audio_states", {})
for request_id in self._deferred_cleanup_ids:
self._request_generators.pop(request_id, None)
request_audio_states.pop(request_id, None)
self._deferred_cleanup_ids.clear()
@@ -723,18 +540,38 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
def compute_logits(self, hidden_states, *args, **kwargs):
if not isinstance(hidden_states, torch.Tensor):
return None
if self._batch_stop_logits is None:
return torch.zeros(
hidden_states.shape[0],
2,
device=hidden_states.device,
dtype=torch.float32,
)
logits = self._batch_stop_logits
self._batch_stop_logits = None
if hidden_states.numel() == 0:
return hidden_states.new_empty((0, int(self._num_audio_tokens)))
logits = self.head_code[0](hidden_states).float()
force_eos = self._force_eos_rows
mask_eos = self._mask_eos_rows
self._force_eos_rows = None
self._mask_eos_rows = None
need_force = bool(force_eos and len(force_eos) == logits.shape[0] and any(force_eos))
need_mask = bool(mask_eos and len(mask_eos) == logits.shape[0] and any(mask_eos))
if need_force or need_mask:
logits = logits.clone()
eos_id = int(self._codec_eos_id)
if need_force:
assert force_eos is not None
forced = torch.tensor(force_eos, dtype=torch.bool, device=logits.device)
logits[forced] = float("-inf")
logits[forced, eos_id] = 0.0
if need_mask:
assert mask_eos is not None
masked = torch.tensor(mask_eos, dtype=torch.bool, device=logits.device)
logits[masked, eos_id] = float("-inf")
return logits
def sample(self, logits, sampling_metadata):
prompt_ids = getattr(sampling_metadata, "prompt_token_ids", None)
if isinstance(logits, torch.Tensor) and isinstance(prompt_ids, torch.Tensor):
# Copy rather than mutate: the runner may hand us the input batch's
# own persistent SamplingMetadata.
sampling_metadata = replace(
sampling_metadata,
prompt_token_ids=blank_scheduler_prompt_for_penalties(prompt_ids, logits.shape[-1]),
)
return Sampler()(logits, sampling_metadata)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
@@ -782,9 +619,10 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
return loaded
def get_input_embeddings(self, input_ids, multimodal_embeddings=None, **kwargs):
if hasattr(self, "emb_text") and self.emb_text is not None:
return self.emb_text(input_ids)
return torch.zeros(input_ids.shape[0], 1)
del multimodal_embeddings
# Decode tokens live in the codec table. Prefill overwrites these
# embeddings in preprocess with emb_text + projected thinker hidden.
return self.emb_code[0](input_ids)
def embed_input_ids(self, input_ids, **kwargs):
return self.get_input_embeddings(input_ids, **kwargs)
@@ -62,7 +62,12 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
custom_process_input_func=f"{_PROC}.llm2tts",
custom_process_next_stage_input_func=f"{_PROC}.tts2code2wav_full_payload",
async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk",
sampling_constraints={"detokenize": False},
sampling_constraints={
"detokenize": False,
# MiniCPM-o 4.5 codec EOS is tts_config.num_audio_tokens - 1.
# Same pattern as Qwen3 talker's stop_token_ids: [2150].
"stop_token_ids": [6561],
},
),
StagePipelineConfig(
stage_id=2,