[Bugfix][MiniCPM-o] Stop Talker mid-utterance truncation and load cod… (#6346)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
47b1893676
commit
9c61d5b67b
27 files changed
+1118
-678
No files matched your search
@@ -19,6 +19,65 @@ from .cuda_graph_wrapper import CFMGraphWrapper, HiFTGraphWrapper
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SILENCE_TOKEN = 4218
|
||||
# CosyVoice2 RelPos PE is built for max_len=5000 and `forward_chunk` never
|
||||
# calls ``extend_pe``. After the 2x upsample a single Code2Wav prefill longer
|
||||
# than this many codec tokens overflows the PE slice (matrix_ac vs matrix_bd).
|
||||
_DEFAULT_RELPOS_MAX_POS = 5000
|
||||
_MAX_ENCODE_TOKEN_CAP = 1024
|
||||
|
||||
|
||||
def relpos_encode_token_budget(
|
||||
*,
|
||||
max_pos: int,
|
||||
stride: int,
|
||||
cache_offset: int,
|
||||
lookahead: int,
|
||||
cap: int = _MAX_ENCODE_TOKEN_CAP,
|
||||
) -> int:
|
||||
"""How many codec tokens ``forward_chunk`` can take before RelPos PE wraps.
|
||||
|
||||
``position_encoding(size)`` needs ``size <= max_pos``. After upsample,
|
||||
``size = cache_offset * stride + ~stride * token_frames`` (plus last-chunk
|
||||
lookahead pad).
|
||||
"""
|
||||
stride = max(1, int(stride))
|
||||
lookahead = max(0, int(lookahead))
|
||||
room = int(max_pos) // stride - max(0, int(cache_offset)) - lookahead - 1
|
||||
return max(lookahead + 1, min(int(cap), room))
|
||||
|
||||
|
||||
def plan_token2wav_encode_slices(
|
||||
num_frames: int,
|
||||
*,
|
||||
max_frames: int,
|
||||
min_nonfinal: int,
|
||||
last_chunk: bool,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Split a Code2Wav token span so every non-final piece has a lookahead window."""
|
||||
if num_frames <= 0:
|
||||
return []
|
||||
min_nonfinal = max(1, int(min_nonfinal))
|
||||
max_frames = max(int(max_frames), min_nonfinal)
|
||||
last_min = 1 if last_chunk else min_nonfinal
|
||||
if num_frames <= max_frames:
|
||||
return [(0, num_frames)]
|
||||
|
||||
slices: list[tuple[int, int]] = []
|
||||
start = 0
|
||||
while start < num_frames:
|
||||
remaining = num_frames - start
|
||||
if remaining <= max_frames:
|
||||
slices.append((start, num_frames))
|
||||
break
|
||||
take = max_frames
|
||||
tail = remaining - take
|
||||
if tail <= max_frames and tail < last_min:
|
||||
take = remaining - last_min
|
||||
if take < min_nonfinal:
|
||||
take = remaining
|
||||
slices.append((start, start + take))
|
||||
start += take
|
||||
return slices
|
||||
|
||||
|
||||
def _autocast_disabled(device: torch.device):
|
||||
@@ -237,6 +296,41 @@ class BatchedToken2Wav(nn.Module):
|
||||
width = getattr(layer, "pre_lookahead_len", None)
|
||||
return int(width) if width is not None else None
|
||||
|
||||
def _relpos_max_pos(self) -> int:
|
||||
embed = getattr(self.flow.encoder, "embed", None)
|
||||
pe = getattr(embed, "pe", None)
|
||||
if isinstance(pe, torch.Tensor) and pe.numel() > 0:
|
||||
return max(1, int(pe.size(1) // 2))
|
||||
return _DEFAULT_RELPOS_MAX_POS
|
||||
|
||||
def _upsample_stride(self) -> int:
|
||||
stride = getattr(getattr(self.flow.encoder, "up_layer", None), "stride", None)
|
||||
return max(1, int(stride)) if stride is not None else 2
|
||||
|
||||
def _max_encode_token_frames(self, states: list[BatchedToken2WavState]) -> int:
|
||||
att = None
|
||||
if states:
|
||||
att = self._stack_flow_cache(states).get("conformer_att_cache")
|
||||
offset1 = int(att.shape[3] // 2) if att is not None else 0
|
||||
lookahead = self._pre_lookahead_len() or 0
|
||||
return relpos_encode_token_budget(
|
||||
max_pos=self._relpos_max_pos(),
|
||||
stride=self._upsample_stride(),
|
||||
cache_offset=offset1,
|
||||
lookahead=lookahead,
|
||||
)
|
||||
|
||||
def _ensure_relpos_pe(self, tokens: torch.Tensor, att_cache: torch.Tensor | None) -> None:
|
||||
"""Grow CosyVoice RelPos PE before ``forward_chunk``, which never calls extend_pe."""
|
||||
embed = getattr(self.flow.encoder, "embed", None)
|
||||
extend_pe = getattr(embed, "extend_pe", None)
|
||||
if not callable(extend_pe):
|
||||
return
|
||||
offset1 = int(att_cache.shape[3] // 2) if att_cache is not None else 0
|
||||
lookahead = self._pre_lookahead_len() or 0
|
||||
needed = offset1 * self._upsample_stride() + self._upsample_stride() * (int(tokens.shape[1]) + lookahead + 1)
|
||||
extend_pe(tokens.new_zeros((1, max(needed, 1))))
|
||||
|
||||
def _encode_chunk(
|
||||
self,
|
||||
tokens: torch.Tensor,
|
||||
@@ -245,6 +339,7 @@ class BatchedToken2Wav(nn.Module):
|
||||
cnn_cache: torch.Tensor | None,
|
||||
att_cache: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
self._ensure_relpos_pe(tokens, att_cache)
|
||||
embedded = self.flow.input_embedding(tokens)
|
||||
hidden, new_cnn, new_att = self.flow.encoder.forward_chunk(
|
||||
xs=embedded,
|
||||
@@ -508,14 +603,68 @@ class BatchedToken2Wav(nn.Module):
|
||||
# must carry at least one full kernel. Only the final chunk is allowed
|
||||
# to be shorter: ``forward_chunk`` zero-pads it by the lookahead width.
|
||||
lookahead = self._pre_lookahead_len()
|
||||
num_frames = int(tokens.shape[1])
|
||||
if lookahead is not None and not last_chunk:
|
||||
num_frames = int(tokens.shape[1])
|
||||
if num_frames <= lookahead:
|
||||
raise RuntimeError(
|
||||
"MiniCPMO45Code2WavBatchError "
|
||||
f'{{"reason":"chunk_below_lookahead_window","frames":{num_frames},'
|
||||
f'"minimum":{lookahead + 1}}}'
|
||||
)
|
||||
# A non-async Talker dump can land thousands of codec tokens in one
|
||||
# Code2Wav prefill. CosyVoice RelPos PE (max_len=5000) plus 2x upsample
|
||||
# cannot score that in one ``forward_chunk`` (6968 vs 985 on NPU).
|
||||
max_frames = self._max_encode_token_frames(states)
|
||||
slices = plan_token2wav_encode_slices(
|
||||
num_frames,
|
||||
max_frames=max_frames,
|
||||
min_nonfinal=(lookahead + 1) if lookahead is not None else 1,
|
||||
last_chunk=last_chunk,
|
||||
)
|
||||
if len(slices) > 1:
|
||||
logger.info(
|
||||
"MiniCPM-o Code2Wav splitting %d codec tokens into %d encoder windows "
|
||||
"(max_frames=%d) to stay inside RelPos PE.",
|
||||
num_frames,
|
||||
len(slices),
|
||||
max_frames,
|
||||
)
|
||||
parts: list[list[torch.Tensor]] = [[] for _ in range(batch_size)]
|
||||
current = states
|
||||
for index, (start, end) in enumerate(slices):
|
||||
is_last_piece = index == len(slices) - 1
|
||||
audios, current = self._decode_batch_once(
|
||||
tokens[:, start:end],
|
||||
features,
|
||||
current,
|
||||
last_chunk=last_chunk and is_last_piece,
|
||||
flush_encoder=flush_encoder and is_last_piece,
|
||||
)
|
||||
for row, audio in enumerate(audios):
|
||||
parts[row].append(audio)
|
||||
merged = [
|
||||
torch.cat(row_parts) if row_parts else tokens.new_zeros((0,), dtype=torch.float32)
|
||||
for row_parts in parts
|
||||
]
|
||||
return merged, current
|
||||
return self._decode_batch_once(
|
||||
tokens,
|
||||
features,
|
||||
states,
|
||||
last_chunk=last_chunk,
|
||||
flush_encoder=flush_encoder,
|
||||
)
|
||||
|
||||
def _decode_batch_once(
|
||||
self,
|
||||
tokens: torch.Tensor,
|
||||
features: PromptFeatures,
|
||||
states: list[BatchedToken2WavState],
|
||||
*,
|
||||
last_chunk: bool,
|
||||
flush_encoder: bool = False,
|
||||
) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]:
|
||||
batch_size = int(tokens.shape[0])
|
||||
flow_cache = self._stack_flow_cache(states)
|
||||
speakers = features.speaker_embedding.expand(batch_size, -1)
|
||||
with self._autocast(tokens.device):
|
||||
|
||||
@@ -301,6 +301,10 @@ class ConditionalChatTTSConfig(PretrainedConfig):
|
||||
self.top_p = top_p
|
||||
self.top_k = top_k
|
||||
self.repetition_penalty = repetition_penalty
|
||||
# Talker stage is a single-vocab codec LM: vLLM's Sampler and
|
||||
# embed_tokens both address this table, not the Thinker text vocab.
|
||||
self.vocab_size = int(num_audio_tokens)
|
||||
self.eos_token_id = int(num_audio_tokens) - 1
|
||||
|
||||
|
||||
class MiniCPMOConfig(Qwen2Config):
|
||||
|
||||
@@ -8,11 +8,12 @@ Pipeline:
|
||||
1. Receive thinker hidden_states + full token IDs via additional_information
|
||||
2. Extract tts_bos..tts_eos region
|
||||
3. Build condition: emb_text(tokens) + projector_semantic(hidden) (hidden_text_merge)
|
||||
4. Continuously generate request-aligned discrete audio-code deltas
|
||||
4. Project last hidden through head_code; vLLM Sampler picks the codec id
|
||||
5. Next decode embeds that id with emb_code and emits it to Code2Wav
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -32,45 +33,34 @@ from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_REPETITION_WINDOW = 16
|
||||
_REPETITION_PENALTY_CHUNK_SIZE = 16
|
||||
_MIN_AUDIO_TOKENS = 64
|
||||
_MAX_AUDIO_TOKENS = 2048
|
||||
_AUDIO_TOKENS_PER_TEXT_TOKEN = 10
|
||||
# Codec-token sampling happens inside the model; vLLM sampling parameters
|
||||
# only choose the Talker's binary continue/stop row.
|
||||
_CODEC_SEED = 42
|
||||
_CODEC_TEMPERATURE = 0.8
|
||||
_CODEC_TOP_K = 25
|
||||
_CODEC_TOP_P = 0.85
|
||||
_CODEC_REPETITION_PENALTY = 1.05
|
||||
_CODEC_MIN_TOKENS = 50
|
||||
# Native duplex Talker must finish after one MiniCPMTTS.generate_chunk:
|
||||
# 25 codec frames (``codec_chunk_frames``) plus the terminating sample.
|
||||
# Without this, the single-vocab Sampler keeps the stage-1 request alive
|
||||
# until codec EOS / 4096 and Thinker never starts the next model turn.
|
||||
_DUPLEX_CODEC_TOKENS_PER_CHUNK = 26
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _PendingCodecSample:
|
||||
output_index: int
|
||||
hidden_row: torch.Tensor
|
||||
codes: torch.Tensor
|
||||
request_id: str
|
||||
step: int
|
||||
state: dict[str, Any]
|
||||
info: dict[str, Any]
|
||||
def _native_duplex_chunk_budget(meta: Mapping[str, Any] | None) -> tuple[int, int]:
|
||||
"""Return ``(max_tokens, min_tokens)`` for one native-duplex Talker request."""
|
||||
boundary = isinstance(meta, Mapping) and (bool(meta.get("turn_start")) or bool(meta.get("turn_end")))
|
||||
ceiling = _DUPLEX_CODEC_TOKENS_PER_CHUNK
|
||||
return ceiling, 0 if boundary else ceiling
|
||||
|
||||
|
||||
def _max_audio_tokens(condition_tokens: int) -> int:
|
||||
"""Bound codec generation with a conservative text-length estimate.
|
||||
def blank_scheduler_prompt_for_penalties(
|
||||
prompt_token_ids: torch.Tensor,
|
||||
vocab_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Return a penalty prompt whose every position is the pad id (``vocab_size``).
|
||||
|
||||
EOS is masked for the first 50 steps, so a direct ``text_tokens * 10``
|
||||
limit can terminate short responses before EOS is eligible. The 2048
|
||||
ceiling matches the checkpoint's native generation default and keeps the
|
||||
sequence within the Talker's 4096-position context.
|
||||
No Talker prompt position is codec history: prefill conditioning arrives as
|
||||
embeddings from ``preprocess`` and decode embeds sampled ids with
|
||||
``emb_code``. The scheduler ids are placeholders (``llm2tts`` fills them
|
||||
with ``0``, or with thinker token ids on the non-handoff path), so scoring
|
||||
them would tax unrelated codec tokens.
|
||||
"""
|
||||
return max(
|
||||
_MIN_AUDIO_TOKENS,
|
||||
min(_MAX_AUDIO_TOKENS, condition_tokens * _AUDIO_TOKENS_PER_TEXT_TOKEN),
|
||||
)
|
||||
return torch.full_like(prompt_token_ids, int(vocab_size))
|
||||
|
||||
|
||||
def _restore_weight_norm_weight(weight_g: torch.Tensor, weight_v: torch.Tensor) -> torch.Tensor:
|
||||
@@ -122,32 +112,6 @@ def _apply_batched_repetition_penalty(
|
||||
return penalized
|
||||
|
||||
|
||||
def _apply_top_k_top_p(
|
||||
logits: torch.Tensor,
|
||||
*,
|
||||
top_k: int | None,
|
||||
top_p: float | None,
|
||||
min_tokens_to_keep: int = 3,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the same candidate floors as the upstream Transformers warpers."""
|
||||
filtered = logits.clone()
|
||||
vocab_size = filtered.shape[-1]
|
||||
# MiniCPM-o's gen_logits() appends TopPLogitsWarper before
|
||||
# TopKLogitsWarper. The order is observable for fixed-seed sampling.
|
||||
if top_p is not None and 0.0 < top_p < 1.0:
|
||||
sorted_logits, sorted_indices = torch.sort(filtered, descending=False, dim=-1)
|
||||
cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
|
||||
remove = cumulative_probs <= (1.0 - float(top_p))
|
||||
remove[..., -min_tokens_to_keep:] = False
|
||||
remove = remove.scatter(-1, sorted_indices, remove)
|
||||
filtered.masked_fill_(remove, float("-inf"))
|
||||
if top_k is not None and top_k > 0:
|
||||
keep = min(vocab_size, max(int(top_k), min_tokens_to_keep))
|
||||
threshold = torch.topk(filtered, keep, dim=-1).values[..., -1, None]
|
||||
filtered.masked_fill_(filtered < threshold, float("-inf"))
|
||||
return filtered
|
||||
|
||||
|
||||
class _MiniCPMTTSProjector(nn.Module):
|
||||
"""Checkpoint-compatible hidden-state projector used by MiniCPMTTS."""
|
||||
|
||||
@@ -173,8 +137,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
config: MiniCPMOConfig = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.vllm_config = vllm_config
|
||||
self._batch_stop_logits: torch.Tensor | None = None
|
||||
self._request_generators: dict[str, torch.Generator] = {}
|
||||
self._force_eos_rows: list[bool] | None = None
|
||||
self._mask_eos_rows: list[bool] | None = None
|
||||
self._request_audio_states: dict[str, dict[str, Any]] = {}
|
||||
self._deferred_cleanup_ids: set[str] = set()
|
||||
|
||||
@@ -186,23 +150,18 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
self._tts_bos_id = getattr(tts_config, "audio_bos_token_id", 151687)
|
||||
self._text_eos_id = getattr(tts_config, "text_eos_token_id", 151692)
|
||||
self._num_audio_tokens = getattr(tts_config, "num_audio_tokens", 6562)
|
||||
self._codec_eos_id = int(getattr(tts_config, "eos_token_id", self._num_audio_tokens - 1))
|
||||
self._hidden_size = getattr(tts_config, "hidden_size", 768)
|
||||
self._normalize = getattr(tts_config, "normalize_projected_hidden", True)
|
||||
self._codec_seed = int(getattr(tts_config, "seed", _CODEC_SEED))
|
||||
self._codec_temperature = float(getattr(tts_config, "temperature", _CODEC_TEMPERATURE))
|
||||
self._codec_top_k = int(getattr(tts_config, "top_k", _CODEC_TOP_K))
|
||||
self._codec_top_p = float(getattr(tts_config, "top_p", _CODEC_TOP_P))
|
||||
self._codec_repetition_penalty = float(getattr(tts_config, "repetition_penalty", _CODEC_REPETITION_PENALTY))
|
||||
self._codec_min_tokens = int(getattr(tts_config, "min_new_tokens", _CODEC_MIN_TOKENS))
|
||||
else:
|
||||
self._tts_config = None
|
||||
self._codec_eos_id = 0
|
||||
|
||||
self.has_preprocess = True
|
||||
self.has_postprocess = False
|
||||
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {
|
||||
("audio_codes", "current"),
|
||||
("audio_codes", "accumulated"),
|
||||
}
|
||||
# Same-step codes travel through make_omni_output from the previous
|
||||
# sampled id (decode preprocess embeds that id via emb_code).
|
||||
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {("codes", "audio")}
|
||||
self._init_native_talker(prefix)
|
||||
|
||||
def _init_native_talker(self, prefix: str) -> None:
|
||||
@@ -326,6 +285,7 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
info_dict.get("request_id"),
|
||||
)
|
||||
native_duplex = bool(info_dict.get("native_duplex", False))
|
||||
meta = info_dict.get("meta")
|
||||
full_embeds = self._build_condition_embeddings(
|
||||
token_ids,
|
||||
hidden_states,
|
||||
@@ -333,7 +293,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
)
|
||||
offset = int(info_dict.get("_omni_num_computed_tokens", 0))
|
||||
request_id = str(info_dict.get("request_id", "0"))
|
||||
meta = info_dict.get("meta")
|
||||
# The handoff rebuilds only the tail-aligned Talker condition.
|
||||
# Materialize zero-token embeddings for any scheduler prompt
|
||||
# prefix so chunked prefill can slice from a non-zero offset.
|
||||
@@ -356,21 +315,11 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
f"tts_ids={token_ids.shape[0]} tts_hidden={hidden_states.shape[0]} "
|
||||
f"prompt_len={info_dict.get('_omni_prompt_len')}"
|
||||
)
|
||||
duplex_boundary = isinstance(meta, dict) and (
|
||||
bool(meta.get("turn_start", False)) or bool(meta.get("turn_end", False))
|
||||
)
|
||||
state: dict[str, Any] = {"finished": empty_condition, "step": 0}
|
||||
if native_duplex:
|
||||
max_tokens = _DUPLEX_CODEC_TOKENS_PER_CHUNK
|
||||
min_tokens = 0 if duplex_boundary else _DUPLEX_CODEC_TOKENS_PER_CHUNK
|
||||
else:
|
||||
max_tokens = _max_audio_tokens(int(token_ids.numel()))
|
||||
min_tokens = self._codec_min_tokens
|
||||
state = {
|
||||
"step": 0,
|
||||
"max_tokens": max_tokens,
|
||||
"min_tokens": min_tokens,
|
||||
"finished": empty_condition,
|
||||
}
|
||||
max_tokens, min_tokens = _native_duplex_chunk_budget(meta if isinstance(meta, Mapping) else None)
|
||||
state["max_tokens"] = max_tokens
|
||||
state["min_tokens"] = min_tokens
|
||||
request_states = getattr(self, "_request_audio_states", None)
|
||||
if request_states is None:
|
||||
request_states = {}
|
||||
@@ -382,98 +331,38 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
embeds,
|
||||
{
|
||||
"audio_state": state,
|
||||
"audio_codes": {
|
||||
"current": empty_codes,
|
||||
"accumulated": empty_codes,
|
||||
},
|
||||
# Prefill has no previous codec id. vLLM samples the first
|
||||
# code after this forward; the next decode emits it.
|
||||
"codes": {"audio": empty_codes},
|
||||
},
|
||||
)
|
||||
|
||||
current = (info_dict.get("audio_codes", {}) or {}).get("current")
|
||||
if not isinstance(current, torch.Tensor) or current.numel() != 1:
|
||||
if state.get("finished"):
|
||||
# A request that finished before sampling any code can still be
|
||||
# scheduled for decode steps while sampling min_tokens masks the
|
||||
# stop token. make_omni_output ignores its hidden states, so any
|
||||
# shape-correct embedding will do.
|
||||
weight = self.emb_code[0].weight
|
||||
return input_ids, weight.new_zeros((span_len, weight.shape[1])), {}
|
||||
raise RuntimeError("MiniCPM-o Talker decode is missing the previous request-local audio code")
|
||||
code = current.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(1)
|
||||
request_id = str(info_dict.get("request_id", "0"))
|
||||
stored = self._request_audio_states.get(request_id)
|
||||
if isinstance(stored, dict):
|
||||
state = stored
|
||||
if isinstance(state, dict) and state.get("finished"):
|
||||
# An empty speech segment can still be scheduled until EOS is
|
||||
# eligible. The sampler is forced to EOS; any shape-correct
|
||||
# embedding is enough for these leftover decode rows.
|
||||
weight = self.emb_code[0].weight
|
||||
empty_codes = torch.empty(0, dtype=torch.long, device=weight.device)
|
||||
return input_ids, weight.new_zeros((span_len, weight.shape[1])), {"codes": {"audio": empty_codes}}
|
||||
|
||||
# Decode: vLLM's previous sampled codec id is this step's input.
|
||||
# Embed it with the codec table (not the 32k Llama embed_tokens) and
|
||||
# hand the same id to make_omni_output so Code2Wav sees it this step.
|
||||
code = input_ids.to(device=self.emb_code[0].weight.device, dtype=torch.long).reshape(-1)[-1:]
|
||||
embeds = self.emb_code[0](code)
|
||||
return input_ids, embeds, {}
|
||||
|
||||
def _request_generator(self, request_id: str, device: torch.device) -> torch.Generator:
|
||||
generator = self._request_generators.get(request_id)
|
||||
if generator is None:
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(self._codec_seed)
|
||||
self._request_generators[request_id] = generator
|
||||
return generator
|
||||
|
||||
def _sample_audio_codes(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
histories: Sequence[torch.Tensor],
|
||||
request_ids: Sequence[str],
|
||||
steps: Sequence[int],
|
||||
) -> torch.Tensor:
|
||||
batch_size = int(hidden_states.shape[0])
|
||||
if not (len(histories) == len(request_ids) == len(steps) == batch_size):
|
||||
raise ValueError(
|
||||
"MiniCPM-o batched codec sampling requires one history, request id, "
|
||||
f"and step per hidden row, got batch={batch_size}, histories={len(histories)}, "
|
||||
f"request_ids={len(request_ids)}, steps={len(steps)}"
|
||||
)
|
||||
if batch_size == 0:
|
||||
return torch.empty(0, dtype=torch.long, device=hidden_states.device)
|
||||
|
||||
logits = self.head_code[0](hidden_states).float() / self._codec_temperature
|
||||
eos_id = self._num_audio_tokens - 1
|
||||
logits = _apply_batched_repetition_penalty(
|
||||
logits,
|
||||
histories,
|
||||
penalty=self._codec_repetition_penalty,
|
||||
window_size=_REPETITION_WINDOW,
|
||||
)
|
||||
request_states = getattr(self, "_request_audio_states", {})
|
||||
mask_eos_values: list[bool] = []
|
||||
for request_id, step in zip(request_ids, steps, strict=True):
|
||||
state = request_states.get(request_id)
|
||||
min_tokens = (
|
||||
int(state.get("min_tokens", self._codec_min_tokens))
|
||||
if isinstance(state, dict)
|
||||
else self._codec_min_tokens
|
||||
)
|
||||
mask_eos_values.append(step < min_tokens)
|
||||
mask_eos = torch.tensor(
|
||||
mask_eos_values,
|
||||
dtype=torch.bool,
|
||||
device=logits.device,
|
||||
)
|
||||
logits[:, eos_id].masked_fill_(mask_eos, float("-inf"))
|
||||
logits = _apply_top_k_top_p(
|
||||
logits,
|
||||
top_k=self._codec_top_k,
|
||||
top_p=self._codec_top_p,
|
||||
min_tokens_to_keep=3,
|
||||
)
|
||||
probabilities = torch.softmax(logits, dim=-1)
|
||||
# torch.multinomial accepts one generator for the entire call. Keep it
|
||||
# per row so compaction, request ordering, or another request finishing
|
||||
# cannot perturb request-local RNG streams; the expensive projection
|
||||
# and logit transforms above remain fully batched.
|
||||
return torch.cat(
|
||||
[
|
||||
torch.multinomial(
|
||||
probabilities[row : row + 1],
|
||||
num_samples=1,
|
||||
generator=self._request_generator(request_id, probabilities.device),
|
||||
)
|
||||
for row, request_id in enumerate(request_ids)
|
||||
],
|
||||
dim=0,
|
||||
).reshape(-1)
|
||||
if int(code.item()) == int(self._codec_eos_id):
|
||||
if isinstance(state, dict):
|
||||
state["finished"] = True
|
||||
elif stored is None:
|
||||
self._request_audio_states[request_id] = {"finished": True, "step": 0}
|
||||
delta = torch.empty(0, dtype=torch.long, device=code.device)
|
||||
else:
|
||||
delta = code.reshape(1, 1)
|
||||
return input_ids, embeds, {"codes": {"audio": delta}}
|
||||
|
||||
def make_omni_output(
|
||||
self,
|
||||
@@ -487,18 +376,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
spans = kwargs.get("request_token_spans")
|
||||
if spans is None or len(spans) != len(infos):
|
||||
raise RuntimeError("MiniCPM-o continuous Talker requires one request_token_span per request")
|
||||
sample_eligible = kwargs.get("request_sample_eligible")
|
||||
if sample_eligible is None:
|
||||
sample_eligible = [True] * len(infos)
|
||||
if len(sample_eligible) != len(infos):
|
||||
raise RuntimeError(
|
||||
f"MiniCPM-o continuous Talker received {len(sample_eligible)} sampling flags for {len(infos)} requests"
|
||||
)
|
||||
emit_duplex_metadata = any(isinstance(info, dict) and info.get("native_duplex") is True for info in infos)
|
||||
|
||||
# Rows default to continue. Only previously finished or newly terminal
|
||||
# requests stop; prefill/ineligible rows stay aligned as False.
|
||||
stop_flags = [False] * len(infos)
|
||||
native_duplex_flags: list[torch.Tensor] = []
|
||||
duplex_epochs: list[torch.Tensor] = []
|
||||
duplex_turn_ids: list[torch.Tensor] = []
|
||||
@@ -507,7 +386,8 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
empty_delta = hidden.new_empty((0, 1), dtype=torch.long)
|
||||
codec_deltas = [empty_delta for _ in infos]
|
||||
terminal_flags = [torch.tensor(False, dtype=torch.bool) for _ in infos]
|
||||
pending_samples: list[_PendingCodecSample] = []
|
||||
force_eos_rows = [False] * len(infos)
|
||||
mask_eos_rows = [False] * len(infos)
|
||||
for index, info in enumerate(infos):
|
||||
info_dict = info if isinstance(info, dict) else {}
|
||||
native_duplex = info_dict.get("native_duplex") is True
|
||||
@@ -554,100 +434,39 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
start, end = spans[index]
|
||||
end = min(int(end), int(hidden.shape[0]))
|
||||
if int(start) >= end:
|
||||
continue
|
||||
request_id = str(info.get("request_id", index))
|
||||
request_states = getattr(self, "_request_audio_states", None)
|
||||
if request_states is None:
|
||||
request_states = {}
|
||||
self._request_audio_states = request_states
|
||||
state = request_states.get(request_id)
|
||||
state = self._request_audio_states.get(request_id)
|
||||
if not isinstance(state, dict):
|
||||
state = dict(info.get("audio_state", {}) or {})
|
||||
request_states[request_id] = state
|
||||
if state.get("finished"):
|
||||
stop_flags[index] = True
|
||||
continue
|
||||
if not sample_eligible[index]:
|
||||
# vLLM computes a logit row for incomplete chunked prefills but
|
||||
# discards its sampled token. Advancing codec/RNG state here
|
||||
# would make output depend on prefill chunking and compaction.
|
||||
continue
|
||||
codes = state.get("codes")
|
||||
if not isinstance(codes, torch.Tensor):
|
||||
codes = (info.get("audio_codes", {}) or {}).get("accumulated")
|
||||
if not isinstance(codes, torch.Tensor):
|
||||
codes = torch.empty(0, dtype=torch.long, device=hidden.device)
|
||||
else:
|
||||
codes = codes.to(device=hidden.device, dtype=torch.long).reshape(-1)
|
||||
self._request_audio_states[request_id] = state
|
||||
empty_speech = bool(state.get("finished"))
|
||||
codes = info.get("codes", {})
|
||||
audio = codes.get("audio") if isinstance(codes, Mapping) else None
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
codec_deltas[index] = audio.to(device=hidden.device, dtype=torch.long).reshape(-1, 1)
|
||||
if state.get("max_tokens") is not None:
|
||||
state["step"] = int(state.get("step", 0)) + 1
|
||||
max_tokens = state.get("max_tokens")
|
||||
min_tokens = state.get("min_tokens")
|
||||
step = int(state.get("step", 0))
|
||||
pending_samples.append(
|
||||
_PendingCodecSample(
|
||||
output_index=index,
|
||||
hidden_row=hidden[end - 1 : end],
|
||||
codes=codes,
|
||||
request_id=request_id,
|
||||
step=step,
|
||||
state=state,
|
||||
info=info,
|
||||
)
|
||||
)
|
||||
# 26 samples include the terminating EOS, so force it after 25
|
||||
# forwarded frames — the same cadence as generate_chunk.
|
||||
hit_chunk_limit = max_tokens is not None and step >= int(max_tokens) - 1
|
||||
chunk_done = empty_speech or hit_chunk_limit
|
||||
if hit_chunk_limit:
|
||||
# The next sampled id is forced EOS and will finish the
|
||||
# request; mark the chunk done on this step so the data
|
||||
# plane does not wait for a follow-up empty decode.
|
||||
state["finished"] = True
|
||||
force_eos_rows[index] = chunk_done
|
||||
mask_eos_rows[index] = not force_eos_rows[index] and min_tokens is not None and step < int(min_tokens)
|
||||
terminal_flags[index] = torch.tensor(chunk_done, dtype=torch.bool)
|
||||
|
||||
if pending_samples:
|
||||
active_hidden_rows = [pending.hidden_row for pending in pending_samples]
|
||||
active_hidden = (
|
||||
active_hidden_rows[0] if len(active_hidden_rows) == 1 else torch.cat(active_hidden_rows, dim=0)
|
||||
)
|
||||
sampled_batch = self._sample_audio_codes(
|
||||
active_hidden,
|
||||
[pending.codes for pending in pending_samples],
|
||||
[pending.request_id for pending in pending_samples],
|
||||
[pending.step for pending in pending_samples],
|
||||
)
|
||||
# One batched device-to-host synchronization replaces one .item()
|
||||
# synchronization per request.
|
||||
sampled_ids = sampled_batch.detach().to(device="cpu").tolist()
|
||||
else:
|
||||
sampled_batch = hidden.new_empty((0,), dtype=torch.long)
|
||||
sampled_ids = []
|
||||
|
||||
for row, pending in enumerate(pending_samples):
|
||||
sampled = sampled_batch[row].reshape(())
|
||||
sampled_id = int(sampled_ids[row])
|
||||
codes = pending.codes
|
||||
state = pending.state
|
||||
info = pending.info
|
||||
is_eos = sampled_id == self._num_audio_tokens - 1
|
||||
state["step"] = int(state.get("step", 0)) + 1
|
||||
reached_limit = int(state["step"]) >= int(state.get("max_tokens", 2048))
|
||||
finished = is_eos or reached_limit
|
||||
state["finished"] = finished
|
||||
# MiniCPMTTS.generate_chunk consumes the boundary sample but
|
||||
# returns only codes that were fed into the retained KV state.
|
||||
if not is_eos and not reached_limit:
|
||||
codes = torch.cat([codes[-(_REPETITION_WINDOW - 1) :], sampled.reshape(1)])
|
||||
delta = sampled.reshape(1, 1)
|
||||
else:
|
||||
delta = empty_delta
|
||||
state["codes"] = codes
|
||||
info["audio_state"] = state
|
||||
info["audio_codes"] = {
|
||||
"current": sampled.reshape(1),
|
||||
"accumulated": codes,
|
||||
}
|
||||
codec_deltas[pending.output_index] = delta
|
||||
terminal_flags[pending.output_index] = torch.tensor(finished, dtype=torch.bool)
|
||||
stop_flags[pending.output_index] = finished
|
||||
|
||||
self._batch_stop_logits = (
|
||||
hidden.new_tensor([[float("-inf"), 0.0] if finished else [0.0, float("-inf")] for finished in stop_flags])
|
||||
if stop_flags
|
||||
else hidden.new_empty((0, 2))
|
||||
)
|
||||
# Lists are deliberate: the runner routes element i to request i,
|
||||
# preserving compaction alignment while emitting only this step's code.
|
||||
# Empty-speech and finished duplex chunks must sample codec EOS so
|
||||
# the scheduler releases the request; mid-chunk rows mask EOS until
|
||||
# the 26-token generate_chunk budget is met.
|
||||
self._force_eos_rows = force_eos_rows
|
||||
self._mask_eos_rows = mask_eos_rows
|
||||
meta_outputs = {"finished": terminal_flags}
|
||||
if emit_duplex_metadata:
|
||||
meta_outputs.update(
|
||||
@@ -659,13 +478,12 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
"turn_end": turn_end_flags,
|
||||
}
|
||||
)
|
||||
multimodal_outputs: dict[str, Any] = {
|
||||
"codes": {"audio": codec_deltas},
|
||||
"meta": meta_outputs,
|
||||
}
|
||||
return OmniOutput(
|
||||
text_hidden_states=hidden,
|
||||
multimodal_outputs=multimodal_outputs,
|
||||
multimodal_outputs={
|
||||
"codes": {"audio": codec_deltas},
|
||||
"meta": meta_outputs,
|
||||
},
|
||||
)
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
@@ -674,7 +492,6 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
def _flush_deferred_cleanup(self) -> None:
|
||||
request_audio_states = getattr(self, "_request_audio_states", {})
|
||||
for request_id in self._deferred_cleanup_ids:
|
||||
self._request_generators.pop(request_id, None)
|
||||
request_audio_states.pop(request_id, None)
|
||||
self._deferred_cleanup_ids.clear()
|
||||
|
||||
@@ -723,18 +540,38 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
def compute_logits(self, hidden_states, *args, **kwargs):
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
return None
|
||||
if self._batch_stop_logits is None:
|
||||
return torch.zeros(
|
||||
hidden_states.shape[0],
|
||||
2,
|
||||
device=hidden_states.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
logits = self._batch_stop_logits
|
||||
self._batch_stop_logits = None
|
||||
if hidden_states.numel() == 0:
|
||||
return hidden_states.new_empty((0, int(self._num_audio_tokens)))
|
||||
logits = self.head_code[0](hidden_states).float()
|
||||
force_eos = self._force_eos_rows
|
||||
mask_eos = self._mask_eos_rows
|
||||
self._force_eos_rows = None
|
||||
self._mask_eos_rows = None
|
||||
need_force = bool(force_eos and len(force_eos) == logits.shape[0] and any(force_eos))
|
||||
need_mask = bool(mask_eos and len(mask_eos) == logits.shape[0] and any(mask_eos))
|
||||
if need_force or need_mask:
|
||||
logits = logits.clone()
|
||||
eos_id = int(self._codec_eos_id)
|
||||
if need_force:
|
||||
assert force_eos is not None
|
||||
forced = torch.tensor(force_eos, dtype=torch.bool, device=logits.device)
|
||||
logits[forced] = float("-inf")
|
||||
logits[forced, eos_id] = 0.0
|
||||
if need_mask:
|
||||
assert mask_eos is not None
|
||||
masked = torch.tensor(mask_eos, dtype=torch.bool, device=logits.device)
|
||||
logits[masked, eos_id] = float("-inf")
|
||||
return logits
|
||||
|
||||
def sample(self, logits, sampling_metadata):
|
||||
prompt_ids = getattr(sampling_metadata, "prompt_token_ids", None)
|
||||
if isinstance(logits, torch.Tensor) and isinstance(prompt_ids, torch.Tensor):
|
||||
# Copy rather than mutate: the runner may hand us the input batch's
|
||||
# own persistent SamplingMetadata.
|
||||
sampling_metadata = replace(
|
||||
sampling_metadata,
|
||||
prompt_token_ids=blank_scheduler_prompt_for_penalties(prompt_ids, logits.shape[-1]),
|
||||
)
|
||||
return Sampler()(logits, sampling_metadata)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
@@ -782,9 +619,10 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
return loaded
|
||||
|
||||
def get_input_embeddings(self, input_ids, multimodal_embeddings=None, **kwargs):
|
||||
if hasattr(self, "emb_text") and self.emb_text is not None:
|
||||
return self.emb_text(input_ids)
|
||||
return torch.zeros(input_ids.shape[0], 1)
|
||||
del multimodal_embeddings
|
||||
# Decode tokens live in the codec table. Prefill overwrites these
|
||||
# embeddings in preprocess with emb_text + projected thinker hidden.
|
||||
return self.emb_code[0](input_ids)
|
||||
|
||||
def embed_input_ids(self, input_ids, **kwargs):
|
||||
return self.get_input_embeddings(input_ids, **kwargs)
|
||||
@@ -62,7 +62,12 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
|
||||
custom_process_input_func=f"{_PROC}.llm2tts",
|
||||
custom_process_next_stage_input_func=f"{_PROC}.tts2code2wav_full_payload",
|
||||
async_chunk_process_next_stage_input_func=f"{_PROC}.tts2code2wav_async_chunk",
|
||||
sampling_constraints={"detokenize": False},
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
# MiniCPM-o 4.5 codec EOS is tts_config.num_audio_tokens - 1.
|
||||
# Same pattern as Qwen3 talker's stop_token_ids: [2150].
|
||||
"stop_token_ids": [6561],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=2,
|
||||
|
||||
Reference in new issue
Block a user