[Model] Preserve MiniCPM-o Talker full attention until capacity (#6799)

Signed-off-by: natureofnature <wzliu@connect.hku.hk>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
NATUREandamy-why-3459 authored and GitHub committed 2026-09-06 22:25:35 +08:00
1 parent 715b8b8746
commit ff6e906a25
12 files changed
+913 -108

No files matched your search

@@ -319,10 +319,26 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
if not recompute:
if condition_seq > previous_seq:
raise ValueError(
"a native duplex Talker condition advanced without its streaming recompute marker: "
f"current={condition_seq}, previous={previous_seq}"
)
attention_type = getattr(self._tts_config, "attention_type", "full_attention")
if attention_type == "sliding_recompute":
raise ValueError(
"a native duplex Talker condition advanced without its streaming recompute marker: "
f"current={condition_seq}, previous={previous_seq}"
)
audio_state = self._request_audio_states.get(request_id)
recent_codes = audio_state.get("recent_codes") if isinstance(audio_state, dict) else None
if isinstance(recent_codes, list):
base_recent_codes = tuple(int(code_id) for code_id in recent_codes[-_CODEC_PENALTY_WINDOW:])
else:
base_recent_codes = state.get("base_recent_codes")
if not isinstance(base_recent_codes, tuple):
raise ValueError("streaming Talker condition lost its frozen codec history")
states[request_id] = {
"condition_seq": condition_seq,
"condition": current_condition.detach().clone(),
"base_recent_codes": base_recent_codes,
}
return current_condition
if "active_embeddings" in state:
raise ValueError("an active streaming recompute was replayed without its recompute marker")
return current_condition
@@ -637,6 +637,7 @@ def _native_duplex_segment_output_ids(
state["request_id"] = request_id
state["sent_output_len"] = 0
state["sent_output_ids"] = []
state["condition_seq"] = -1
previous_turn_id = state.get("turn_id")
sent_len = state.get("sent_output_len", 0)
prev_output_ids = state.get("sent_output_ids", [])
@@ -1000,6 +1001,8 @@ def llm2tts(
if native_turn_end_handoff:
model_intermediate_buffer.setdefault("meta", {})["turn_end"] = True
condition_sequence_state = None
condition_sequence_value = None
if handoff_ids is not None and handoff_hidden is not None:
condition_suffix_length = 1 if is_native_duplex_handoff else 2
condition_length = max(len(handoff_ids), len(handoff_hidden)) + condition_suffix_length
@@ -1011,6 +1014,20 @@ def llm2tts(
handoff_meta["next_stage_prompt_len"] = condition_length
if is_native_duplex_handoff:
handoff_meta["next_stage_generation_tokens"] = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
bridge_states = getattr(_streaming_context, "bridge_states", None)
handoff_state = bridge_states.get("minicpmo45_tts_handoff") if isinstance(bridge_states, dict) else None
if not isinstance(handoff_state, dict) or handoff_state.get("request_id") != str(llm_output.request_id):
raise ValueError("native duplex Talker handoff is missing request-local streaming state")
previous_condition_seq = handoff_state.get("condition_seq", -1)
if (
not isinstance(previous_condition_seq, int)
or isinstance(previous_condition_seq, bool)
or previous_condition_seq < -1
):
raise ValueError("native duplex Talker condition sequence state is invalid")
condition_sequence_state = handoff_state
condition_sequence_value = previous_condition_seq + 1
handoff_meta["streaming_condition_seq"] = condition_sequence_value
# Native duplex resumes one Talker request within a turn, but a new
# assistant turn must discard the previous turn's prompt and KV.
if not is_native_duplex_handoff or native_turn_start:
@@ -1033,6 +1050,8 @@ def llm2tts(
mm_processor_kwargs=None,
)
)
if condition_sequence_state is not None:
condition_sequence_state["condition_seq"] = condition_sequence_value
if native_turn_end_handoff:
bridge_states = getattr(_streaming_context, "bridge_states", None)
duplex_state = bridge_states.get("duplex") if isinstance(bridge_states, dict) else None