[Model] Preserve MiniCPM-o Talker full attention until capacity (#6799)
Signed-off-by: natureofnature <wzliu@connect.hku.hk> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
715b8b8746
commit
ff6e906a25
12 files changed
+913
-108
No files matched your search
@@ -319,10 +319,26 @@ class MiniCPMO45OmniTTSForConditionalGeneration(nn.Module, SupportsPP):
|
||||
|
||||
if not recompute:
|
||||
if condition_seq > previous_seq:
|
||||
raise ValueError(
|
||||
"a native duplex Talker condition advanced without its streaming recompute marker: "
|
||||
f"current={condition_seq}, previous={previous_seq}"
|
||||
)
|
||||
attention_type = getattr(self._tts_config, "attention_type", "full_attention")
|
||||
if attention_type == "sliding_recompute":
|
||||
raise ValueError(
|
||||
"a native duplex Talker condition advanced without its streaming recompute marker: "
|
||||
f"current={condition_seq}, previous={previous_seq}"
|
||||
)
|
||||
audio_state = self._request_audio_states.get(request_id)
|
||||
recent_codes = audio_state.get("recent_codes") if isinstance(audio_state, dict) else None
|
||||
if isinstance(recent_codes, list):
|
||||
base_recent_codes = tuple(int(code_id) for code_id in recent_codes[-_CODEC_PENALTY_WINDOW:])
|
||||
else:
|
||||
base_recent_codes = state.get("base_recent_codes")
|
||||
if not isinstance(base_recent_codes, tuple):
|
||||
raise ValueError("streaming Talker condition lost its frozen codec history")
|
||||
states[request_id] = {
|
||||
"condition_seq": condition_seq,
|
||||
"condition": current_condition.detach().clone(),
|
||||
"base_recent_codes": base_recent_codes,
|
||||
}
|
||||
return current_condition
|
||||
if "active_embeddings" in state:
|
||||
raise ValueError("an active streaming recompute was replayed without its recompute marker")
|
||||
return current_condition
|
||||
|
||||
@@ -637,6 +637,7 @@ def _native_duplex_segment_output_ids(
|
||||
state["request_id"] = request_id
|
||||
state["sent_output_len"] = 0
|
||||
state["sent_output_ids"] = []
|
||||
state["condition_seq"] = -1
|
||||
previous_turn_id = state.get("turn_id")
|
||||
sent_len = state.get("sent_output_len", 0)
|
||||
prev_output_ids = state.get("sent_output_ids", [])
|
||||
@@ -1000,6 +1001,8 @@ def llm2tts(
|
||||
if native_turn_end_handoff:
|
||||
model_intermediate_buffer.setdefault("meta", {})["turn_end"] = True
|
||||
|
||||
condition_sequence_state = None
|
||||
condition_sequence_value = None
|
||||
if handoff_ids is not None and handoff_hidden is not None:
|
||||
condition_suffix_length = 1 if is_native_duplex_handoff else 2
|
||||
condition_length = max(len(handoff_ids), len(handoff_hidden)) + condition_suffix_length
|
||||
@@ -1011,6 +1014,20 @@ def llm2tts(
|
||||
handoff_meta["next_stage_prompt_len"] = condition_length
|
||||
if is_native_duplex_handoff:
|
||||
handoff_meta["next_stage_generation_tokens"] = MINICPMO45_DUPLEX_CODEC_TOKENS_PER_CHUNK
|
||||
bridge_states = getattr(_streaming_context, "bridge_states", None)
|
||||
handoff_state = bridge_states.get("minicpmo45_tts_handoff") if isinstance(bridge_states, dict) else None
|
||||
if not isinstance(handoff_state, dict) or handoff_state.get("request_id") != str(llm_output.request_id):
|
||||
raise ValueError("native duplex Talker handoff is missing request-local streaming state")
|
||||
previous_condition_seq = handoff_state.get("condition_seq", -1)
|
||||
if (
|
||||
not isinstance(previous_condition_seq, int)
|
||||
or isinstance(previous_condition_seq, bool)
|
||||
or previous_condition_seq < -1
|
||||
):
|
||||
raise ValueError("native duplex Talker condition sequence state is invalid")
|
||||
condition_sequence_state = handoff_state
|
||||
condition_sequence_value = previous_condition_seq + 1
|
||||
handoff_meta["streaming_condition_seq"] = condition_sequence_value
|
||||
# Native duplex resumes one Talker request within a turn, but a new
|
||||
# assistant turn must discard the previous turn's prompt and KV.
|
||||
if not is_native_duplex_handoff or native_turn_start:
|
||||
@@ -1033,6 +1050,8 @@ def llm2tts(
|
||||
mm_processor_kwargs=None,
|
||||
)
|
||||
)
|
||||
if condition_sequence_state is not None:
|
||||
condition_sequence_state["condition_seq"] = condition_sequence_value
|
||||
if native_turn_end_handoff:
|
||||
bridge_states = getattr(_streaming_context, "bridge_states", None)
|
||||
duplex_state = bridge_states.get("duplex") if isinstance(bridge_states, dict) else None
|
||||
|
||||
Reference in new issue
Block a user