[Bugfix][MiniCPM-o] Forward a speech unit closed by LISTEN after turn_eos to the Talker (#8227)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Opus 5.5 authored and GitHub committed 2026-09-28 11:11:42 +00:00
1 parent c0331048d0
commit 5ce6f5dc49
4 files changed
+136 -1

No files matched your search

@@ -13,6 +13,7 @@ from vllm_omni.engine.duplex.contracts import (
duplex_resource_request_belongs_to_session,
)
from vllm_omni.engine.duplex.plugin import DuplexDataPlane, DuplexDataPlaneContext
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy
from vllm_omni.outputs.duplex import get_duplex_output_decision
logger = init_logger(__name__)
@@ -638,9 +639,13 @@ def _native_decision(
return None
if mm_output.get("duplex_native_decision") == "listen" or mm_output.get("model_listen") is True:
return "listen"
listen_id = _special_token_ids(mm_output).get("listen_token_id")
special_token_ids = _special_token_ids(mm_output)
listen_id = special_token_ids.get("listen_token_id")
if listen_id is None:
return None
unit_ids = max(token_ids, _coerce_int_list(getattr(completion, "cumulative_token_ids", None)), key=len)
if MiniCPMO45DuplexPolicy.speech_unit_closed_by_listen(unit_ids, special_token_ids):
return None
stop_reason = getattr(completion, "stop_reason", None) if completion is not None else None
if coerce_int(stop_reason) == listen_id:
return "listen"
@@ -639,6 +639,14 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
token_ids = _completion_token_ids(completion) or list(segment_token_ids)
if _coerce_int(stop_reason) != listen_id and (not token_ids or token_ids[-1] != listen_id):
return None
unit_ids = max(
(token_ids, _coerce_int_list(getattr(completion, "cumulative_token_ids", None)), list(segment_token_ids)),
key=len,
)
if MiniCPMO45DuplexPolicy.speech_unit_closed_by_listen(unit_ids, special_token_ids):
# The unit's final speech and <|turn_eos|> must reach the Talker,
# or the response never ends.
return None
metadata = dict(output_metadata)
for key, value in special_token_ids.items():
@@ -7,6 +7,8 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
from transformers import PreTrainedTokenizerBase
@@ -93,6 +95,33 @@ class MiniCPMO45DuplexPolicy:
DEFAULT_MIN_NEW_SPEAK_TOKENS_BEFORE_CHUNK_BOUNDARY = 8
REPETITION_HISTORY_SIZE = 512
@staticmethod
def speech_unit_closed_by_listen(token_ids: Sequence[int], special_token_ids: Mapping[str, int]) -> bool:
"""Whether a unit ending in LISTEN closes speech rather than deciding to listen.
``<|turn_eos|>`` ends the turn but not the unit: the model keeps
decoding until a unit terminator, and on silent input the policy forces
that terminator to be LISTEN. Such a unit still carries the turn's final
speech and ``<|turn_eos|>`` for the Talker, so it is not a LISTEN
decision. Only the current unit counts: the scan stops at the previous
unit terminator.
"""
listen_id = special_token_ids.get("listen_token_id")
turn_eos_id = special_token_ids.get("turn_eos_token_id")
if listen_id is None or turn_eos_id is None or not token_ids or token_ids[-1] != listen_id:
return False
unit_terminators = {
listen_id,
special_token_ids.get("chunk_eos_token_id"),
special_token_ids.get("chunk_tts_eos_token_id"),
}
for token_id in reversed(token_ids[:-1]):
if token_id == turn_eos_id:
return True
if token_id in unit_terminators:
return False
return False
@classmethod
def audio_token_count(cls, sample_count: int) -> int:
"""Audio embedding count for a clip of ``sample_count`` samples.