[Bugfix][MiniCPM-o] Forward a speech unit closed by LISTEN after turn_eos to the Talker (#8227)
Signed-off-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c0331048d0
commit
5ce6f5dc49
4 files changed
+136
-1
No files matched your search
@@ -13,6 +13,7 @@ from vllm_omni.engine.duplex.contracts import (
|
||||
duplex_resource_request_belongs_to_session,
|
||||
)
|
||||
from vllm_omni.engine.duplex.plugin import DuplexDataPlane, DuplexDataPlaneContext
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy
|
||||
from vllm_omni.outputs.duplex import get_duplex_output_decision
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -638,9 +639,13 @@ def _native_decision(
|
||||
return None
|
||||
if mm_output.get("duplex_native_decision") == "listen" or mm_output.get("model_listen") is True:
|
||||
return "listen"
|
||||
listen_id = _special_token_ids(mm_output).get("listen_token_id")
|
||||
special_token_ids = _special_token_ids(mm_output)
|
||||
listen_id = special_token_ids.get("listen_token_id")
|
||||
if listen_id is None:
|
||||
return None
|
||||
unit_ids = max(token_ids, _coerce_int_list(getattr(completion, "cumulative_token_ids", None)), key=len)
|
||||
if MiniCPMO45DuplexPolicy.speech_unit_closed_by_listen(unit_ids, special_token_ids):
|
||||
return None
|
||||
stop_reason = getattr(completion, "stop_reason", None) if completion is not None else None
|
||||
if coerce_int(stop_reason) == listen_id:
|
||||
return "listen"
|
||||
|
||||
@@ -639,6 +639,14 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
|
||||
token_ids = _completion_token_ids(completion) or list(segment_token_ids)
|
||||
if _coerce_int(stop_reason) != listen_id and (not token_ids or token_ids[-1] != listen_id):
|
||||
return None
|
||||
unit_ids = max(
|
||||
(token_ids, _coerce_int_list(getattr(completion, "cumulative_token_ids", None)), list(segment_token_ids)),
|
||||
key=len,
|
||||
)
|
||||
if MiniCPMO45DuplexPolicy.speech_unit_closed_by_listen(unit_ids, special_token_ids):
|
||||
# The unit's final speech and <|turn_eos|> must reach the Talker,
|
||||
# or the response never ends.
|
||||
return None
|
||||
|
||||
metadata = dict(output_metadata)
|
||||
for key, value in special_token_ids.items():
|
||||
|
||||
@@ -7,6 +7,8 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
|
||||
@@ -93,6 +95,33 @@ class MiniCPMO45DuplexPolicy:
|
||||
DEFAULT_MIN_NEW_SPEAK_TOKENS_BEFORE_CHUNK_BOUNDARY = 8
|
||||
REPETITION_HISTORY_SIZE = 512
|
||||
|
||||
@staticmethod
|
||||
def speech_unit_closed_by_listen(token_ids: Sequence[int], special_token_ids: Mapping[str, int]) -> bool:
|
||||
"""Whether a unit ending in LISTEN closes speech rather than deciding to listen.
|
||||
|
||||
``<|turn_eos|>`` ends the turn but not the unit: the model keeps
|
||||
decoding until a unit terminator, and on silent input the policy forces
|
||||
that terminator to be LISTEN. Such a unit still carries the turn's final
|
||||
speech and ``<|turn_eos|>`` for the Talker, so it is not a LISTEN
|
||||
decision. Only the current unit counts: the scan stops at the previous
|
||||
unit terminator.
|
||||
"""
|
||||
listen_id = special_token_ids.get("listen_token_id")
|
||||
turn_eos_id = special_token_ids.get("turn_eos_token_id")
|
||||
if listen_id is None or turn_eos_id is None or not token_ids or token_ids[-1] != listen_id:
|
||||
return False
|
||||
unit_terminators = {
|
||||
listen_id,
|
||||
special_token_ids.get("chunk_eos_token_id"),
|
||||
special_token_ids.get("chunk_tts_eos_token_id"),
|
||||
}
|
||||
for token_id in reversed(token_ids[:-1]):
|
||||
if token_id == turn_eos_id:
|
||||
return True
|
||||
if token_id in unit_terminators:
|
||||
return False
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def audio_token_count(cls, sample_count: int) -> int:
|
||||
"""Audio embedding count for a clip of ``sample_count`` samples.
|
||||
|
||||
Reference in new issue
Block a user