[Bugfix] Add opt-in MiniCPM-o 4.5 Stage-0 sliding window (#7631)
Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de> Signed-off-by: 0z5a <dezhen.lu@uni-tuebingen.de> Co-authored-by: 0z5a <dezhen.lu@uni-tuebingen.de>
This commit is contained in:
1 parent
c11596c637
commit
8bc0c38b90
14 files changed
+1230
-12
No files matched your search
@@ -45,7 +45,10 @@ from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import (
|
||||
MiniCPMO45DataPlaneContext,
|
||||
MiniCPMO45DataPlaneSession,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
|
||||
MiniCPMO45DuplexPolicy,
|
||||
MiniCPMO45DuplexWindowConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import (
|
||||
MiniCPMO45ServingSessionState,
|
||||
)
|
||||
@@ -85,6 +88,11 @@ PRIVATE_RUNTIME_CONFIG_KEYS = frozenset(
|
||||
"ref_audio_format",
|
||||
"ref_audio_sample_rate_hz",
|
||||
"initial_user_text",
|
||||
"duplex_window_config",
|
||||
"duplex_window_prefix_tokens",
|
||||
"duplex_window_suffix_token_ids",
|
||||
"duplex_window_previous_marker_token_ids",
|
||||
"duplex_window_special_token_ids",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -245,8 +253,8 @@ def build_duplex_data_plane_prompt(
|
||||
token_budget = context_reserve + first_units * 12 - 1 + vision_tokens
|
||||
if seq > 1 and duplex_payload_is_exact_chunks(payload):
|
||||
token_budget += 1
|
||||
if final and duplex_payload_is_exact_chunks(payload):
|
||||
token_budget += 12
|
||||
# Serving already pads the final residual audio. Stage0 does not
|
||||
# append another silent unit, so final must not reserve extra slots.
|
||||
extra_body = session_config.get("extra_body")
|
||||
raw_token_id = runtime_config.get("duplex_scheduler_token_id")
|
||||
try:
|
||||
@@ -515,6 +523,16 @@ def _apply_first_append_context_tokens(
|
||||
return
|
||||
ref_tokens = MiniCPMO45DuplexPolicy.audio_token_count(ref_sample_count or 0)
|
||||
runtime_config["duplex_first_append_context_tokens"] = len(prefix_ids) + ref_tokens + len(suffix_ids)
|
||||
runtime_config["duplex_window_prefix_tokens"] = len(prefix_ids) + ref_tokens
|
||||
runtime_config["duplex_window_suffix_token_ids"] = [int(token_id) for token_id in suffix_ids]
|
||||
marker_ids = tokenizer.encode("\n\nprevious: ", add_special_tokens=False)
|
||||
runtime_config["duplex_window_previous_marker_token_ids"] = [int(token_id) for token_id in marker_ids]
|
||||
runtime_config["duplex_window_special_token_ids"] = sorted(
|
||||
MiniCPMO45DuplexPolicy.native_special_token_ids(
|
||||
MiniCPMO45DuplexPolicy.token_ids_from_tokenizer(tokenizer),
|
||||
tokenizer_special_ids=list(getattr(tokenizer, "all_special_ids", ()) or ()),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _model_vision_tile_pixels(model_config: ModelConfig | None) -> int | None:
|
||||
@@ -722,6 +740,8 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
|
||||
code="unsupported_ref_audio_path",
|
||||
)
|
||||
runtime_config: dict[str, object] = {"instructions": config.instructions}
|
||||
window_config = self._pop_window_config(extra_body) or MiniCPMO45DuplexWindowConfig()
|
||||
runtime_config["duplex_window_config"] = window_config.as_dict()
|
||||
# ``duplex_initial_user_text`` is the older extra_body spelling and
|
||||
# still works; the session field is the framework-level one.
|
||||
initial_user_text = extra_body.pop("duplex_initial_user_text", None)
|
||||
@@ -788,6 +808,17 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
|
||||
current: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
runtime_config = deepcopy(dict(current))
|
||||
extra_body = dict(config.extra_body)
|
||||
requested_window = self._pop_window_config(extra_body)
|
||||
if requested_window is not None:
|
||||
reject_changed_runtime_value(
|
||||
requested_window.as_dict(),
|
||||
runtime_config.get("duplex_window_config"),
|
||||
message="sliding-window configuration cannot be changed after the session is created",
|
||||
code="sliding_window_update_unsupported",
|
||||
error_cls=MiniCPMO45ClientRuntimeConfigError,
|
||||
)
|
||||
config.extra_body = extra_body
|
||||
reject_changed_runtime_value(
|
||||
config.instructions,
|
||||
runtime_config.get("instructions"),
|
||||
@@ -812,6 +843,23 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
|
||||
runtime_config["duplex_stage_sampling_params"] = stage_sampling
|
||||
return runtime_config
|
||||
|
||||
@staticmethod
|
||||
def _pop_window_config(extra_body: dict[str, object]) -> MiniCPMO45DuplexWindowConfig | None:
|
||||
names = (
|
||||
"sliding_window_mode",
|
||||
"basic_window_high_tokens",
|
||||
"basic_window_low_tokens",
|
||||
"context_previous_max_tokens",
|
||||
"context_max_units",
|
||||
)
|
||||
provided = {name: extra_body.pop(name) for name in names if name in extra_body}
|
||||
if not provided:
|
||||
return None
|
||||
try:
|
||||
return MiniCPMO45DuplexWindowConfig.from_mapping(provided)
|
||||
except ValueError as exc:
|
||||
raise MiniCPMO45ClientRuntimeConfigError(str(exc), code="invalid_sliding_window_config") from exc
|
||||
|
||||
def data_plane_context(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -3,12 +3,63 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MiniCPMO45DuplexWindowConfig:
|
||||
"""Stage-0 window settings matching the released checkpoint defaults."""
|
||||
|
||||
sliding_window_mode: str = "off"
|
||||
basic_window_high_tokens: int = 8000
|
||||
basic_window_low_tokens: int = 6000
|
||||
context_previous_max_tokens: int = 500
|
||||
context_max_units: int = 24
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: object) -> MiniCPMO45DuplexWindowConfig:
|
||||
source = value if isinstance(value, dict) else {}
|
||||
|
||||
def integer(name: str, default: int) -> int:
|
||||
raw = source.get(name, default)
|
||||
if isinstance(raw, bool):
|
||||
raise ValueError(f"{name} must be an integer")
|
||||
try:
|
||||
parsed = int(raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{name} must be an integer") from exc
|
||||
if parsed <= 0:
|
||||
raise ValueError(f"{name} must be greater than zero")
|
||||
return parsed
|
||||
|
||||
mode = source.get("sliding_window_mode", "off")
|
||||
if mode not in {"off", "basic", "context"}:
|
||||
raise ValueError("sliding_window_mode must be one of: off, basic, context")
|
||||
config = cls(
|
||||
sliding_window_mode=str(mode),
|
||||
basic_window_high_tokens=integer("basic_window_high_tokens", 8000),
|
||||
basic_window_low_tokens=integer("basic_window_low_tokens", 6000),
|
||||
context_previous_max_tokens=integer("context_previous_max_tokens", 500),
|
||||
context_max_units=integer("context_max_units", 24),
|
||||
)
|
||||
if config.basic_window_low_tokens >= config.basic_window_high_tokens:
|
||||
raise ValueError("basic_window_low_tokens must be less than basic_window_high_tokens")
|
||||
return config
|
||||
|
||||
def as_dict(self) -> dict[str, int | str]:
|
||||
return {
|
||||
"sliding_window_mode": self.sliding_window_mode,
|
||||
"basic_window_high_tokens": self.basic_window_high_tokens,
|
||||
"basic_window_low_tokens": self.basic_window_low_tokens,
|
||||
"context_previous_max_tokens": self.context_previous_max_tokens,
|
||||
"context_max_units": self.context_max_units,
|
||||
}
|
||||
|
||||
|
||||
class MiniCPMO45DuplexPolicy:
|
||||
"""MiniCPM-o 4.5 native duplex model policy.
|
||||
|
||||
|
||||
@@ -27,6 +27,12 @@ _MINICPMO45_OPTIONAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.OPTIONAL_TOKEN_FIELDS
|
||||
_MINICPMO45_PROCESSOR_LOAD_LOCK = Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MiniCPMO45WindowUnit:
|
||||
embeds: list[Any] = field(default_factory=list)
|
||||
token_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MiniCPMO45Stage0SessionState:
|
||||
session_id: str
|
||||
@@ -36,6 +42,14 @@ class _MiniCPMO45Stage0SessionState:
|
||||
audio_chunk_idx: int = 0
|
||||
context_embeds: list[torch.Tensor] = field(default_factory=list)
|
||||
context_token_ids: list[int] = field(default_factory=list)
|
||||
context_prefix_embeds: list[Any] = field(default_factory=list)
|
||||
context_prefix_token_ids: list[int] = field(default_factory=list)
|
||||
context_suffix_embeds: list[Any] = field(default_factory=list)
|
||||
context_suffix_token_ids: list[int] = field(default_factory=list)
|
||||
window_units: list[_MiniCPMO45WindowUnit] = field(default_factory=list)
|
||||
window_enabled: bool = False
|
||||
pending_window_unit: _MiniCPMO45WindowUnit | None = None
|
||||
pending_window_generated_tokens: list[int] = field(default_factory=list)
|
||||
current_turn_ended: bool = True
|
||||
prepared_append_identity: tuple[int | None, int] | None = None
|
||||
prepared_inputs_embeds: torch.Tensor | None = None
|
||||
@@ -142,6 +156,11 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
*,
|
||||
runtime_config: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
window_config = (runtime_config or {}).get("duplex_window_config")
|
||||
state.window_enabled = isinstance(window_config, dict) and window_config.get("sliding_window_mode", "off") in {
|
||||
"basic",
|
||||
"context",
|
||||
}
|
||||
if not self._stage_runtime_ready():
|
||||
return
|
||||
self._require_special_token_ids()
|
||||
@@ -157,17 +176,26 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
(runtime_config or {}).get("initial_user_text"),
|
||||
)
|
||||
for token_id in self._encode_text(prefix):
|
||||
state.context_embeds.append(self._embed_token(token_id))
|
||||
embed = self._embed_token(token_id)
|
||||
state.context_embeds.append(embed)
|
||||
state.context_token_ids.append(token_id)
|
||||
state.context_prefix_embeds.append(embed)
|
||||
state.context_prefix_token_ids.append(token_id)
|
||||
if ref_audio is not None:
|
||||
ref_audio_embeds = self._stage_ref_audio_embeddings(ref_audio, state=state)
|
||||
if ref_audio_embeds is not None:
|
||||
ref_audio_embeds = self._as_2d_tensor(ref_audio_embeds)
|
||||
placeholder_ids = [self.unit_token_id] * int(ref_audio_embeds.shape[0])
|
||||
state.context_embeds.append(ref_audio_embeds)
|
||||
state.context_token_ids.extend([self.unit_token_id] * int(ref_audio_embeds.shape[0]))
|
||||
state.context_token_ids.extend(placeholder_ids)
|
||||
state.context_prefix_embeds.append(ref_audio_embeds)
|
||||
state.context_prefix_token_ids.extend(placeholder_ids)
|
||||
for token_id in self._encode_text(suffix):
|
||||
state.context_embeds.append(self._embed_token(token_id))
|
||||
embed = self._embed_token(token_id)
|
||||
state.context_embeds.append(embed)
|
||||
state.context_token_ids.append(token_id)
|
||||
state.context_suffix_embeds.append(embed)
|
||||
state.context_suffix_token_ids.append(token_id)
|
||||
|
||||
def _stage_prefill_embeddings_only(
|
||||
self,
|
||||
@@ -179,6 +207,7 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
seq: int | None = None,
|
||||
is_speech: bool = False,
|
||||
final: bool = False,
|
||||
stage0_window: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build scheduler-owned Stage0 input embeddings for one audio append.
|
||||
|
||||
@@ -199,6 +228,16 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
result["input_token_ids"] = list(state.prepared_input_token_ids)
|
||||
return result
|
||||
self._require_special_token_ids()
|
||||
if isinstance(stage0_window, dict):
|
||||
completed_ids = stage0_window.get("completed_token_ids")
|
||||
if isinstance(completed_ids, list):
|
||||
state.pending_window_generated_tokens = [int(token_id) for token_id in completed_ids]
|
||||
completed_terminator = stage0_window.get("completed_terminator_token_id")
|
||||
if isinstance(completed_terminator, int):
|
||||
# Async sampling can overwrite the worker's pending token
|
||||
# after the scheduler accepted a segment stop. Rebuild from
|
||||
# the scheduler's canonical boundary, just like its content.
|
||||
state.pending_terminator_token = completed_terminator
|
||||
if audio_waveform is None or len(audio_waveform) == 0:
|
||||
return self._stage_prefill_result(False, start_time, "empty audio")
|
||||
state.audio_buffer = np.concatenate([state.audio_buffer, np.asarray(audio_waveform, dtype=np.float32)])
|
||||
@@ -279,12 +318,25 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
# the closure; the model's listen/speak policy depends on
|
||||
# seeing its own past decisions in context.
|
||||
pending_terminator = state.pending_terminator_token
|
||||
closure_token_ids: list[int] = []
|
||||
if pending_terminator is not None and units_built == 0:
|
||||
state.pending_terminator_token = None
|
||||
embed_parts.append(self._embed_token(pending_terminator))
|
||||
token_ids.append(int(pending_terminator))
|
||||
closure_token_ids.append(int(pending_terminator))
|
||||
embed_parts.append(self._embed_token(self.unit_end_token_id))
|
||||
token_ids.append(self.unit_end_token_id)
|
||||
closure_token_ids.append(self.unit_end_token_id)
|
||||
if units_built == 0:
|
||||
self._finalize_window_unit(state, closure_token_ids)
|
||||
elif state.pending_window_unit is not None:
|
||||
# Internal processor chunks belong to this same append.
|
||||
state.pending_window_unit.embeds.extend(
|
||||
self._embed_token(token_id) for token_id in closure_token_ids
|
||||
)
|
||||
state.pending_window_unit.token_ids.extend(closure_token_ids)
|
||||
unit_embed_start = len(embed_parts)
|
||||
unit_token_start = len(token_ids)
|
||||
embed_parts.append(self._embed_token(self.unit_token_id))
|
||||
token_ids.append(self.unit_token_id)
|
||||
if frame_blocks:
|
||||
@@ -311,6 +363,11 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
state.audio_buffer = state.audio_buffer[consumed_samples:]
|
||||
state.audio_chunk_idx += 1
|
||||
units_built += 1
|
||||
if state.window_enabled:
|
||||
if state.pending_window_unit is None:
|
||||
state.pending_window_unit = _MiniCPMO45WindowUnit()
|
||||
state.pending_window_unit.embeds.extend(embed_parts[unit_embed_start:])
|
||||
state.pending_window_unit.token_ids.extend(token_ids[unit_token_start:])
|
||||
chunk_size = self._streaming_chunk_size(processor)
|
||||
# Match official streaming_prefill: per chunk feed ONLY <unit>+audio. The assistant
|
||||
# turn is opened once at session init; re-emitting the turn-open prefix per chunk
|
||||
@@ -320,6 +377,9 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
|
||||
import torch
|
||||
|
||||
window_result = self._window_replacement_parts(state, stage0_window)
|
||||
if window_result is not None:
|
||||
embed_parts, token_ids = window_result
|
||||
inputs_embeds = torch.cat([self._as_2d_tensor(embed) for embed in embed_parts], dim=0)
|
||||
result = self._stage_prefill_result(True, start_time)
|
||||
result.update(
|
||||
@@ -332,6 +392,7 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
"uses_model_runner_scheduler": True,
|
||||
"runner_kv_backed": True,
|
||||
"runtime_impl": "scheduler_data_plane",
|
||||
"stage0_window_replaced": window_result is not None,
|
||||
}
|
||||
)
|
||||
if is_speech and (append_identity is None or state.pending_speech_append_identity != append_identity):
|
||||
@@ -344,6 +405,85 @@ class MiniCPMO45Stage0DuplexRuntime:
|
||||
state.prepared_result = {k: v for k, v in result.items() if k not in {"inputs_embeds", "input_token_ids"}}
|
||||
return result
|
||||
|
||||
def _finalize_window_unit(
|
||||
self,
|
||||
state: _MiniCPMO45Stage0SessionState,
|
||||
closure_token_ids: list[int],
|
||||
) -> None:
|
||||
pending = state.pending_window_unit
|
||||
if not state.window_enabled or pending is None:
|
||||
state.pending_window_unit = None
|
||||
state.pending_window_generated_tokens.clear()
|
||||
return
|
||||
generated = list(state.pending_window_generated_tokens)
|
||||
token_ids = [*pending.token_ids, *generated, *closure_token_ids]
|
||||
embeds = list(pending.embeds)
|
||||
embeds.extend(self._embed_token(token_id) for token_id in generated)
|
||||
embeds.extend(self._embed_token(token_id) for token_id in closure_token_ids)
|
||||
state.window_units.append(_MiniCPMO45WindowUnit(embeds=embeds, token_ids=token_ids))
|
||||
state.pending_window_unit = None
|
||||
state.pending_window_generated_tokens.clear()
|
||||
|
||||
def _window_replacement_parts(
|
||||
self,
|
||||
state: _MiniCPMO45Stage0SessionState,
|
||||
stage0_window: dict[str, Any] | None,
|
||||
) -> tuple[list[Any], list[int]] | None:
|
||||
if not isinstance(stage0_window, dict) or stage0_window.get("replace") is not True:
|
||||
return None
|
||||
try:
|
||||
drop_units = max(0, int(stage0_window.get("drop_units", 0)))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if drop_units > len(state.window_units):
|
||||
raise RuntimeError(
|
||||
"MiniCPM-o Stage-0 window history is shorter than the scheduler drop plan: "
|
||||
f"drop={drop_units}, completed={len(state.window_units)}"
|
||||
)
|
||||
del state.window_units[:drop_units]
|
||||
|
||||
mode = stage0_window.get("mode")
|
||||
embeds: list[Any] = []
|
||||
token_ids: list[int] = []
|
||||
previous_ids: list[int] = []
|
||||
if mode == "context":
|
||||
embeds.extend(state.context_prefix_embeds)
|
||||
token_ids.extend(state.context_prefix_token_ids)
|
||||
previous = stage0_window.get("previous_token_ids")
|
||||
previous_ids = [int(token_id) for token_id in previous] if isinstance(previous, list) else []
|
||||
if previous_ids:
|
||||
# The plan carries the marker ids the scheduler sized the
|
||||
# `previous` region from. Re-tokenizing here would be a second
|
||||
# source of truth and could disagree with that length.
|
||||
marker = stage0_window.get("previous_marker_token_ids")
|
||||
marker_ids = [int(token_id) for token_id in marker] if isinstance(marker, list) else []
|
||||
embeds.extend(self._embed_token(token_id) for token_id in [*marker_ids, *previous_ids])
|
||||
token_ids.extend(marker_ids)
|
||||
token_ids.extend(previous_ids)
|
||||
embeds.extend(state.context_suffix_embeds)
|
||||
token_ids.extend(state.context_suffix_token_ids)
|
||||
else:
|
||||
embeds.extend(state.context_embeds)
|
||||
token_ids.extend(state.context_token_ids)
|
||||
for unit in state.window_units:
|
||||
embeds.extend(unit.embeds)
|
||||
token_ids.extend(unit.token_ids)
|
||||
if state.pending_window_unit is not None:
|
||||
embeds.extend(state.pending_window_unit.embeds)
|
||||
token_ids.extend(state.pending_window_unit.token_ids)
|
||||
expected = stage0_window.get("replacement_prompt_len")
|
||||
if isinstance(expected, int) and expected != len(token_ids):
|
||||
unit_lengths = [len(unit.token_ids) for unit in state.window_units]
|
||||
pending_len = len(state.pending_window_unit.token_ids) if state.pending_window_unit is not None else 0
|
||||
raise RuntimeError(
|
||||
"MiniCPM-o Stage-0 window rebuild length mismatch: "
|
||||
f"worker={len(token_ids)}, scheduler={expected}, mode={mode}, "
|
||||
f"prefix={len(state.context_prefix_token_ids)}, "
|
||||
f"suffix={len(state.context_suffix_token_ids)}, previous={len(previous_ids)}, "
|
||||
f"units={unit_lengths}, pending={pending_len}, drop={drop_units}"
|
||||
)
|
||||
return embeds, token_ids
|
||||
|
||||
@staticmethod
|
||||
def _stage_prefill_result(success: bool, start_time: float, reason: str = "") -> dict[str, object]:
|
||||
return {
|
||||
|
||||
@@ -371,8 +371,18 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
seq=seq,
|
||||
is_speech=bool(payload.get("is_speech", False)),
|
||||
final=bool(duplex.get("final")),
|
||||
stage0_window=(duplex.get("stage0_window") if isinstance(duplex.get("stage0_window"), dict) else None),
|
||||
)
|
||||
update_result = dict(result)
|
||||
if result.get("stage0_window_replaced") is True:
|
||||
window = duplex.get("stage0_window", {})
|
||||
logger.info(
|
||||
"MiniCPM-o Stage-0 window replaced: mode=%s drop_units=%s tokens=%s seq=%s",
|
||||
window.get("mode"),
|
||||
window.get("drop_units"),
|
||||
result.get("num_input_tokens"),
|
||||
seq,
|
||||
)
|
||||
update_result.pop("inputs_embeds", None)
|
||||
if result.get("success") is not True:
|
||||
embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids)
|
||||
|
||||
Reference in new issue
Block a user