[Bugfix] Add opt-in MiniCPM-o 4.5 Stage-0 sliding window (#7631)

Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de>
Signed-off-by: 0z5a <dezhen.lu@uni-tuebingen.de>
Co-authored-by: 0z5a <dezhen.lu@uni-tuebingen.de>
This commit is contained in:
0z5aand0z5a authored and GitHub committed 2026-09-21 18:34:22 -07:00
1 parent c11596c637
commit 8bc0c38b90
14 files changed
+1230 -12

No files matched your search

@@ -45,7 +45,10 @@ from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import (
MiniCPMO45DataPlaneContext,
MiniCPMO45DataPlaneSession,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import MiniCPMO45DuplexPolicy
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
MiniCPMO45DuplexPolicy,
MiniCPMO45DuplexWindowConfig,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import (
MiniCPMO45ServingSessionState,
)
@@ -85,6 +88,11 @@ PRIVATE_RUNTIME_CONFIG_KEYS = frozenset(
"ref_audio_format",
"ref_audio_sample_rate_hz",
"initial_user_text",
"duplex_window_config",
"duplex_window_prefix_tokens",
"duplex_window_suffix_token_ids",
"duplex_window_previous_marker_token_ids",
"duplex_window_special_token_ids",
}
)
@@ -245,8 +253,8 @@ def build_duplex_data_plane_prompt(
token_budget = context_reserve + first_units * 12 - 1 + vision_tokens
if seq > 1 and duplex_payload_is_exact_chunks(payload):
token_budget += 1
if final and duplex_payload_is_exact_chunks(payload):
token_budget += 12
# Serving already pads the final residual audio. Stage0 does not
# append another silent unit, so final must not reserve extra slots.
extra_body = session_config.get("extra_body")
raw_token_id = runtime_config.get("duplex_scheduler_token_id")
try:
@@ -515,6 +523,16 @@ def _apply_first_append_context_tokens(
return
ref_tokens = MiniCPMO45DuplexPolicy.audio_token_count(ref_sample_count or 0)
runtime_config["duplex_first_append_context_tokens"] = len(prefix_ids) + ref_tokens + len(suffix_ids)
runtime_config["duplex_window_prefix_tokens"] = len(prefix_ids) + ref_tokens
runtime_config["duplex_window_suffix_token_ids"] = [int(token_id) for token_id in suffix_ids]
marker_ids = tokenizer.encode("\n\nprevious: ", add_special_tokens=False)
runtime_config["duplex_window_previous_marker_token_ids"] = [int(token_id) for token_id in marker_ids]
runtime_config["duplex_window_special_token_ids"] = sorted(
MiniCPMO45DuplexPolicy.native_special_token_ids(
MiniCPMO45DuplexPolicy.token_ids_from_tokenizer(tokenizer),
tokenizer_special_ids=list(getattr(tokenizer, "all_special_ids", ()) or ()),
)
)
def _model_vision_tile_pixels(model_config: ModelConfig | None) -> int | None:
@@ -722,6 +740,8 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
code="unsupported_ref_audio_path",
)
runtime_config: dict[str, object] = {"instructions": config.instructions}
window_config = self._pop_window_config(extra_body) or MiniCPMO45DuplexWindowConfig()
runtime_config["duplex_window_config"] = window_config.as_dict()
# ``duplex_initial_user_text`` is the older extra_body spelling and
# still works; the session field is the framework-level one.
initial_user_text = extra_body.pop("duplex_initial_user_text", None)
@@ -788,6 +808,17 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
current: Mapping[str, object],
) -> dict[str, object]:
runtime_config = deepcopy(dict(current))
extra_body = dict(config.extra_body)
requested_window = self._pop_window_config(extra_body)
if requested_window is not None:
reject_changed_runtime_value(
requested_window.as_dict(),
runtime_config.get("duplex_window_config"),
message="sliding-window configuration cannot be changed after the session is created",
code="sliding_window_update_unsupported",
error_cls=MiniCPMO45ClientRuntimeConfigError,
)
config.extra_body = extra_body
reject_changed_runtime_value(
config.instructions,
runtime_config.get("instructions"),
@@ -812,6 +843,23 @@ class MiniCPMO45DuplexPlugin(DuplexModelPlugin):
runtime_config["duplex_stage_sampling_params"] = stage_sampling
return runtime_config
@staticmethod
def _pop_window_config(extra_body: dict[str, object]) -> MiniCPMO45DuplexWindowConfig | None:
names = (
"sliding_window_mode",
"basic_window_high_tokens",
"basic_window_low_tokens",
"context_previous_max_tokens",
"context_max_units",
)
provided = {name: extra_body.pop(name) for name in names if name in extra_body}
if not provided:
return None
try:
return MiniCPMO45DuplexWindowConfig.from_mapping(provided)
except ValueError as exc:
raise MiniCPMO45ClientRuntimeConfigError(str(exc), code="invalid_sliding_window_config") from exc
def data_plane_context(
self,
*,
@@ -3,12 +3,63 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from transformers import PreTrainedTokenizerBase
@dataclass(frozen=True, slots=True)
class MiniCPMO45DuplexWindowConfig:
"""Stage-0 window settings matching the released checkpoint defaults."""
sliding_window_mode: str = "off"
basic_window_high_tokens: int = 8000
basic_window_low_tokens: int = 6000
context_previous_max_tokens: int = 500
context_max_units: int = 24
@classmethod
def from_mapping(cls, value: object) -> MiniCPMO45DuplexWindowConfig:
source = value if isinstance(value, dict) else {}
def integer(name: str, default: int) -> int:
raw = source.get(name, default)
if isinstance(raw, bool):
raise ValueError(f"{name} must be an integer")
try:
parsed = int(raw)
except (TypeError, ValueError) as exc:
raise ValueError(f"{name} must be an integer") from exc
if parsed <= 0:
raise ValueError(f"{name} must be greater than zero")
return parsed
mode = source.get("sliding_window_mode", "off")
if mode not in {"off", "basic", "context"}:
raise ValueError("sliding_window_mode must be one of: off, basic, context")
config = cls(
sliding_window_mode=str(mode),
basic_window_high_tokens=integer("basic_window_high_tokens", 8000),
basic_window_low_tokens=integer("basic_window_low_tokens", 6000),
context_previous_max_tokens=integer("context_previous_max_tokens", 500),
context_max_units=integer("context_max_units", 24),
)
if config.basic_window_low_tokens >= config.basic_window_high_tokens:
raise ValueError("basic_window_low_tokens must be less than basic_window_high_tokens")
return config
def as_dict(self) -> dict[str, int | str]:
return {
"sliding_window_mode": self.sliding_window_mode,
"basic_window_high_tokens": self.basic_window_high_tokens,
"basic_window_low_tokens": self.basic_window_low_tokens,
"context_previous_max_tokens": self.context_previous_max_tokens,
"context_max_units": self.context_max_units,
}
class MiniCPMO45DuplexPolicy:
"""MiniCPM-o 4.5 native duplex model policy.
@@ -27,6 +27,12 @@ _MINICPMO45_OPTIONAL_TOKEN_FIELDS = MiniCPMO45DuplexPolicy.OPTIONAL_TOKEN_FIELDS
_MINICPMO45_PROCESSOR_LOAD_LOCK = Lock()
@dataclass
class _MiniCPMO45WindowUnit:
embeds: list[Any] = field(default_factory=list)
token_ids: list[int] = field(default_factory=list)
@dataclass
class _MiniCPMO45Stage0SessionState:
session_id: str
@@ -36,6 +42,14 @@ class _MiniCPMO45Stage0SessionState:
audio_chunk_idx: int = 0
context_embeds: list[torch.Tensor] = field(default_factory=list)
context_token_ids: list[int] = field(default_factory=list)
context_prefix_embeds: list[Any] = field(default_factory=list)
context_prefix_token_ids: list[int] = field(default_factory=list)
context_suffix_embeds: list[Any] = field(default_factory=list)
context_suffix_token_ids: list[int] = field(default_factory=list)
window_units: list[_MiniCPMO45WindowUnit] = field(default_factory=list)
window_enabled: bool = False
pending_window_unit: _MiniCPMO45WindowUnit | None = None
pending_window_generated_tokens: list[int] = field(default_factory=list)
current_turn_ended: bool = True
prepared_append_identity: tuple[int | None, int] | None = None
prepared_inputs_embeds: torch.Tensor | None = None
@@ -142,6 +156,11 @@ class MiniCPMO45Stage0DuplexRuntime:
*,
runtime_config: dict[str, object] | None = None,
) -> None:
window_config = (runtime_config or {}).get("duplex_window_config")
state.window_enabled = isinstance(window_config, dict) and window_config.get("sliding_window_mode", "off") in {
"basic",
"context",
}
if not self._stage_runtime_ready():
return
self._require_special_token_ids()
@@ -157,17 +176,26 @@ class MiniCPMO45Stage0DuplexRuntime:
(runtime_config or {}).get("initial_user_text"),
)
for token_id in self._encode_text(prefix):
state.context_embeds.append(self._embed_token(token_id))
embed = self._embed_token(token_id)
state.context_embeds.append(embed)
state.context_token_ids.append(token_id)
state.context_prefix_embeds.append(embed)
state.context_prefix_token_ids.append(token_id)
if ref_audio is not None:
ref_audio_embeds = self._stage_ref_audio_embeddings(ref_audio, state=state)
if ref_audio_embeds is not None:
ref_audio_embeds = self._as_2d_tensor(ref_audio_embeds)
placeholder_ids = [self.unit_token_id] * int(ref_audio_embeds.shape[0])
state.context_embeds.append(ref_audio_embeds)
state.context_token_ids.extend([self.unit_token_id] * int(ref_audio_embeds.shape[0]))
state.context_token_ids.extend(placeholder_ids)
state.context_prefix_embeds.append(ref_audio_embeds)
state.context_prefix_token_ids.extend(placeholder_ids)
for token_id in self._encode_text(suffix):
state.context_embeds.append(self._embed_token(token_id))
embed = self._embed_token(token_id)
state.context_embeds.append(embed)
state.context_token_ids.append(token_id)
state.context_suffix_embeds.append(embed)
state.context_suffix_token_ids.append(token_id)
def _stage_prefill_embeddings_only(
self,
@@ -179,6 +207,7 @@ class MiniCPMO45Stage0DuplexRuntime:
seq: int | None = None,
is_speech: bool = False,
final: bool = False,
stage0_window: dict[str, object] | None = None,
) -> dict[str, object]:
"""Build scheduler-owned Stage0 input embeddings for one audio append.
@@ -199,6 +228,16 @@ class MiniCPMO45Stage0DuplexRuntime:
result["input_token_ids"] = list(state.prepared_input_token_ids)
return result
self._require_special_token_ids()
if isinstance(stage0_window, dict):
completed_ids = stage0_window.get("completed_token_ids")
if isinstance(completed_ids, list):
state.pending_window_generated_tokens = [int(token_id) for token_id in completed_ids]
completed_terminator = stage0_window.get("completed_terminator_token_id")
if isinstance(completed_terminator, int):
# Async sampling can overwrite the worker's pending token
# after the scheduler accepted a segment stop. Rebuild from
# the scheduler's canonical boundary, just like its content.
state.pending_terminator_token = completed_terminator
if audio_waveform is None or len(audio_waveform) == 0:
return self._stage_prefill_result(False, start_time, "empty audio")
state.audio_buffer = np.concatenate([state.audio_buffer, np.asarray(audio_waveform, dtype=np.float32)])
@@ -279,12 +318,25 @@ class MiniCPMO45Stage0DuplexRuntime:
# the closure; the model's listen/speak policy depends on
# seeing its own past decisions in context.
pending_terminator = state.pending_terminator_token
closure_token_ids: list[int] = []
if pending_terminator is not None and units_built == 0:
state.pending_terminator_token = None
embed_parts.append(self._embed_token(pending_terminator))
token_ids.append(int(pending_terminator))
closure_token_ids.append(int(pending_terminator))
embed_parts.append(self._embed_token(self.unit_end_token_id))
token_ids.append(self.unit_end_token_id)
closure_token_ids.append(self.unit_end_token_id)
if units_built == 0:
self._finalize_window_unit(state, closure_token_ids)
elif state.pending_window_unit is not None:
# Internal processor chunks belong to this same append.
state.pending_window_unit.embeds.extend(
self._embed_token(token_id) for token_id in closure_token_ids
)
state.pending_window_unit.token_ids.extend(closure_token_ids)
unit_embed_start = len(embed_parts)
unit_token_start = len(token_ids)
embed_parts.append(self._embed_token(self.unit_token_id))
token_ids.append(self.unit_token_id)
if frame_blocks:
@@ -311,6 +363,11 @@ class MiniCPMO45Stage0DuplexRuntime:
state.audio_buffer = state.audio_buffer[consumed_samples:]
state.audio_chunk_idx += 1
units_built += 1
if state.window_enabled:
if state.pending_window_unit is None:
state.pending_window_unit = _MiniCPMO45WindowUnit()
state.pending_window_unit.embeds.extend(embed_parts[unit_embed_start:])
state.pending_window_unit.token_ids.extend(token_ids[unit_token_start:])
chunk_size = self._streaming_chunk_size(processor)
# Match official streaming_prefill: per chunk feed ONLY <unit>+audio. The assistant
# turn is opened once at session init; re-emitting the turn-open prefix per chunk
@@ -320,6 +377,9 @@ class MiniCPMO45Stage0DuplexRuntime:
import torch
window_result = self._window_replacement_parts(state, stage0_window)
if window_result is not None:
embed_parts, token_ids = window_result
inputs_embeds = torch.cat([self._as_2d_tensor(embed) for embed in embed_parts], dim=0)
result = self._stage_prefill_result(True, start_time)
result.update(
@@ -332,6 +392,7 @@ class MiniCPMO45Stage0DuplexRuntime:
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
"stage0_window_replaced": window_result is not None,
}
)
if is_speech and (append_identity is None or state.pending_speech_append_identity != append_identity):
@@ -344,6 +405,85 @@ class MiniCPMO45Stage0DuplexRuntime:
state.prepared_result = {k: v for k, v in result.items() if k not in {"inputs_embeds", "input_token_ids"}}
return result
def _finalize_window_unit(
self,
state: _MiniCPMO45Stage0SessionState,
closure_token_ids: list[int],
) -> None:
pending = state.pending_window_unit
if not state.window_enabled or pending is None:
state.pending_window_unit = None
state.pending_window_generated_tokens.clear()
return
generated = list(state.pending_window_generated_tokens)
token_ids = [*pending.token_ids, *generated, *closure_token_ids]
embeds = list(pending.embeds)
embeds.extend(self._embed_token(token_id) for token_id in generated)
embeds.extend(self._embed_token(token_id) for token_id in closure_token_ids)
state.window_units.append(_MiniCPMO45WindowUnit(embeds=embeds, token_ids=token_ids))
state.pending_window_unit = None
state.pending_window_generated_tokens.clear()
def _window_replacement_parts(
self,
state: _MiniCPMO45Stage0SessionState,
stage0_window: dict[str, Any] | None,
) -> tuple[list[Any], list[int]] | None:
if not isinstance(stage0_window, dict) or stage0_window.get("replace") is not True:
return None
try:
drop_units = max(0, int(stage0_window.get("drop_units", 0)))
except (TypeError, ValueError):
return None
if drop_units > len(state.window_units):
raise RuntimeError(
"MiniCPM-o Stage-0 window history is shorter than the scheduler drop plan: "
f"drop={drop_units}, completed={len(state.window_units)}"
)
del state.window_units[:drop_units]
mode = stage0_window.get("mode")
embeds: list[Any] = []
token_ids: list[int] = []
previous_ids: list[int] = []
if mode == "context":
embeds.extend(state.context_prefix_embeds)
token_ids.extend(state.context_prefix_token_ids)
previous = stage0_window.get("previous_token_ids")
previous_ids = [int(token_id) for token_id in previous] if isinstance(previous, list) else []
if previous_ids:
# The plan carries the marker ids the scheduler sized the
# `previous` region from. Re-tokenizing here would be a second
# source of truth and could disagree with that length.
marker = stage0_window.get("previous_marker_token_ids")
marker_ids = [int(token_id) for token_id in marker] if isinstance(marker, list) else []
embeds.extend(self._embed_token(token_id) for token_id in [*marker_ids, *previous_ids])
token_ids.extend(marker_ids)
token_ids.extend(previous_ids)
embeds.extend(state.context_suffix_embeds)
token_ids.extend(state.context_suffix_token_ids)
else:
embeds.extend(state.context_embeds)
token_ids.extend(state.context_token_ids)
for unit in state.window_units:
embeds.extend(unit.embeds)
token_ids.extend(unit.token_ids)
if state.pending_window_unit is not None:
embeds.extend(state.pending_window_unit.embeds)
token_ids.extend(state.pending_window_unit.token_ids)
expected = stage0_window.get("replacement_prompt_len")
if isinstance(expected, int) and expected != len(token_ids):
unit_lengths = [len(unit.token_ids) for unit in state.window_units]
pending_len = len(state.pending_window_unit.token_ids) if state.pending_window_unit is not None else 0
raise RuntimeError(
"MiniCPM-o Stage-0 window rebuild length mismatch: "
f"worker={len(token_ids)}, scheduler={expected}, mode={mode}, "
f"prefix={len(state.context_prefix_token_ids)}, "
f"suffix={len(state.context_suffix_token_ids)}, previous={len(previous_ids)}, "
f"units={unit_lengths}, pending={pending_len}, drop={drop_units}"
)
return embeds, token_ids
@staticmethod
def _stage_prefill_result(success: bool, start_time: float, reason: str = "") -> dict[str, object]:
return {
@@ -371,8 +371,18 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
seq=seq,
is_speech=bool(payload.get("is_speech", False)),
final=bool(duplex.get("final")),
stage0_window=(duplex.get("stage0_window") if isinstance(duplex.get("stage0_window"), dict) else None),
)
update_result = dict(result)
if result.get("stage0_window_replaced") is True:
window = duplex.get("stage0_window", {})
logger.info(
"MiniCPM-o Stage-0 window replaced: mode=%s drop_units=%s tokens=%s seq=%s",
window.get("mode"),
window.get("drop_units"),
result.get("num_input_tokens"),
seq,
)
update_result.pop("inputs_embeds", None)
if result.get("success") is not True:
embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids)