[Bugfix] Add opt-in MiniCPM-o 4.5 Stage-0 sliding window (#7631)

Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de>
Signed-off-by: 0z5a <dezhen.lu@uni-tuebingen.de>
Co-authored-by: 0z5a <dezhen.lu@uni-tuebingen.de>
This commit is contained in:
0z5aand0z5a authored and GitHub committed 2026-09-21 18:34:22 -07:00
1 parent c11596c637
commit 8bc0c38b90
14 files changed
+1230 -12

No files matched your search

@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Drive complete native duplex turns while Stage-0 repeatedly rebuilds KV."""
from __future__ import annotations
import asyncio
import base64
from pathlib import Path
from vllm_omni.clients.duplex import DuplexClient, EventCollector, read_pcm16_wav, wait_for_condition
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
async def run_window_turn(
*,
url: str,
model: str,
input_wav: Path,
mode: str,
ref_audio: Path | None = None,
repeats: int = 2,
video_frames: list[str] | None = None,
buffered_flush: bool = False,
timeout_s: float = 240.0,
) -> dict[str, object]:
"""Use aggressive limits to exercise multiple windows in one audio turn."""
pcm = read_pcm16_wav(input_wav) * repeats
collector = EventCollector()
reference = (
"data:audio/wav;base64," + base64.b64encode(ref_audio.read_bytes()).decode("ascii")
if ref_audio is not None
else None
)
config = create_duplex_session_config(
modalities=("text", "audio") if ref_audio is not None else ("text",),
ref_audio=reference,
temperature=0.0,
extra_body={
"sliding_window_mode": mode,
"context_max_units": 1,
"context_previous_max_tokens": 16,
"basic_window_high_tokens": 64,
"basic_window_low_tokens": 32,
},
)
client = DuplexClient(
url, model=model, config=config, reconnect=None, heartbeat_interval_s=None, handshake_timeout_s=timeout_s
)
async with client:
reader = asyncio.create_task(collector.consume(client))
collector.add({"type": "session.created", "session": client.session_info})
try:
frames_sent = await client.stream_pcm(
pcm,
chunk_ms=2500 if buffered_flush else 200,
realtime=not buffered_flush,
is_speech=True,
video_frames=video_frames,
)
await client.commit(final=True)
await wait_for_condition(
lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()),
timeout_s=timeout_s,
label="final input commit",
)
await wait_for_condition(
lambda: collector.count("response.done") > 0 or bool(collector.errors()),
timeout_s=timeout_s,
label="completed windowed response",
)
# Final execution follows the commit acknowledgement asynchronously.
await asyncio.sleep(3.0)
await client.close(timeout_s=timeout_s)
await asyncio.wait_for(reader, timeout=5.0)
finally:
if not reader.done():
reader.cancel()
await asyncio.gather(reader, return_exceptions=True)
return {
"created": collector.count("session.created"),
"committed": collector.count("input_audio_buffer.committed"),
"done": collector.count("response.done"),
"closed": collector.count("session.closed"),
"errors": collector.errors(),
"audio_bytes": len(collector.audio_bytes()),
"frames_sent": frames_sent,
"input_seconds": len(pcm) / 32000,
}
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Real-model regression for the scheduler-owned Thinker sliding window."""
import asyncio
import pytest
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
SERVER_PARAMS,
duplex_camera_frames,
realtime_url,
resolve_ref_audio,
validated_input_wav,
)
from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn
from tests.helpers.mark import hardware_test
pytestmark = [pytest.mark.omni, pytest.mark.advanced_model]
def _assert_complete(result, *, require_audio=False):
assert result["created"] == 1
assert result["committed"] > 0
assert result["done"] > 0
assert result["closed"] == 1
assert result["errors"] == []
if require_audio:
assert result["audio_bytes"] > 0
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
@pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)])
def test_window_rebuild_and_next_session(omni_server, mode, reference):
# Two independent sessions exercise admission after final-unit execution.
for _ in range(2):
result = asyncio.run(
run_window_turn(
url=realtime_url(omni_server),
model=omni_server.model,
input_wav=validated_input_wav(),
mode=mode,
ref_audio=resolve_ref_audio() if reference else None,
)
)
_assert_complete(result, require_audio=reference)
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
@pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)])
def test_window_continuous_input(omni_server, tmp_path, mode, camera):
# 24 repeats cover over two minutes and many canonical KV replacements.
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera") if camera else None
result = asyncio.run(
run_window_turn(
url=realtime_url(omni_server),
model=omni_server.model,
input_wav=validated_input_wav(),
mode=mode,
repeats=24,
ref_audio=resolve_ref_audio(),
video_frames=frames,
)
)
_assert_complete(result, require_audio=True)
assert isinstance(result["input_seconds"], (int, float)) and result["input_seconds"] > 120
if camera:
assert isinstance(result["frames_sent"], int) and result["frames_sent"] >= 120
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
@pytest.mark.parametrize("mode", ["off", "basic", "context"])
def test_window_buffered_flush(omni_server, mode):
# Burst appends cross processor chunk boundaries and leave a final tail.
result = asyncio.run(
run_window_turn(
url=realtime_url(omni_server),
model=omni_server.model,
input_wav=validated_input_wav(),
mode=mode,
ref_audio=resolve_ref_audio(),
buffered_flush=True,
)
)
_assert_complete(result, require_audio=True)