[Bugfix] Add opt-in MiniCPM-o 4.5 Stage-0 sliding window (#7631)
Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de> Signed-off-by: 0z5a <dezhen.lu@uni-tuebingen.de> Co-authored-by: 0z5a <dezhen.lu@uni-tuebingen.de>
This commit is contained in:
1 parent
c11596c637
commit
8bc0c38b90
14 files changed
+1230
-12
No files matched your search
@@ -69,6 +69,39 @@ def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate:
|
||||
)
|
||||
|
||||
|
||||
def _make_minicpm_window_update(
|
||||
*,
|
||||
seq: int,
|
||||
mode: str,
|
||||
high: int = 15,
|
||||
low: int = 10,
|
||||
context_max_units: int = 1,
|
||||
) -> StreamingUpdate:
|
||||
update = _make_update([0] * 8)
|
||||
update.model_intermediate_buffer = {
|
||||
"duplex": {
|
||||
"data_plane": True,
|
||||
"seq": seq,
|
||||
"runtime_config": {
|
||||
"duplex_scheduler_token_id": 0,
|
||||
"duplex_first_append_context_tokens": 3,
|
||||
"duplex_window_prefix_tokens": 2,
|
||||
"duplex_window_suffix_token_ids": [3],
|
||||
"duplex_window_previous_marker_token_ids": [70, 71],
|
||||
"duplex_window_special_token_ids": [99],
|
||||
"duplex_window_config": {
|
||||
"sliding_window_mode": mode,
|
||||
"basic_window_high_tokens": high,
|
||||
"basic_window_low_tokens": low,
|
||||
"context_previous_max_tokens": 4,
|
||||
"context_max_units": context_max_units,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return update
|
||||
|
||||
|
||||
def _make_talker_adapter(
|
||||
*,
|
||||
max_model_len: int = 100,
|
||||
@@ -510,6 +543,225 @@ def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder()
|
||||
assert sched._new_prompt_len_snapshot[session.request_id] == 2
|
||||
|
||||
|
||||
def test_stage0_basic_window_rebuilds_below_low_watermark() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids.clear()
|
||||
session._all_token_ids.extend(session.prompt_token_ids)
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40])
|
||||
session.num_computed_tokens = 10
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == [0] * 9
|
||||
assert session.num_computed_tokens == 0
|
||||
assert update.model_intermediate_buffer["meta"]["replace_streaming_prompt"] is True
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan == {
|
||||
"completed_token_ids": [40],
|
||||
"replace": True,
|
||||
"mode": "basic",
|
||||
"drop_units": 1,
|
||||
"dropped_tokens": 9,
|
||||
"previous_token_ids": [],
|
||||
"previous_marker_token_ids": [70, 71],
|
||||
"replacement_prompt_len": 9,
|
||||
}
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
|
||||
|
||||
def test_stage0_context_window_compacts_dropped_speech() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids.clear()
|
||||
session._all_token_ids.extend(session.prompt_token_ids)
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40])
|
||||
session.num_computed_tokens = 10
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
|
||||
first = _make_minicpm_window_update(seq=2, mode="context")
|
||||
sched._update_request_as_session(session, first)
|
||||
assert session.num_prompt_tokens == 18
|
||||
assert first.model_intermediate_buffer["duplex"]["stage0_window"] == {"completed_token_ids": [40]}
|
||||
|
||||
session.append_output_token_ids([50])
|
||||
session.num_computed_tokens = 19
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
second = _make_minicpm_window_update(seq=3, mode="context")
|
||||
sched._update_request_as_session(session, second)
|
||||
|
||||
assert session.prompt_token_ids == [0] * 21
|
||||
plan = second.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["completed_token_ids"] == [50]
|
||||
assert plan["drop_units"] == 1
|
||||
assert plan["previous_token_ids"] == [40]
|
||||
# The worker embeds this exact marker, not a re-tokenized one, so the
|
||||
# previous region is len(marker) + len(previous).
|
||||
assert plan["previous_marker_token_ids"] == [70, 71]
|
||||
assert plan["replacement_prompt_len"] == 21
|
||||
assert getattr(session, "_minicpmo45_window_previous_len") == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cleared_outputs", [False, True])
|
||||
@pytest.mark.parametrize("in_flight", [0, 1])
|
||||
def test_stage0_window_uses_confirmed_span_and_terminator(cleared_outputs, in_flight) -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids[:] = session.prompt_token_ids
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40, 99])
|
||||
session.num_computed_tokens = 10 + in_flight
|
||||
session.num_output_placeholders = in_flight
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
if cleared_outputs:
|
||||
# A stop without a queued append clears this before the next update.
|
||||
session._output_token_ids.clear()
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["completed_token_ids"] == [40]
|
||||
assert plan["completed_terminator_token_id"] == 99
|
||||
assert plan["replacement_prompt_len"] == 9
|
||||
assert plan["dropped_tokens"] == 9
|
||||
|
||||
|
||||
def test_stage0_window_rebuild_that_overflows_max_model_len_finishes_the_session() -> None:
|
||||
"""A rebuilt window prompt is bounded by the client's window settings, not
|
||||
by the model: a unit carrying camera frames is hundreds of tokens, and a
|
||||
``context`` window adds the ``previous`` region on top. The replacement
|
||||
branch must therefore check ``replacement_prompt_len`` against
|
||||
``max_model_len - sample_room`` and finish the session with
|
||||
``context_length_exceeded``, exactly like a plain extension."""
|
||||
sched = _make_live_session_scheduler(max_model_len=21)
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids[:] = [0] * 9
|
||||
session.num_prompt_tokens = 9
|
||||
session._output_token_ids[:] = [40]
|
||||
session._all_token_ids.append(40)
|
||||
session.num_computed_tokens = 10
|
||||
session.num_output_placeholders = 0
|
||||
_park_session(sched, session)
|
||||
# The parked append state above is the frame before this update: one
|
||||
# confirmed output token, fully computed.
|
||||
session._output_token_ids[:] = [40]
|
||||
session._all_token_ids[:] = [*session.prompt_token_ids, 40]
|
||||
session.num_computed_tokens = 10
|
||||
session.num_in_flight_tokens = 0
|
||||
sched.num_sampled_tokens_per_step = 1
|
||||
# The replacement plan rebuilds 21 tokens (retained context plus the
|
||||
# append) and one step samples one more, so 21 leaves no room.
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 20
|
||||
original_prompt = list(session.prompt_token_ids)
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert update.model_intermediate_buffer["duplex"]["stage0_window"]["replacement_prompt_len"] == 21
|
||||
assert session.prompt_token_ids == original_prompt
|
||||
assert session.num_prompt_tokens == 9
|
||||
assert session.status == RequestStatus.FINISHED_ERROR
|
||||
assert session.request_id not in sched.requests
|
||||
assert sched.finished_req_ids == {session.request_id}
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
client_index, reason = sched._streaming_context_overflow[session.request_id]
|
||||
assert client_index == session.client_index
|
||||
assert reason.startswith("context_length_exceeded: ")
|
||||
assert "21 tokens" in reason and "max_model_len 21" in reason
|
||||
|
||||
engine_core_outputs = _run_idle_step(sched)
|
||||
|
||||
(output,) = engine_core_outputs[session.client_index].outputs
|
||||
assert output.request_id == session.request_id
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
assert output.stop_reason == reason
|
||||
|
||||
|
||||
def test_stage0_window_rebuild_that_leaves_room_to_sample_replaces_the_prompt() -> None:
|
||||
"""One slot above the plan fits: the replacement applies through the normal
|
||||
replacement path."""
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
sched.max_model_len = 25
|
||||
sched.num_sampled_tokens_per_step = 1
|
||||
session = _make_request()
|
||||
session.prompt_token_ids = [0] * 9
|
||||
session._all_token_ids[:] = session.prompt_token_ids
|
||||
session.num_prompt_tokens = 9
|
||||
session.append_output_token_ids([40])
|
||||
session.num_computed_tokens = 10
|
||||
session.num_output_placeholders = 0
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 20
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert update.model_intermediate_buffer["meta"]["replace_streaming_prompt"] is True
|
||||
assert update.model_intermediate_buffer["duplex"]["stage0_window"]["replacement_prompt_len"] == 21
|
||||
assert session.prompt_token_ids == [0] * 21
|
||||
assert session.num_prompt_tokens == 21
|
||||
assert session.num_computed_tokens == 0
|
||||
assert session.status == RequestStatus.WAITING
|
||||
sched._free_request_blocks.assert_called_once_with(session)
|
||||
assert not getattr(sched, "_streaming_context_overflow", {})
|
||||
|
||||
|
||||
def test_stage0_window_open_start_zero_is_not_replaced_by_the_context_reserve() -> None:
|
||||
"""A recorded ``open_start`` of 0 is a legitimate empty context prefix. The
|
||||
fallback must be an explicit ``is None`` check: treating 0 as missing
|
||||
substitutes the context reserve, counts the suffix twice, and the worker's
|
||||
rebuild-length check raises on the first replacement."""
|
||||
session = SimpleNamespace(
|
||||
num_prompt_tokens=9,
|
||||
_all_token_ids=[0] * 9 + [40],
|
||||
num_computed_tokens=10,
|
||||
num_output_placeholders=0,
|
||||
_minicpmo45_window_open_start=0,
|
||||
)
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 21
|
||||
|
||||
assert OmniARScheduler._prepare_minicpmo45_stage0_window(
|
||||
session, update, segment_output_ids=[40], completed_terminator=3
|
||||
)
|
||||
|
||||
# open_start stays 0, so the reported unit spans the whole 12-token
|
||||
# boundary instead of the 9 the context reserve would fold in.
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["dropped_tokens"] == 12
|
||||
assert plan["replacement_prompt_len"] == 19
|
||||
|
||||
|
||||
def test_stage0_window_open_start_falls_back_to_the_context_reserve_when_unset() -> None:
|
||||
session = SimpleNamespace(
|
||||
num_prompt_tokens=9,
|
||||
_all_token_ids=[0] * 9 + [40],
|
||||
num_computed_tokens=10,
|
||||
num_output_placeholders=0,
|
||||
)
|
||||
update = _make_minicpm_window_update(seq=2, mode="basic")
|
||||
update.prompt_token_ids = [0] * 21
|
||||
|
||||
assert OmniARScheduler._prepare_minicpmo45_stage0_window(
|
||||
session, update, segment_output_ids=[40], completed_terminator=3
|
||||
)
|
||||
|
||||
# fallback open_start is preserve_len 3 from duplex_first_append_context_tokens,
|
||||
# so the unit is three tokens shorter.
|
||||
plan = update.model_intermediate_buffer["duplex"]["stage0_window"]
|
||||
assert plan["dropped_tokens"] == 9
|
||||
assert plan["replacement_prompt_len"] == 22
|
||||
|
||||
|
||||
def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None:
|
||||
sched = _make_scheduler(stage_id=1)
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Drive complete native duplex turns while Stage-0 repeatedly rebuilds KV."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.clients.duplex import DuplexClient, EventCollector, read_pcm16_wav, wait_for_condition
|
||||
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
|
||||
|
||||
|
||||
async def run_window_turn(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
mode: str,
|
||||
ref_audio: Path | None = None,
|
||||
repeats: int = 2,
|
||||
video_frames: list[str] | None = None,
|
||||
buffered_flush: bool = False,
|
||||
timeout_s: float = 240.0,
|
||||
) -> dict[str, object]:
|
||||
"""Use aggressive limits to exercise multiple windows in one audio turn."""
|
||||
pcm = read_pcm16_wav(input_wav) * repeats
|
||||
collector = EventCollector()
|
||||
reference = (
|
||||
"data:audio/wav;base64," + base64.b64encode(ref_audio.read_bytes()).decode("ascii")
|
||||
if ref_audio is not None
|
||||
else None
|
||||
)
|
||||
config = create_duplex_session_config(
|
||||
modalities=("text", "audio") if ref_audio is not None else ("text",),
|
||||
ref_audio=reference,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"sliding_window_mode": mode,
|
||||
"context_max_units": 1,
|
||||
"context_previous_max_tokens": 16,
|
||||
"basic_window_high_tokens": 64,
|
||||
"basic_window_low_tokens": 32,
|
||||
},
|
||||
)
|
||||
client = DuplexClient(
|
||||
url, model=model, config=config, reconnect=None, heartbeat_interval_s=None, handshake_timeout_s=timeout_s
|
||||
)
|
||||
async with client:
|
||||
reader = asyncio.create_task(collector.consume(client))
|
||||
collector.add({"type": "session.created", "session": client.session_info})
|
||||
try:
|
||||
frames_sent = await client.stream_pcm(
|
||||
pcm,
|
||||
chunk_ms=2500 if buffered_flush else 200,
|
||||
realtime=not buffered_flush,
|
||||
is_speech=True,
|
||||
video_frames=video_frames,
|
||||
)
|
||||
await client.commit(final=True)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()),
|
||||
timeout_s=timeout_s,
|
||||
label="final input commit",
|
||||
)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("response.done") > 0 or bool(collector.errors()),
|
||||
timeout_s=timeout_s,
|
||||
label="completed windowed response",
|
||||
)
|
||||
# Final execution follows the commit acknowledgement asynchronously.
|
||||
await asyncio.sleep(3.0)
|
||||
await client.close(timeout_s=timeout_s)
|
||||
await asyncio.wait_for(reader, timeout=5.0)
|
||||
finally:
|
||||
if not reader.done():
|
||||
reader.cancel()
|
||||
await asyncio.gather(reader, return_exceptions=True)
|
||||
return {
|
||||
"created": collector.count("session.created"),
|
||||
"committed": collector.count("input_audio_buffer.committed"),
|
||||
"done": collector.count("response.done"),
|
||||
"closed": collector.count("session.closed"),
|
||||
"errors": collector.errors(),
|
||||
"audio_bytes": len(collector.audio_bytes()),
|
||||
"frames_sent": frames_sent,
|
||||
"input_seconds": len(pcm) / 32000,
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Real-model regression for the scheduler-owned Thinker sliding window."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
duplex_camera_frames,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
)
|
||||
from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.advanced_model]
|
||||
|
||||
|
||||
def _assert_complete(result, *, require_audio=False):
|
||||
assert result["created"] == 1
|
||||
assert result["committed"] > 0
|
||||
assert result["done"] > 0
|
||||
assert result["closed"] == 1
|
||||
assert result["errors"] == []
|
||||
if require_audio:
|
||||
assert result["audio_bytes"] > 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_rebuild_and_next_session(omni_server, mode, reference):
|
||||
# Two independent sessions exercise admission after final-unit execution.
|
||||
for _ in range(2):
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
ref_audio=resolve_ref_audio() if reference else None,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=reference)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_continuous_input(omni_server, tmp_path, mode, camera):
|
||||
# 24 repeats cover over two minutes and many canonical KV replacements.
|
||||
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera") if camera else None
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
repeats=24,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
video_frames=frames,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=True)
|
||||
assert isinstance(result["input_seconds"], (int, float)) and result["input_seconds"] > 120
|
||||
if camera:
|
||||
assert isinstance(result["frames_sent"], int) and result["frames_sent"] >= 120
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode", ["off", "basic", "context"])
|
||||
def test_window_buffered_flush(omni_server, mode):
|
||||
# Burst appends cross processor chunk boundaries and leave a final tail.
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
buffered_flush=True,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=True)
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.config import DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex import plugin as module
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _plugin():
|
||||
return module.MiniCPMO45DuplexPlugin(lambda *args: None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_window_configuration_is_prepared_by_plugin(monkeypatch):
|
||||
tokenizer = SimpleNamespace(
|
||||
all_special_ids=[90],
|
||||
unk_token_id=-1,
|
||||
eos_token_id=99,
|
||||
encode=lambda text, add_special_tokens=False: [ord(char) for char in text],
|
||||
convert_tokens_to_ids=lambda token: {"<|listen|>": 91}.get(token, -1),
|
||||
)
|
||||
monkeypatch.setattr(module, "_load_tokenizer", lambda config: tokenizer)
|
||||
config = DuplexSessionConfig(
|
||||
modalities=("text",),
|
||||
extra_body={
|
||||
"sliding_window_mode": "basic",
|
||||
"basic_window_high_tokens": 120,
|
||||
"basic_window_low_tokens": 80,
|
||||
},
|
||||
)
|
||||
runtime = await _plugin().prepare_runtime_config(config, model_config=None)
|
||||
assert runtime["duplex_window_config"]["sliding_window_mode"] == "basic"
|
||||
assert "sliding_window_mode" not in config.extra_body
|
||||
assert runtime["duplex_window_prefix_tokens"] > 0
|
||||
assert runtime["duplex_window_suffix_token_ids"]
|
||||
assert runtime["duplex_window_previous_marker_token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_window_configuration_is_rejected():
|
||||
config = DuplexSessionConfig(
|
||||
modalities=("text",),
|
||||
extra_body={
|
||||
"basic_window_high_tokens": 80,
|
||||
"basic_window_low_tokens": 80,
|
||||
},
|
||||
)
|
||||
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError) as exc:
|
||||
await _plugin().prepare_runtime_config(config, model_config=None)
|
||||
assert exc.value.code == "invalid_sliding_window_config"
|
||||
|
||||
|
||||
def test_window_update_cannot_change_created_configuration():
|
||||
config = DuplexSessionConfig(extra_body={"sliding_window_mode": "basic"})
|
||||
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError) as exc:
|
||||
_plugin().runtime_config_for_update(config, {"duplex_window_config": {"sliding_window_mode": "off"}})
|
||||
assert exc.value.code == "sliding_window_update_unsupported"
|
||||
|
||||
|
||||
def test_window_internal_configuration_is_server_owned():
|
||||
with pytest.raises(module.MiniCPMO45ClientRuntimeConfigError):
|
||||
_plugin().validate_client_extra_body({"duplex_window_config": {}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("final", [False, True])
|
||||
@pytest.mark.parametrize("seq,samples,expected", [(1, 32000, 16), (1, 48000, 28), (2, 16000, 13)])
|
||||
def test_first_and_final_append_reserve_exact_window_input(seq, samples, expected, final):
|
||||
prompt = module.build_duplex_data_plane_prompt(
|
||||
request_id="window-request",
|
||||
fence=DuplexFence("sid", turn_id=1),
|
||||
session_config={},
|
||||
runtime_config={"duplex_first_append_context_tokens": 5},
|
||||
seq=seq,
|
||||
turn_seq=seq,
|
||||
payload={
|
||||
"audio": base64.b64encode(bytes(samples * 4)).decode(),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
},
|
||||
final=final,
|
||||
)
|
||||
assert len(prompt["prompt_token_ids"]) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq", [1, 2, 3])
|
||||
def test_final_exact_chunk_append_reserves_one_unit_not_two(seq):
|
||||
"""A final append whose audio is already an exact number of chunks must
|
||||
reserve exactly what Stage0 feeds: one unit per chunk (``<unit>`` plus its
|
||||
audio) and the closure pair for every unit after the first. Serving pads
|
||||
the final residual itself, so the old extra 12-slot silent-unit
|
||||
reservation only put 12 pad positions ahead of the audio in the KV."""
|
||||
|
||||
def _budget(*, seq, final):
|
||||
prompt = module.build_duplex_data_plane_prompt(
|
||||
request_id="final-request",
|
||||
fence=DuplexFence("sid", turn_id=1),
|
||||
session_config={},
|
||||
runtime_config={"duplex_first_append_context_tokens": 0},
|
||||
seq=seq,
|
||||
turn_seq=seq,
|
||||
payload={
|
||||
"audio": base64.b64encode(bytes(16000 * 4)).decode(),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
},
|
||||
final=final,
|
||||
)
|
||||
return len(prompt["prompt_token_ids"])
|
||||
|
||||
assert _budget(seq=seq, final=True) == _budget(seq=seq, final=False)
|
||||
@@ -142,6 +142,37 @@ def test_native_duplex_speak_segment_reaches_split_talker() -> None:
|
||||
assert info["duplex"]["turn_id"] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize("folded_decisions", [0, 1, 2])
|
||||
@pytest.mark.parametrize("bos_in_prompt", [False, True])
|
||||
def test_native_duplex_tts_bos_aligns_after_window_rebuild(folded_decisions, bos_in_prompt) -> None:
|
||||
prompt_ids = [101] * 10
|
||||
output_ids = [9303] * folded_decisions + [9301, 21, 22, 9308]
|
||||
if bos_in_prompt:
|
||||
prompt_ids[-1] = 9301
|
||||
output_ids = [21, 22, 9308]
|
||||
rows = len(prompt_ids) + len(output_ids) - folded_decisions
|
||||
latent = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4)
|
||||
source = _output(
|
||||
prompt_ids=prompt_ids,
|
||||
output_ids=output_ids,
|
||||
latent=latent,
|
||||
multimodal_output={
|
||||
"duplex_prompt_token_ids": prompt_ids,
|
||||
"meta": {
|
||||
"tts_bos_token_id": 9301,
|
||||
"tts_eos_token_id": 9302,
|
||||
"listen_token_id": 9303,
|
||||
"speak_token_id": 9304,
|
||||
"chunk_eos_token_id": 9308,
|
||||
},
|
||||
},
|
||||
)
|
||||
converted = llm2tts([source], prompt=[{}], _streaming_context=SimpleNamespace(bridge_states={}))[0]
|
||||
info = converted["model_intermediate_buffer"]
|
||||
assert info["ids"]["tts"] == [21, 22]
|
||||
torch.testing.assert_close(torch.as_tensor(info["hidden_states"]["tts"]), latent[-3:-1])
|
||||
|
||||
|
||||
def test_native_duplex_continuation_appends_only_new_talker_condition() -> None:
|
||||
prompt_ids = [101, 102]
|
||||
token_ids = {
|
||||
|
||||
@@ -848,6 +848,176 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
|
||||
assert result["prompt_suffix_len"] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [None, "off", "basic", "context"])
|
||||
def test_minicpmo_stage0_history_collection_starts_with_session_config(mode):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="history-lifecycle")
|
||||
config = {} if mode is None else {"duplex_window_config": {"sliding_window_mode": mode}}
|
||||
runtime._prepare_session_context(state, {}, runtime_config=config)
|
||||
for seq in range(1, 101):
|
||||
runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=seq)
|
||||
enabled = mode in {"basic", "context"}
|
||||
assert state.window_enabled is enabled
|
||||
assert len(state.window_units) == (99 if enabled else 0)
|
||||
assert (state.pending_window_unit is not None) is enabled
|
||||
assert state.pending_window_generated_tokens == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["basic", "context"])
|
||||
@pytest.mark.parametrize("buffered", [False, True])
|
||||
def test_minicpmo_stage0_multi_chunk_append_matches_scheduler_window(mode, buffered):
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="multi-chunk-window", window_enabled=True)
|
||||
if buffered:
|
||||
# Keep the tiny fake processor's initial chunk unpadded to model carry.
|
||||
runtime._pad_first_audio_chunk_if_needed = lambda *args: None
|
||||
assert runtime._stage_prefill_embeddings_only(state, np.zeros(2, dtype=np.float32), seq=0)["success"] is False
|
||||
first = runtime._stage_prefill_embeddings_only(
|
||||
state, np.zeros(6 if buffered else 8, dtype=np.float32), seq=1, final=buffered
|
||||
)
|
||||
assert first["input_token_ids"] == [1, 11, 2, 1, 11]
|
||||
assert state.window_units == []
|
||||
assert state.pending_window_unit.token_ids == first["input_token_ids"]
|
||||
session = SimpleNamespace(num_prompt_tokens=5)
|
||||
replaced = False
|
||||
for seq in (2, 3):
|
||||
info = {
|
||||
"duplex": {
|
||||
"data_plane": True,
|
||||
"seq": seq,
|
||||
"runtime_config": {
|
||||
"duplex_window_config": {
|
||||
"sliding_window_mode": mode,
|
||||
"basic_window_high_tokens": 1,
|
||||
"basic_window_low_tokens": 2,
|
||||
"context_max_units": 1,
|
||||
"context_previous_max_tokens": 16,
|
||||
},
|
||||
"duplex_window_previous_marker_token_ids": [201, 5],
|
||||
},
|
||||
}
|
||||
}
|
||||
update = SimpleNamespace(prompt_token_ids=[3, 2, 1, 11], model_intermediate_buffer=info)
|
||||
planned = OmniARScheduler._prepare_minicpmo45_stage0_window(
|
||||
session, update, segment_output_ids=[44], completed_terminator=3
|
||||
)
|
||||
result = runtime._stage_prefill_embeddings_only(
|
||||
state, np.zeros(4, dtype=np.float32), seq=seq, stage0_window=info["duplex"]["stage0_window"]
|
||||
)
|
||||
assert result["stage0_window_replaced"] is planned
|
||||
if planned:
|
||||
assert result["num_input_tokens"] == len(update.prompt_token_ids)
|
||||
replaced = True
|
||||
break
|
||||
session.num_prompt_tokens += 1 + len(update.prompt_token_ids)
|
||||
assert replaced
|
||||
|
||||
|
||||
def test_minicpmo_stage0_basic_window_rebuilds_from_retained_units():
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-basic-window", window_enabled=True)
|
||||
first = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
|
||||
assert first["input_token_ids"] == [1, 11]
|
||||
state.pending_window_generated_tokens.append(5)
|
||||
state.pending_terminator_token = 3
|
||||
|
||||
rebuilt = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(4, dtype=np.float32),
|
||||
seq=2,
|
||||
stage0_window={
|
||||
"replace": True,
|
||||
"mode": "basic",
|
||||
"drop_units": 1,
|
||||
"replacement_prompt_len": 2,
|
||||
},
|
||||
)
|
||||
|
||||
assert rebuilt["stage0_window_replaced"] is True
|
||||
assert rebuilt["input_token_ids"] == [1, 11]
|
||||
assert state.window_units == []
|
||||
|
||||
|
||||
def test_minicpmo_stage0_context_window_inserts_previous_before_suffix():
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-context-window", window_enabled=True)
|
||||
prefix = runtime._embed_token(200)
|
||||
suffix = runtime._embed_token(202)
|
||||
state.context_embeds = [prefix, suffix]
|
||||
state.context_token_ids = [200, 202]
|
||||
state.context_prefix_embeds = [prefix]
|
||||
state.context_prefix_token_ids = [200]
|
||||
state.context_suffix_embeds = [suffix]
|
||||
state.context_suffix_token_ids = [202]
|
||||
first = runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
|
||||
assert first["input_token_ids"] == [200, 202, 1, 11]
|
||||
state.pending_window_generated_tokens.append(42)
|
||||
state.pending_terminator_token = 3
|
||||
|
||||
rebuilt = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(4, dtype=np.float32),
|
||||
seq=2,
|
||||
stage0_window={
|
||||
"replace": True,
|
||||
"mode": "context",
|
||||
"drop_units": 1,
|
||||
"previous_token_ids": [42],
|
||||
"previous_marker_token_ids": [201, 5],
|
||||
"replacement_prompt_len": 7,
|
||||
},
|
||||
)
|
||||
|
||||
assert rebuilt["input_token_ids"] == [200, 201, 5, 42, 202, 1, 11]
|
||||
assert rebuilt["num_input_tokens"] == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pending_terminator", [None, 3, 99])
|
||||
def test_minicpmo_stage0_window_uses_accepted_output_not_async_sampler_history(pending_terminator):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
runtime._stage_audio_embeddings = lambda *args, **kwargs: torch.zeros((10, 2))
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-accepted-window", window_enabled=True)
|
||||
state.context_prefix_token_ids = [200] * 7
|
||||
state.context_suffix_token_ids = [202]
|
||||
state.context_token_ids = [*state.context_prefix_token_ids, 202]
|
||||
state.context_prefix_embeds = [runtime._embed_token(token_id) for token_id in state.context_prefix_token_ids]
|
||||
state.context_suffix_embeds = [runtime._embed_token(202)]
|
||||
state.context_embeds = [*state.context_prefix_embeds, *state.context_suffix_embeds]
|
||||
runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)
|
||||
for seq in (2, 3):
|
||||
state.pending_window_generated_tokens = [3, 40, 41]
|
||||
state.pending_terminator_token = pending_terminator
|
||||
plan = {"completed_token_ids": [], "completed_terminator_token_id": 3}
|
||||
if seq == 3:
|
||||
plan.update(replace=True, mode="context", drop_units=1, replacement_prompt_len=32)
|
||||
result = runtime._stage_prefill_embeddings_only(
|
||||
state, np.zeros(4, dtype=np.float32), seq=seq, stage0_window=plan
|
||||
)
|
||||
|
||||
assert result["num_input_tokens"] == 32
|
||||
assert [len(unit.token_ids) for unit in state.window_units] == [13]
|
||||
assert len(state.pending_window_unit.token_ids) == 11
|
||||
assert 40 not in result["input_token_ids"]
|
||||
assert 41 not in result["input_token_ids"]
|
||||
|
||||
|
||||
def _stage0_vision_runtime():
|
||||
import torch
|
||||
|
||||
|
||||
Reference in new issue
Block a user