[Bugfix][MiniCPM-o] Bind omni-duplex video frames to the unit they close (#6404)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-08-28 16:58:20 +08:00
1 parent 13173d6bbb
commit da94c3ebe1
17 files changed
+1530 -144

No files matched your search

@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
@@ -27,6 +30,8 @@ _MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
if _MODULE_NAME not in sys.modules:
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
if _spec is None or _spec.loader is None:
raise ImportError(f"Cannot load {_MODULE_NAME} from {_MODULE_PATH}")
_mod = importlib.util.module_from_spec(_spec)
sys.modules[_MODULE_NAME] = _mod
_spec.loader.exec_module(_mod)
@@ -87,7 +92,9 @@ def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) ->
def fake_extract(video_path, *, audio_path, include_audio, **_kw):
counter.append(1)
frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
segments: list[np.ndarray] = (
[np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
)
return frames, segments
monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
@@ -18,6 +18,10 @@ from vllm_omni.experimental.fullduplex.client import (
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
MiniCPMO45DuplexPolicy,
)
from vllm_omni.experimental.fullduplex.video_stacking import (
concat_frames,
unit_subframe_offsets,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -418,3 +422,62 @@ def test_realtime_client_pcm16_wav_round_trip(tmp_path):
assert wav_file.getnchannels() == 1
assert wav_file.getframerate() == 16_000
assert read_pcm16_wav(path) == pcm16
def _stack_test_image(color, size=(40, 30)):
from PIL import Image
return Image.new("RGB", size, color)
def test_video_stacking_tiles_a_units_subframes_into_one_image():
frames = [_stack_test_image((255, 0, 0)), _stack_test_image((0, 255, 0))]
composite = concat_frames(frames)
# Landscape cells stack into a column because that canvas is squarer, and
# the interior seam carries the separator band that lets the model tell the
# sub-frames apart.
assert composite.size == (40, 30 * 2 + 6)
assert composite.getpixel((20, 30 + 3)) == (0, 0, 0)
def test_video_stacking_picks_the_squarest_grid():
portrait = [_stack_test_image((255, 0, 0), size=(20, 60)) for _ in range(2)]
four = [_stack_test_image((0, 0, 255)) for _ in range(4)]
# A portrait clip tiles side by side, four frames always tile 2x2.
assert concat_frames(portrait).size == (20 * 2 + 6, 60)
assert concat_frames(four).size == (40 * 2 + 6, 30 * 2 + 6)
def test_video_stacking_skips_the_subframe_that_duplicates_the_base_frame():
# Official samples stack_frames=5 as 0.2/0.4/0.6/0.8 s: offset 0 is the base
# frame, already sent as frame_list[0].
assert unit_subframe_offsets(5) == pytest.approx([0.2, 0.4, 0.6, 0.8])
assert unit_subframe_offsets(1) == []
def test_realtime_client_sends_each_units_composite_beside_its_base_frame():
sent: list[dict] = []
class _Client(RealtimeDuplexClient):
def __init__(self):
pass
async def send(self, event):
sent.append(event)
asyncio.run(
_Client().stream_pcm16(
b"\x01\x00" * (16_000 * 3),
chunk_ms=200,
realtime=False,
video_frames=["f0", "f1"],
stacked_video_frames=["s0", None],
)
)
# A composite belongs to the unit it was captured in, so it rides the same
# append as that unit's base frame; a unit without one sends the base alone.
assert [event["video_frames"] for event in sent if "video_frames" in event] == [["f0", "s0"], ["f1"]]
@@ -91,6 +91,19 @@ def resolve_ref_audio() -> Path:
return ref_audio
def duplex_camera_frames(*, seconds: int, cache_dir: Path) -> list[str]:
"""Base64 JPEG frames of a moving synthetic scene, one per second.
Mirrors what the realtime web client captures (1 fps, no client-side
resize: the server normalizes at ``scale_resolution=448``).
"""
from tests.e2e.online_serving.helpers.minicpmo_realtime_duplex_scenarios import _video_frames_from_file
from tests.helpers.media import generate_synthetic_video
video = generate_synthetic_video(448, 448, 30 * seconds, cache_dir=cache_dir)
return _video_frames_from_file(Path(video["file_path"]))
def realtime_url(omni_server) -> str:
return f"ws://{omni_server.host}:{omni_server.port}/v1/realtime?duplex=1"
@@ -113,6 +126,8 @@ def demo_args(
output_audio_format="pcm16",
chunk_ms=200,
realtime_input=True,
input_video=None,
stack_frames=1,
first_turn_ms=1400,
turn_duration_ms=[],
first_turn_transcript="duplex CI speech",
@@ -37,8 +37,10 @@ import json
import sys
import time
import wave
from collections.abc import Sequence
from dataclasses import dataclass, field
from pathlib import Path
from types import SimpleNamespace
_REPO_ROOT = Path(__file__).resolve().parents[4]
if str(_REPO_ROOT) not in sys.path:
@@ -55,15 +57,100 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
PCM16_SAMPLE_RATE,
RealtimeEventCollector,
build_realtime_url,
duplex_unit_boundary_ms,
read_pcm16_wav,
summarize_session_request_metrics,
)
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
reference_audio_data_url as _ref_audio_data_url,
)
from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402
concat_frames_b64,
unit_subframe_offsets,
)
_url_with_model = build_realtime_url
_read_wav_pcm16 = read_pcm16_wav
DemoArgs = argparse.Namespace | SimpleNamespace
def _video_frames_from_file(
path: Path,
*,
fps: float = 1.0,
jpeg_quality: int = 70,
stack_frames: int = 1,
) -> tuple[list[str], list[str | None]]:
"""Decode ``path`` into base64 JPEG frames at the omni duplex cadence.
One frame per second matches what the realtime web client captures, so the
stream stays representative of a live camera. ``stack_frames > 1`` also
samples each unit's interior sub-frames and tiles them into one composite,
the official way to raise visual refresh rate without changing the audio
cadence. Returns ``(base_frames, stacked_frames)`` with the tracks parallel.
"""
import cv2
offsets = unit_subframe_offsets(stack_frames)
capture = cv2.VideoCapture(str(path))
if not capture.isOpened():
raise ValueError(f"cannot open video: {path}")
def encode(frame_bgr, index: int) -> str:
ok, jpeg = cv2.imencode(".jpg", frame_bgr, [int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality])
if not ok:
raise ValueError(f"cannot JPEG-encode frame {index} of {path}")
return base64.b64encode(jpeg.tobytes()).decode("ascii")
try:
source_fps = capture.get(cv2.CAP_PROP_FPS) or 0.0
stride = max(1, round(source_fps / fps)) if source_fps > 0 else 1
# Sub-frame k of a unit sits ``offset * stride`` frames past its base.
sub_strides = {max(1, round(offset * stride)): position for position, offset in enumerate(offsets)}
frames: list[str] = []
interiors: list[list[str]] = []
index = 0
while True:
ok, frame_bgr = capture.read()
if not ok:
break
position, within_unit = divmod(index, stride)
if within_unit == 0:
frames.append(encode(frame_bgr, index))
interiors.append([])
elif within_unit in sub_strides and position < len(interiors):
interiors[position].append(encode(frame_bgr, index))
index += 1
finally:
capture.release()
if not frames:
raise ValueError(f"video has no decodable frames: {path}")
stacked = [concat_frames_b64(interior) if interior else None for interior in interiors]
return frames, stacked
def _resolve_video_frames(args: DemoArgs) -> tuple[list[str], list[str | None]]:
"""Base64 JPEG frames for the duplex camera track, newest last.
Returns the base track plus the parallel stacked composites. Only a video
source has sub-frames to stack; an explicit frame list or a still image
stacks nothing.
"""
frames = list(getattr(args, "video_frames_b64", None) or [])
if frames:
stacked = list(getattr(args, "video_stacked_frames_b64", None) or [])
stacked += [None] * (len(frames) - len(stacked))
return frames, stacked[: len(frames)]
input_video = getattr(args, "input_video", None)
if input_video:
return _video_frames_from_file(
Path(input_video).expanduser(),
stack_frames=max(1, int(getattr(args, "stack_frames", 1) or 1)),
)
frame_image = getattr(args, "frame_image", None)
if frame_image:
return [base64.b64encode(Path(frame_image).expanduser().read_bytes()).decode("ascii")], [None]
return [], []
@dataclass
@@ -373,17 +460,20 @@ class DemoState:
commit_indices = [
index for index, event in enumerate(self.events) if event.get("type") == "input_audio_buffer.committed"
]
decision_indices = [
index
for index, event in enumerate(self.events)
if event.get("type") == "response.created"
or (
event.get("type") == "response.listen"
and isinstance(event.get("response"), dict)
and isinstance(event["response"].get("metadata"), dict)
and event["response"]["metadata"].get("model_listen") is True
)
]
decision_indices: list[int] = []
for index, event in enumerate(self.events):
event_type = event.get("type")
if event_type == "response.created":
decision_indices.append(index)
continue
if event_type != "response.listen":
continue
response = event.get("response")
if not isinstance(response, dict):
continue
metadata = response.get("metadata")
if isinstance(metadata, dict) and metadata.get("model_listen") is True:
decision_indices.append(index)
first_input_index = self.first_index("input_audio_buffer.speech_started")
if first_input_index is None:
first_input_index = commit_indices[0] if commit_indices else None
@@ -500,7 +590,7 @@ class DemoState:
return stale
def _session_update_event(args: argparse.Namespace) -> dict[str, object]:
def _session_update_event(args: DemoArgs) -> dict[str, object]:
session_payload: dict[str, object] = {
"model": args.model,
"modalities": ["audio", "text"],
@@ -836,7 +926,8 @@ async def _send_pcm16(
hints: dict[str, object] | None = None,
first_chunk_hints: dict[str, object] | None = None,
on_model_unit_ready=None,
frame_b64: str | None = None,
frames_b64: Sequence[str] | None = None,
stacked_frames_b64: Sequence[str | None] | None = None,
) -> None:
hints = hints or {}
first_chunk_hints = first_chunk_hints or {}
@@ -844,6 +935,8 @@ async def _send_pcm16(
model_unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE
audio_ms = 0
next_model_unit_bytes = model_unit_bytes
frames = list(frames_b64 or [])
stacked = list(stacked_frames_b64 or [])
frames_sent = 0
for offset in range(0, len(pcm16), chunk_bytes):
chunk = pcm16[offset : offset + chunk_bytes]
@@ -852,9 +945,18 @@ async def _send_pcm16(
chunk_hints = dict(hints)
if offset == 0:
chunk_hints.update(first_chunk_hints)
if frame_b64 is not None and audio_ms > frames_sent * 1000:
# Omni duplex cadence: one camera frame per 1 s of audio.
chunk_hints["video_frames"] = [frame_b64]
if frames and audio_ms >= duplex_unit_boundary_ms(frames_sent):
# Omni duplex cadence: one camera frame per model unit, riding the
# append that closes it, so the frame and the second of audio it was
# captured during enter the same unit. A video advances one frame per
# unit and holds its last frame when the audio outlives the clip; a
# still image is a one-element list and therefore repeats.
# A stacked composite of that unit's interior sub-frames rides the
# same append right after the base frame (wire maximum of 2).
index = min(frames_sent, len(frames) - 1)
composite = stacked[index] if index < len(stacked) else None
base = frames[index]
chunk_hints["video_frames"] = [base] if composite is None else [base, composite]
frames_sent += 1
await ws.send(
json.dumps(
@@ -935,7 +1037,8 @@ async def _send_clean_turn(
realtime_input: bool = False,
model_policy_settle_s: float = 2.0,
commit_input: bool = True,
frame_b64: str | None = None,
frames_b64: Sequence[str] | None = None,
stacked_frames_b64: Sequence[str | None] | None = None,
) -> tuple[str | None, str]:
before_created = state.count("response.created")
before_model_listen = state.model_listen_count
@@ -957,7 +1060,8 @@ async def _send_clean_turn(
chunk_ms=chunk_ms,
realtime_delay=realtime_input,
hints={"transcript": transcript} if send_transcript_hint else {},
frame_b64=frame_b64,
frames_b64=frames_b64,
stacked_frames_b64=stacked_frames_b64,
)
if commit_input:
state.input_commit_sent_at_s.append(time.monotonic())
@@ -1163,7 +1267,8 @@ async def _send_listen_only_overlap_pair(
model_policy_settle_s: float,
validation_mode: str,
silence_ms: int,
frame_b64: str | None = None,
frames_b64: Sequence[str] | None = None,
stacked_frames_b64: Sequence[str | None] | None = None,
) -> tuple[list[str | None], list[str], bool]:
if not realtime_input:
raise ValueError("listen-only-overlap requires --realtime-input")
@@ -1194,7 +1299,8 @@ async def _send_listen_only_overlap_pair(
realtime_delay=True,
hints={"transcript": transcripts[0]} if send_transcript_hint else {},
on_model_unit_ready=continue_until_first_speak,
frame_b64=frame_b64,
frames_b64=frames_b64,
stacked_frames_b64=stacked_frames_b64,
)
state.input_commit_sent_at_s.append(time.monotonic())
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
@@ -1233,7 +1339,8 @@ async def _send_listen_only_overlap_pair(
realtime_delay=True,
hints={"transcript": transcripts[1]} if send_transcript_hint else {},
on_model_unit_ready=record_model_unit_ready,
frame_b64=frame_b64,
frames_b64=frames_b64,
stacked_frames_b64=stacked_frames_b64,
)
state.input_commit_sent_at_s.append(time.monotonic())
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
@@ -1297,10 +1404,8 @@ async def _send_listen_only_overlap_pair(
return [first_response_id, second_response_id], ["speak", second_outcome], overlap_ok
async def run_demo(args: argparse.Namespace) -> dict[str, object]:
demo_frame_b64: str | None = None
if getattr(args, "frame_image", None):
demo_frame_b64 = base64.b64encode(Path(args.frame_image).read_bytes()).decode("ascii")
async def run_demo(args: DemoArgs) -> dict[str, object]:
demo_frames_b64, demo_stacked_b64 = _resolve_video_frames(args)
turn_input_paths = _turn_input_paths(
Path(args.input_wav),
list(getattr(args, "turn_input_wav", []) or []),
@@ -1371,7 +1476,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000),
validation_mode=validation_mode,
silence_ms=max(0, args.silence_ms),
frame_b64=demo_frame_b64,
frames_b64=demo_frames_b64,
stacked_frames_b64=demo_stacked_b64,
)
for turn_index in range(2, len(turn_specs)):
transcript, duration_ms = turn_specs[turn_index]
@@ -1389,7 +1495,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
realtime_input=realtime_input,
model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000),
commit_input=not continuous_input,
frame_b64=demo_frame_b64,
frames_b64=demo_frames_b64,
stacked_frames_b64=demo_stacked_b64,
)
turn_response_ids.append(response_id)
turn_outcomes.append(outcome)
@@ -1409,7 +1516,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
realtime_input=realtime_input,
model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000),
commit_input=not continuous_input,
frame_b64=demo_frame_b64,
frames_b64=demo_frames_b64,
stacked_frames_b64=demo_stacked_b64,
)
turn_response_ids.append(response_id)
turn_outcomes.append(outcome)
@@ -1584,6 +1692,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
"continuous_input": continuous_input,
"continuous_input_ok": continuous_input_ok,
"input_chunk_ms": args.chunk_ms,
"video_frame_count": len(demo_frames_b64),
"video_stacked_frame_count": sum(frame is not None for frame in demo_stacked_b64),
"model_policy_settle_ms": args.model_policy_settle_ms,
"turn_outcomes": turn_outcomes,
"turn_outcomes_ok": turn_outcomes_ok,
@@ -1641,6 +1751,19 @@ def parse_args() -> argparse.Namespace:
default=None,
help="Optional image file sent as an omni-duplex camera frame (one per 1 s of audio)",
)
parser.add_argument(
"--input-video",
default=None,
help="Optional video file streamed as omni-duplex camera frames (decoded at 1 fps, "
"overrides --frame-image; the last frame is held if the audio outlives the clip)",
)
parser.add_argument(
"--stack-frames",
type=int,
default=1,
choices=[1, 2],
help="Frames per append: 2 also resends the previous frame for motion context.",
)
parser.add_argument(
"--turn-input-wav",
action="append",
@@ -16,6 +16,7 @@ import websockets
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
SERVER_PARAMS,
demo_args,
duplex_camera_frames,
multi_session_args,
realtime_url,
resolve_ref_audio,
@@ -30,10 +31,16 @@ from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import
)
from tests.helpers.mark import hardware_test
from vllm_omni.experimental.fullduplex.client import build_realtime_url
from vllm_omni.experimental.fullduplex.video_stacking import concat_frames_b64
pytestmark = pytest.mark.omni
def _assert_positive_int(value: object) -> None:
assert isinstance(value, int)
assert value > 0
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
assert isinstance(metrics, list)
assert len(metrics) == expected_count
@@ -142,7 +149,45 @@ def test_duplex_single_session_response_required(omni_server, tmp_path: Path) ->
args.turn_duration_ms = [args.first_turn_ms] * args.turns
result = asyncio.run(run_demo(args))
assert result["ok"] is True
assert result["audio_delta_count"] > 0
_assert_positive_int(result["audio_delta_count"])
assert result["done_count"] == 2
assert result["error_count"] == 0
assert result["all_audio_responses_have_transcript"] is True
assert result["transcript_delta_done_ok"] is True
_assert_request_metrics(result["request_metrics"], expected_count=2)
_assert_session_metrics(result["session_metrics"], expected_count=2)
@pytest.mark.advanced_model
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
def test_duplex_single_session_video_input(omni_server, tmp_path: Path) -> None:
"""Audio plus a 1 fps camera track, the omni-duplex video contract.
Frames ride the audio appends, so the response contract is unchanged: what
this covers is that interleaved vision input keeps the turn intact instead
of stalling or erroring out mid-segment.
"""
args = demo_args(
omni_server=omni_server,
input_wav=validated_input_wav(),
ref_audio=resolve_ref_audio(),
output_dir=tmp_path / "video_input",
)
args.turns = 2
args.turn_duration_ms = [args.first_turn_ms] * args.turns
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera")
args.video_frames_b64 = frames
# Each unit also carries a composite of its interior sub-frames, so the
# append exercises the official two-image frame_list that carries motion.
args.video_stacked_frames_b64 = [concat_frames_b64([frames[index]] * 2) for index in range(len(frames))]
result = asyncio.run(run_demo(args))
assert result["ok"] is True
assert result["video_frame_count"] == 4
assert result["video_stacked_frame_count"] == 4
_assert_positive_int(result["audio_delta_count"])
assert result["done_count"] == 2
assert result["error_count"] == 0
assert result["all_audio_responses_have_transcript"] is True
@@ -18,6 +18,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
DRIVER_DIR = Path(__file__).resolve().parent
HELPERS_DIR = DRIVER_DIR / "helpers"
DEMO_PATH = HELPERS_DIR / "minicpmo_realtime_duplex_scenarios.py"
EXAMPLE_DEMO_PATH = DRIVER_DIR.parents[2] / "examples/online_serving/minicpmo/realtime_duplex_demo.py"
MULTI_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_multi_session.py"
PAIR_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_demo_pair.py"
SOFT_INTERRUPT_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_soft_interrupt.py"
@@ -50,6 +51,18 @@ def _load_pair_demo_module():
return module
def _load_example_demo_module():
spec = importlib.util.spec_from_file_location(
"minicpmo_realtime_duplex_example_demo_test",
EXAMPLE_DEMO_PATH,
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _load_soft_interrupt_demo_module():
spec = importlib.util.spec_from_file_location(
"minicpmo_realtime_duplex_soft_interrupt_test",
@@ -1813,6 +1826,219 @@ def test_realtime_duplex_demo_waits_at_each_model_unit_and_stops_after_speak():
assert len(ws.messages) == 10
def _sent_video_frames(messages: list[dict[str, object]]) -> list[list[str]]:
frames: list[list[str]] = []
for message in messages:
raw = message.get("video_frames")
if not isinstance(raw, list):
continue
frames.append([frame for frame in raw if isinstance(frame, str)])
return frames
def _decode_jpeg_size(frame_b64: str) -> tuple[int, int]:
import io
from PIL import Image
return Image.open(io.BytesIO(base64.b64decode(frame_b64))).size
class _RecordingWebSocket:
def __init__(self, demo):
self._demo = demo
self.messages: list[dict[str, object]] = []
async def send(self, payload):
self.messages.append(self._demo.json.loads(payload))
def _send_seconds_of_audio(demo, *, seconds: int, frames: list[str], stacked: list[str | None] | None = None):
ws = _RecordingWebSocket(demo)
asyncio.run(
demo._send_pcm16(
ws,
b"\x01\x00" * (demo.PCM16_SAMPLE_RATE * seconds),
chunk_ms=200,
realtime_delay=False,
frames_b64=frames,
stacked_frames_b64=stacked,
)
)
return ws.messages
def test_realtime_duplex_demo_streams_one_video_frame_per_model_unit():
demo = _load_demo_module()
messages = _send_seconds_of_audio(demo, seconds=3, frames=["f0", "f1", "f2", "f3"])
# Frame k rides the append that closes model unit k, so Stage0 can bind it
# to that unit's audio. 3 s of audio closes units at 1030 ms and 2030 ms and
# leaves a residual too short for a third, so only two frames go out.
assert _sent_video_frames(messages) == [["f0"], ["f1"]]
frame_indices = [index for index, message in enumerate(messages) if "video_frames" in message]
assert frame_indices == [5, 10]
# The carrying appends are exactly the ones that reach a unit boundary.
assert [messages[index]["audio_end_ms"] for index in frame_indices] == [1200, 2200]
def test_realtime_duplex_demo_never_sends_a_frame_before_its_unit_can_close():
"""Regression: frame 0 used to ride the very first 200 ms append.
Stage0 cannot close a unit until 1030 ms of audio has arrived, and it does
not carry frames across appends, so a frame sent that early was silently
dropped and every later frame ended up one unit ahead of its audio.
"""
demo = _load_demo_module()
messages = _send_seconds_of_audio(demo, seconds=3, frames=["f0", "f1", "f2"])
carried = [message for message in messages if "video_frames" in message]
assert carried, "expected at least one frame to be sent"
for message in carried:
assert message["audio_end_ms"] >= demo.duplex_unit_boundary_ms(0)
def test_realtime_duplex_demo_holds_the_last_video_frame_when_audio_outlives_the_clip():
demo = _load_demo_module()
# 4 s of audio closes three units (1030/2030/3030 ms); the two-frame clip
# holds its last frame for the third.
assert _sent_video_frames(_send_seconds_of_audio(demo, seconds=4, frames=["a", "b"])) == [["a"], ["b"], ["b"]]
# A still image is a one-element clip and therefore repeats every unit.
assert _sent_video_frames(_send_seconds_of_audio(demo, seconds=3, frames=["still"])) == [["still"], ["still"]]
def test_realtime_duplex_demo_sends_each_units_stacked_composite_next_to_its_base_frame():
demo = _load_demo_module()
messages = _send_seconds_of_audio(
demo,
seconds=4,
frames=["f0", "f1", "f2"],
stacked=["s0", "s1", None],
)
# Official pairing is frame_list=[base, composite of that unit's interior],
# so the composite belongs to the same unit as the base beside it -- never
# to the previous one. A unit without interior sub-frames sends base alone.
assert _sent_video_frames(messages) == [["f0", "s0"], ["f1", "s1"], ["f2"]]
def test_realtime_duplex_demo_decodes_a_video_file_into_one_frame_per_second(tmp_path):
pytest.importorskip("cv2")
pytest.importorskip("imageio")
from tests.helpers.media import generate_synthetic_video
demo = _load_demo_module()
# 30 fps synthetic clip: 90 frames is 3 s of camera input.
video = generate_synthetic_video(64, 64, 90, cache_dir=tmp_path)
frames, stacked = demo._resolve_video_frames(SimpleNamespace(input_video=video["file_path"]))
assert len(frames) == 3
assert all(base64.b64decode(frame, validate=True)[:3] == b"\xff\xd8\xff" for frame in frames)
assert len(set(frames)) == 3
# stack_frames defaults to 1, so there is nothing to stack.
assert stacked == [None, None, None]
def test_realtime_duplex_demo_stacks_a_units_interior_subframes_into_one_composite(tmp_path):
pytest.importorskip("cv2")
pytest.importorskip("imageio")
from tests.helpers.media import generate_synthetic_video
demo = _load_demo_module()
video = generate_synthetic_video(64, 64, 90, cache_dir=tmp_path)
frames, stacked = demo._resolve_video_frames(SimpleNamespace(input_video=video["file_path"], stack_frames=5))
# stack_frames=5 raises the visual refresh rate to 5 fps, but the four
# interior sub-frames of each second collapse into a single composite, so
# the wire still carries 2 images per unit and the audio cadence is
# untouched.
assert len(frames) == 3
assert len(stacked) == 3
assert all(frame is not None for frame in stacked)
assert all(base64.b64decode(frame, validate=True)[:3] == b"\xff\xd8\xff" for frame in stacked)
composite = _decode_jpeg_size(stacked[0])
assert composite != _decode_jpeg_size(frames[0])
def test_example_demo_uses_external_wav_duration_and_skips_video_soundtrack(tmp_path, monkeypatch):
demo = _load_example_demo_module()
wav = tmp_path / "external.wav"
demo.write_pcm16_wav(wav, b"\x00\x00" * (demo.PCM16_SAMPLE_RATE * 2), sample_rate_hz=demo.PCM16_SAMPLE_RATE)
soundtrack_calls: list[object] = []
def fake_soundtrack(*args, **kwargs):
soundtrack_calls.append(1)
raise AssertionError("video soundtrack must not be loaded when --input-wav is set")
def fake_frames(video_path, work_dir, *, fps, max_side, duration_s, stack_frames):
assert video_path == Path("/tmp/silent.mp4")
assert fps == 1.0
assert duration_s == pytest.approx(2.0)
return ["f0", "f1"], [None, None]
monkeypatch.setattr(demo, "_extract_video_soundtrack", fake_soundtrack)
monkeypatch.setattr(demo, "_extract_video_frames", fake_frames)
wav_out, frames, stacked = demo._resolve_duplex_av_inputs(
input_wav=str(wav),
input_video="/tmp/silent.mp4",
work_dir=tmp_path / "video_input",
fps=1.0,
max_side=0,
)
assert wav_out == str(wav)
assert frames == ["f0", "f1"]
assert stacked == [None, None]
assert soundtrack_calls == []
def test_example_demo_extracts_video_soundtrack_only_when_wav_is_absent(tmp_path, monkeypatch):
demo = _load_example_demo_module()
extracted = tmp_path / "from_video.wav"
demo.write_pcm16_wav(extracted, b"\x00\x00" * demo.PCM16_SAMPLE_RATE, sample_rate_hz=demo.PCM16_SAMPLE_RATE)
def fake_extract(video_path, work_dir, *, fps, max_side, stack_frames):
assert video_path == Path("/tmp/clip.mp4")
return extracted, ["v0"], ["s0"]
monkeypatch.setattr(demo, "_extract_video_input", fake_extract)
wav_out, frames, stacked = demo._resolve_duplex_av_inputs(
input_wav=None,
input_video="/tmp/clip.mp4",
work_dir=tmp_path / "video_input",
fps=1.0,
max_side=0,
)
assert wav_out == str(extracted)
assert frames == ["v0"]
assert stacked == ["s0"]
def test_realtime_duplex_demo_prefers_a_video_over_a_still_frame_image(tmp_path):
demo = _load_demo_module()
still = tmp_path / "frame.jpg"
still.write_bytes(b"\xff\xd8\xff-still")
# A still has no interior sub-frames, so it never stacks.
assert demo._resolve_video_frames(SimpleNamespace(frame_image=str(still))) == (
[base64.b64encode(still.read_bytes()).decode("ascii")],
[None],
)
assert demo._resolve_video_frames(
SimpleNamespace(frame_image=str(still), video_frames_b64=["preset"]),
) == (["preset"], [None])
assert demo._resolve_video_frames(SimpleNamespace()) == ([], [])
def test_realtime_duplex_demo_listen_only_overlap_accepts_silence_unit_before_first_done(monkeypatch):
demo = _load_demo_module()
state = demo.DemoState()
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
import importlib.util
import sys
@@ -169,6 +172,64 @@ def test_open_streaming_response_requires_post_commit_drain():
)
def _asymmetric_image():
from PIL import Image
image = Image.new("RGB", (4, 2), (0, 0, 0))
image.putpixel((0, 0), (255, 0, 0))
return image
@pytest.mark.parametrize(
"rotation,expected_corner",
[
(0, (0, 0)),
(90, (0, 3)),
(180, (3, 1)),
(-90, (1, 0)),
(270, (1, 0)),
(-270, (0, 3)),
],
)
def test_upright_applies_the_display_matrix_quarter_turn(rotation, expected_corner):
demo = _load_demo_module()
rotated = demo._upright(_asymmetric_image(), rotation)
assert rotated.getpixel(expected_corner) == (255, 0, 0)
def test_decoded_frames_apply_the_container_display_matrix(monkeypatch, tmp_path):
"""A -90 phone clip must reach the model upright, not on its side."""
demo = _load_demo_module()
source = _asymmetric_image()
class _Frame:
rotation = -90
def to_image(self):
return source
class _Container:
streams = type("_Streams", (), {"video": [type("_Stream", (), {})()]})()
def decode(self, stream):
yield _Frame()
def __enter__(self):
return self
def __exit__(self, *exc):
return False
monkeypatch.setitem(sys.modules, "av", type("_Av", (), {"open": staticmethod(lambda path: _Container())}))
(image,) = demo._decode_video_frames_rgb(tmp_path / "clip.mp4", [0])
assert image.size == (2, 4)
assert image.getpixel((1, 0)) == (255, 0, 0)
def test_streaming_output_writer_persists_audio_deltas_as_they_arrive(tmp_path, capsys):
demo = _load_demo_module()
writer = demo._StreamingOutputWriter(tmp_path)
+159 -1
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import ast
@@ -849,6 +852,161 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
assert result["prompt_suffix_len"] == 0
def _stage0_vision_runtime():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
class _StageModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.embed = torch.nn.Embedding(256, 2)
def get_input_embeddings(self):
return self.embed
def get_audio_hidden_states(self, _data):
return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)]
runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime)
runtime.stage_model = _StageModel()
runtime.thinker = runtime.stage_model
runtime.tokenizer = SimpleNamespace(
unk_token_id=0,
convert_tokens_to_ids=lambda token: {
"<unit>": 1,
"</unit>": 2,
"<|listen|>": 3,
"<|speak|>": 4,
"<|tts_bos|>": 5,
"<|tts_eos|>": 6,
"<|tts_pad|>": 7,
"<|chunk_eos|>": 8,
"<|chunk_tts_eos|>": 9,
"<|turn_eos|>": 10,
"<|audio|>": 11,
"<image>": 12,
"</image>": 13,
"<slice>": 14,
"</slice>": 15,
}.get(token, 0),
encode=lambda text, add_special_tokens=False: [201, 5],
)
runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4)
runtime.device = "cpu"
runtime._init_token_ids()
# One 2-row block per frame keeps the expected token_ids readable; the real
# resampler emits VISION_EMBEDS_PER_FRAME rows per slice.
runtime._stage_vision_embeddings = lambda frames: [
[torch.full((2, 2), float(index), dtype=torch.float32)] for index in range(len(frames))
]
return runtime
def test_minicpmo_stage0_puts_every_frame_of_an_append_in_one_unit():
"""Official streaming_prefill feeds the whole frame_list into one unit.
Stacked pairs used to be spread one frame per unit, so a two-frame append
that closed a single unit left a frame over, failed the prefill, and still
advanced the session state.
"""
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
_MiniCPMO45Stage0SessionState,
)
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-stacked-frames")
result = runtime._stage_prefill_embeddings_only(
state,
np.zeros(4, dtype=np.float32),
video_frames=["frame-prev", "frame-cur"],
seq=1,
)
assert result["success"] is True
# <unit>, then each frame as <image> + embeds + </image>, then the audio.
assert result["input_token_ids"] == [1, 12, 0, 0, 13, 12, 0, 0, 13, 11]
assert state.audio_chunk_idx == 1
def test_minicpmo_stage0_wraps_hd_slices_after_the_source_image():
"""Official stacked pair is max_slice_nums=[2, 1]: source + slices, then audio."""
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
_MiniCPMO45Stage0SessionState,
)
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-hd-slices")
runtime._stage_vision_embeddings = lambda frames: [
[torch.ones((2, 2)), torch.full((2, 2), 2.0), torch.full((2, 2), 3.0)],
[torch.full((2, 2), 4.0)],
]
result = runtime._stage_prefill_embeddings_only(
state,
np.zeros(4, dtype=np.float32),
video_frames=["base", "stack"],
seq=1,
)
assert result["success"] is True
# <unit> <image> src </image> <slice> p0 </slice> <slice> p1 </slice>
# <image> stack </image> audio
assert result["input_token_ids"] == [
1,
12,
0,
0,
13,
14,
0,
0,
15,
14,
0,
0,
15,
12,
0,
0,
13,
11,
]
def test_minicpmo_stage0_reports_frames_that_close_no_unit_instead_of_dropping_them():
"""A frame only binds to the unit its append closes.
Frames are not carried across appends, so one arriving early used to vanish
under a generic "audio not enough" buffering result.
"""
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
_MiniCPMO45Stage0SessionState,
)
runtime = _stage0_vision_runtime()
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-early-frame")
assert runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)["success"] is True
early = runtime._stage_prefill_embeddings_only(
state,
np.zeros(2, dtype=np.float32),
video_frames=["frame-cur"],
seq=2,
)
assert early["success"] is False
assert "closes no model unit" in early["reason"]
# The half-filled append must not advance the unit counter.
assert state.audio_chunk_idx == 1
def test_minicpmo_stage0_speech_append_sets_pending_context_once():
import torch
@@ -2514,7 +2672,7 @@ def _ar_runner_classdefs():
def _method_calls(method):
calls = {}
calls: dict[str, int] = {}
for node in ast.walk(method):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
calls.setdefault(node.func.attr, node.lineno)