[Bugfix][MiniCPM-o] Bind omni-duplex video frames to the unit they close (#6404)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
13173d6bbb
commit
da94c3ebe1
17 files changed
+1530
-144
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
|
||||
|
||||
The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
|
||||
@@ -27,6 +30,8 @@ _MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
|
||||
|
||||
if _MODULE_NAME not in sys.modules:
|
||||
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
|
||||
if _spec is None or _spec.loader is None:
|
||||
raise ImportError(f"Cannot load {_MODULE_NAME} from {_MODULE_PATH}")
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_MODULE_NAME] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
@@ -87,7 +92,9 @@ def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) ->
|
||||
def fake_extract(video_path, *, audio_path, include_audio, **_kw):
|
||||
counter.append(1)
|
||||
frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
|
||||
segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
|
||||
segments: list[np.ndarray] = (
|
||||
[np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
|
||||
)
|
||||
return frames, segments
|
||||
|
||||
monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
|
||||
|
||||
@@ -18,6 +18,10 @@ from vllm_omni.experimental.fullduplex.client import (
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
|
||||
MiniCPMO45DuplexPolicy,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.video_stacking import (
|
||||
concat_frames,
|
||||
unit_subframe_offsets,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -418,3 +422,62 @@ def test_realtime_client_pcm16_wav_round_trip(tmp_path):
|
||||
assert wav_file.getnchannels() == 1
|
||||
assert wav_file.getframerate() == 16_000
|
||||
assert read_pcm16_wav(path) == pcm16
|
||||
|
||||
|
||||
def _stack_test_image(color, size=(40, 30)):
|
||||
from PIL import Image
|
||||
|
||||
return Image.new("RGB", size, color)
|
||||
|
||||
|
||||
def test_video_stacking_tiles_a_units_subframes_into_one_image():
|
||||
frames = [_stack_test_image((255, 0, 0)), _stack_test_image((0, 255, 0))]
|
||||
|
||||
composite = concat_frames(frames)
|
||||
|
||||
# Landscape cells stack into a column because that canvas is squarer, and
|
||||
# the interior seam carries the separator band that lets the model tell the
|
||||
# sub-frames apart.
|
||||
assert composite.size == (40, 30 * 2 + 6)
|
||||
assert composite.getpixel((20, 30 + 3)) == (0, 0, 0)
|
||||
|
||||
|
||||
def test_video_stacking_picks_the_squarest_grid():
|
||||
portrait = [_stack_test_image((255, 0, 0), size=(20, 60)) for _ in range(2)]
|
||||
four = [_stack_test_image((0, 0, 255)) for _ in range(4)]
|
||||
|
||||
# A portrait clip tiles side by side, four frames always tile 2x2.
|
||||
assert concat_frames(portrait).size == (20 * 2 + 6, 60)
|
||||
assert concat_frames(four).size == (40 * 2 + 6, 30 * 2 + 6)
|
||||
|
||||
|
||||
def test_video_stacking_skips_the_subframe_that_duplicates_the_base_frame():
|
||||
# Official samples stack_frames=5 as 0.2/0.4/0.6/0.8 s: offset 0 is the base
|
||||
# frame, already sent as frame_list[0].
|
||||
assert unit_subframe_offsets(5) == pytest.approx([0.2, 0.4, 0.6, 0.8])
|
||||
assert unit_subframe_offsets(1) == []
|
||||
|
||||
|
||||
def test_realtime_client_sends_each_units_composite_beside_its_base_frame():
|
||||
sent: list[dict] = []
|
||||
|
||||
class _Client(RealtimeDuplexClient):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
async def send(self, event):
|
||||
sent.append(event)
|
||||
|
||||
asyncio.run(
|
||||
_Client().stream_pcm16(
|
||||
b"\x01\x00" * (16_000 * 3),
|
||||
chunk_ms=200,
|
||||
realtime=False,
|
||||
video_frames=["f0", "f1"],
|
||||
stacked_video_frames=["s0", None],
|
||||
)
|
||||
)
|
||||
|
||||
# A composite belongs to the unit it was captured in, so it rides the same
|
||||
# append as that unit's base frame; a unit without one sends the base alone.
|
||||
assert [event["video_frames"] for event in sent if "video_frames" in event] == [["f0", "s0"], ["f1"]]
|
||||
@@ -91,6 +91,19 @@ def resolve_ref_audio() -> Path:
|
||||
return ref_audio
|
||||
|
||||
|
||||
def duplex_camera_frames(*, seconds: int, cache_dir: Path) -> list[str]:
|
||||
"""Base64 JPEG frames of a moving synthetic scene, one per second.
|
||||
|
||||
Mirrors what the realtime web client captures (1 fps, no client-side
|
||||
resize: the server normalizes at ``scale_resolution=448``).
|
||||
"""
|
||||
from tests.e2e.online_serving.helpers.minicpmo_realtime_duplex_scenarios import _video_frames_from_file
|
||||
from tests.helpers.media import generate_synthetic_video
|
||||
|
||||
video = generate_synthetic_video(448, 448, 30 * seconds, cache_dir=cache_dir)
|
||||
return _video_frames_from_file(Path(video["file_path"]))
|
||||
|
||||
|
||||
def realtime_url(omni_server) -> str:
|
||||
return f"ws://{omni_server.host}:{omni_server.port}/v1/realtime?duplex=1"
|
||||
|
||||
@@ -113,6 +126,8 @@ def demo_args(
|
||||
output_audio_format="pcm16",
|
||||
chunk_ms=200,
|
||||
realtime_input=True,
|
||||
input_video=None,
|
||||
stack_frames=1,
|
||||
first_turn_ms=1400,
|
||||
turn_duration_ms=[],
|
||||
first_turn_transcript="duplex CI speech",
|
||||
|
||||
@@ -37,8 +37,10 @@ import json
|
||||
import sys
|
||||
import time
|
||||
import wave
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
if str(_REPO_ROOT) not in sys.path:
|
||||
@@ -55,15 +57,100 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
PCM16_SAMPLE_RATE,
|
||||
RealtimeEventCollector,
|
||||
build_realtime_url,
|
||||
duplex_unit_boundary_ms,
|
||||
read_pcm16_wav,
|
||||
summarize_session_request_metrics,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
reference_audio_data_url as _ref_audio_data_url,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402
|
||||
concat_frames_b64,
|
||||
unit_subframe_offsets,
|
||||
)
|
||||
|
||||
_url_with_model = build_realtime_url
|
||||
_read_wav_pcm16 = read_pcm16_wav
|
||||
DemoArgs = argparse.Namespace | SimpleNamespace
|
||||
|
||||
|
||||
def _video_frames_from_file(
|
||||
path: Path,
|
||||
*,
|
||||
fps: float = 1.0,
|
||||
jpeg_quality: int = 70,
|
||||
stack_frames: int = 1,
|
||||
) -> tuple[list[str], list[str | None]]:
|
||||
"""Decode ``path`` into base64 JPEG frames at the omni duplex cadence.
|
||||
|
||||
One frame per second matches what the realtime web client captures, so the
|
||||
stream stays representative of a live camera. ``stack_frames > 1`` also
|
||||
samples each unit's interior sub-frames and tiles them into one composite,
|
||||
the official way to raise visual refresh rate without changing the audio
|
||||
cadence. Returns ``(base_frames, stacked_frames)`` with the tracks parallel.
|
||||
"""
|
||||
import cv2
|
||||
|
||||
offsets = unit_subframe_offsets(stack_frames)
|
||||
capture = cv2.VideoCapture(str(path))
|
||||
if not capture.isOpened():
|
||||
raise ValueError(f"cannot open video: {path}")
|
||||
|
||||
def encode(frame_bgr, index: int) -> str:
|
||||
ok, jpeg = cv2.imencode(".jpg", frame_bgr, [int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality])
|
||||
if not ok:
|
||||
raise ValueError(f"cannot JPEG-encode frame {index} of {path}")
|
||||
return base64.b64encode(jpeg.tobytes()).decode("ascii")
|
||||
|
||||
try:
|
||||
source_fps = capture.get(cv2.CAP_PROP_FPS) or 0.0
|
||||
stride = max(1, round(source_fps / fps)) if source_fps > 0 else 1
|
||||
# Sub-frame k of a unit sits ``offset * stride`` frames past its base.
|
||||
sub_strides = {max(1, round(offset * stride)): position for position, offset in enumerate(offsets)}
|
||||
frames: list[str] = []
|
||||
interiors: list[list[str]] = []
|
||||
index = 0
|
||||
while True:
|
||||
ok, frame_bgr = capture.read()
|
||||
if not ok:
|
||||
break
|
||||
position, within_unit = divmod(index, stride)
|
||||
if within_unit == 0:
|
||||
frames.append(encode(frame_bgr, index))
|
||||
interiors.append([])
|
||||
elif within_unit in sub_strides and position < len(interiors):
|
||||
interiors[position].append(encode(frame_bgr, index))
|
||||
index += 1
|
||||
finally:
|
||||
capture.release()
|
||||
if not frames:
|
||||
raise ValueError(f"video has no decodable frames: {path}")
|
||||
stacked = [concat_frames_b64(interior) if interior else None for interior in interiors]
|
||||
return frames, stacked
|
||||
|
||||
|
||||
def _resolve_video_frames(args: DemoArgs) -> tuple[list[str], list[str | None]]:
|
||||
"""Base64 JPEG frames for the duplex camera track, newest last.
|
||||
|
||||
Returns the base track plus the parallel stacked composites. Only a video
|
||||
source has sub-frames to stack; an explicit frame list or a still image
|
||||
stacks nothing.
|
||||
"""
|
||||
frames = list(getattr(args, "video_frames_b64", None) or [])
|
||||
if frames:
|
||||
stacked = list(getattr(args, "video_stacked_frames_b64", None) or [])
|
||||
stacked += [None] * (len(frames) - len(stacked))
|
||||
return frames, stacked[: len(frames)]
|
||||
input_video = getattr(args, "input_video", None)
|
||||
if input_video:
|
||||
return _video_frames_from_file(
|
||||
Path(input_video).expanduser(),
|
||||
stack_frames=max(1, int(getattr(args, "stack_frames", 1) or 1)),
|
||||
)
|
||||
frame_image = getattr(args, "frame_image", None)
|
||||
if frame_image:
|
||||
return [base64.b64encode(Path(frame_image).expanduser().read_bytes()).decode("ascii")], [None]
|
||||
return [], []
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -373,17 +460,20 @@ class DemoState:
|
||||
commit_indices = [
|
||||
index for index, event in enumerate(self.events) if event.get("type") == "input_audio_buffer.committed"
|
||||
]
|
||||
decision_indices = [
|
||||
index
|
||||
for index, event in enumerate(self.events)
|
||||
if event.get("type") == "response.created"
|
||||
or (
|
||||
event.get("type") == "response.listen"
|
||||
and isinstance(event.get("response"), dict)
|
||||
and isinstance(event["response"].get("metadata"), dict)
|
||||
and event["response"]["metadata"].get("model_listen") is True
|
||||
)
|
||||
]
|
||||
decision_indices: list[int] = []
|
||||
for index, event in enumerate(self.events):
|
||||
event_type = event.get("type")
|
||||
if event_type == "response.created":
|
||||
decision_indices.append(index)
|
||||
continue
|
||||
if event_type != "response.listen":
|
||||
continue
|
||||
response = event.get("response")
|
||||
if not isinstance(response, dict):
|
||||
continue
|
||||
metadata = response.get("metadata")
|
||||
if isinstance(metadata, dict) and metadata.get("model_listen") is True:
|
||||
decision_indices.append(index)
|
||||
first_input_index = self.first_index("input_audio_buffer.speech_started")
|
||||
if first_input_index is None:
|
||||
first_input_index = commit_indices[0] if commit_indices else None
|
||||
@@ -500,7 +590,7 @@ class DemoState:
|
||||
return stale
|
||||
|
||||
|
||||
def _session_update_event(args: argparse.Namespace) -> dict[str, object]:
|
||||
def _session_update_event(args: DemoArgs) -> dict[str, object]:
|
||||
session_payload: dict[str, object] = {
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
@@ -836,7 +926,8 @@ async def _send_pcm16(
|
||||
hints: dict[str, object] | None = None,
|
||||
first_chunk_hints: dict[str, object] | None = None,
|
||||
on_model_unit_ready=None,
|
||||
frame_b64: str | None = None,
|
||||
frames_b64: Sequence[str] | None = None,
|
||||
stacked_frames_b64: Sequence[str | None] | None = None,
|
||||
) -> None:
|
||||
hints = hints or {}
|
||||
first_chunk_hints = first_chunk_hints or {}
|
||||
@@ -844,6 +935,8 @@ async def _send_pcm16(
|
||||
model_unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE
|
||||
audio_ms = 0
|
||||
next_model_unit_bytes = model_unit_bytes
|
||||
frames = list(frames_b64 or [])
|
||||
stacked = list(stacked_frames_b64 or [])
|
||||
frames_sent = 0
|
||||
for offset in range(0, len(pcm16), chunk_bytes):
|
||||
chunk = pcm16[offset : offset + chunk_bytes]
|
||||
@@ -852,9 +945,18 @@ async def _send_pcm16(
|
||||
chunk_hints = dict(hints)
|
||||
if offset == 0:
|
||||
chunk_hints.update(first_chunk_hints)
|
||||
if frame_b64 is not None and audio_ms > frames_sent * 1000:
|
||||
# Omni duplex cadence: one camera frame per 1 s of audio.
|
||||
chunk_hints["video_frames"] = [frame_b64]
|
||||
if frames and audio_ms >= duplex_unit_boundary_ms(frames_sent):
|
||||
# Omni duplex cadence: one camera frame per model unit, riding the
|
||||
# append that closes it, so the frame and the second of audio it was
|
||||
# captured during enter the same unit. A video advances one frame per
|
||||
# unit and holds its last frame when the audio outlives the clip; a
|
||||
# still image is a one-element list and therefore repeats.
|
||||
# A stacked composite of that unit's interior sub-frames rides the
|
||||
# same append right after the base frame (wire maximum of 2).
|
||||
index = min(frames_sent, len(frames) - 1)
|
||||
composite = stacked[index] if index < len(stacked) else None
|
||||
base = frames[index]
|
||||
chunk_hints["video_frames"] = [base] if composite is None else [base, composite]
|
||||
frames_sent += 1
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
@@ -935,7 +1037,8 @@ async def _send_clean_turn(
|
||||
realtime_input: bool = False,
|
||||
model_policy_settle_s: float = 2.0,
|
||||
commit_input: bool = True,
|
||||
frame_b64: str | None = None,
|
||||
frames_b64: Sequence[str] | None = None,
|
||||
stacked_frames_b64: Sequence[str | None] | None = None,
|
||||
) -> tuple[str | None, str]:
|
||||
before_created = state.count("response.created")
|
||||
before_model_listen = state.model_listen_count
|
||||
@@ -957,7 +1060,8 @@ async def _send_clean_turn(
|
||||
chunk_ms=chunk_ms,
|
||||
realtime_delay=realtime_input,
|
||||
hints={"transcript": transcript} if send_transcript_hint else {},
|
||||
frame_b64=frame_b64,
|
||||
frames_b64=frames_b64,
|
||||
stacked_frames_b64=stacked_frames_b64,
|
||||
)
|
||||
if commit_input:
|
||||
state.input_commit_sent_at_s.append(time.monotonic())
|
||||
@@ -1163,7 +1267,8 @@ async def _send_listen_only_overlap_pair(
|
||||
model_policy_settle_s: float,
|
||||
validation_mode: str,
|
||||
silence_ms: int,
|
||||
frame_b64: str | None = None,
|
||||
frames_b64: Sequence[str] | None = None,
|
||||
stacked_frames_b64: Sequence[str | None] | None = None,
|
||||
) -> tuple[list[str | None], list[str], bool]:
|
||||
if not realtime_input:
|
||||
raise ValueError("listen-only-overlap requires --realtime-input")
|
||||
@@ -1194,7 +1299,8 @@ async def _send_listen_only_overlap_pair(
|
||||
realtime_delay=True,
|
||||
hints={"transcript": transcripts[0]} if send_transcript_hint else {},
|
||||
on_model_unit_ready=continue_until_first_speak,
|
||||
frame_b64=frame_b64,
|
||||
frames_b64=frames_b64,
|
||||
stacked_frames_b64=stacked_frames_b64,
|
||||
)
|
||||
state.input_commit_sent_at_s.append(time.monotonic())
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
@@ -1233,7 +1339,8 @@ async def _send_listen_only_overlap_pair(
|
||||
realtime_delay=True,
|
||||
hints={"transcript": transcripts[1]} if send_transcript_hint else {},
|
||||
on_model_unit_ready=record_model_unit_ready,
|
||||
frame_b64=frame_b64,
|
||||
frames_b64=frames_b64,
|
||||
stacked_frames_b64=stacked_frames_b64,
|
||||
)
|
||||
state.input_commit_sent_at_s.append(time.monotonic())
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
@@ -1297,10 +1404,8 @@ async def _send_listen_only_overlap_pair(
|
||||
return [first_response_id, second_response_id], ["speak", second_outcome], overlap_ok
|
||||
|
||||
|
||||
async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
demo_frame_b64: str | None = None
|
||||
if getattr(args, "frame_image", None):
|
||||
demo_frame_b64 = base64.b64encode(Path(args.frame_image).read_bytes()).decode("ascii")
|
||||
async def run_demo(args: DemoArgs) -> dict[str, object]:
|
||||
demo_frames_b64, demo_stacked_b64 = _resolve_video_frames(args)
|
||||
turn_input_paths = _turn_input_paths(
|
||||
Path(args.input_wav),
|
||||
list(getattr(args, "turn_input_wav", []) or []),
|
||||
@@ -1371,7 +1476,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000),
|
||||
validation_mode=validation_mode,
|
||||
silence_ms=max(0, args.silence_ms),
|
||||
frame_b64=demo_frame_b64,
|
||||
frames_b64=demo_frames_b64,
|
||||
stacked_frames_b64=demo_stacked_b64,
|
||||
)
|
||||
for turn_index in range(2, len(turn_specs)):
|
||||
transcript, duration_ms = turn_specs[turn_index]
|
||||
@@ -1389,7 +1495,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
realtime_input=realtime_input,
|
||||
model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000),
|
||||
commit_input=not continuous_input,
|
||||
frame_b64=demo_frame_b64,
|
||||
frames_b64=demo_frames_b64,
|
||||
stacked_frames_b64=demo_stacked_b64,
|
||||
)
|
||||
turn_response_ids.append(response_id)
|
||||
turn_outcomes.append(outcome)
|
||||
@@ -1409,7 +1516,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
realtime_input=realtime_input,
|
||||
model_policy_settle_s=max(0.0, args.model_policy_settle_ms / 1000),
|
||||
commit_input=not continuous_input,
|
||||
frame_b64=demo_frame_b64,
|
||||
frames_b64=demo_frames_b64,
|
||||
stacked_frames_b64=demo_stacked_b64,
|
||||
)
|
||||
turn_response_ids.append(response_id)
|
||||
turn_outcomes.append(outcome)
|
||||
@@ -1584,6 +1692,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
"continuous_input": continuous_input,
|
||||
"continuous_input_ok": continuous_input_ok,
|
||||
"input_chunk_ms": args.chunk_ms,
|
||||
"video_frame_count": len(demo_frames_b64),
|
||||
"video_stacked_frame_count": sum(frame is not None for frame in demo_stacked_b64),
|
||||
"model_policy_settle_ms": args.model_policy_settle_ms,
|
||||
"turn_outcomes": turn_outcomes,
|
||||
"turn_outcomes_ok": turn_outcomes_ok,
|
||||
@@ -1641,6 +1751,19 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help="Optional image file sent as an omni-duplex camera frame (one per 1 s of audio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-video",
|
||||
default=None,
|
||||
help="Optional video file streamed as omni-duplex camera frames (decoded at 1 fps, "
|
||||
"overrides --frame-image; the last frame is held if the audio outlives the clip)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stack-frames",
|
||||
type=int,
|
||||
default=1,
|
||||
choices=[1, 2],
|
||||
help="Frames per append: 2 also resends the previous frame for motion context.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--turn-input-wav",
|
||||
action="append",
|
||||
|
||||
@@ -16,6 +16,7 @@ import websockets
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
demo_args,
|
||||
duplex_camera_frames,
|
||||
multi_session_args,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
@@ -30,10 +31,16 @@ from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||
from vllm_omni.experimental.fullduplex.video_stacking import concat_frames_b64
|
||||
|
||||
pytestmark = pytest.mark.omni
|
||||
|
||||
|
||||
def _assert_positive_int(value: object) -> None:
|
||||
assert isinstance(value, int)
|
||||
assert value > 0
|
||||
|
||||
|
||||
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, list)
|
||||
assert len(metrics) == expected_count
|
||||
@@ -142,7 +149,45 @@ def test_duplex_single_session_response_required(omni_server, tmp_path: Path) ->
|
||||
args.turn_duration_ms = [args.first_turn_ms] * args.turns
|
||||
result = asyncio.run(run_demo(args))
|
||||
assert result["ok"] is True
|
||||
assert result["audio_delta_count"] > 0
|
||||
_assert_positive_int(result["audio_delta_count"])
|
||||
assert result["done_count"] == 2
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
assert result["transcript_delta_done_ok"] is True
|
||||
_assert_request_metrics(result["request_metrics"], expected_count=2)
|
||||
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_single_session_video_input(omni_server, tmp_path: Path) -> None:
|
||||
"""Audio plus a 1 fps camera track, the omni-duplex video contract.
|
||||
|
||||
Frames ride the audio appends, so the response contract is unchanged: what
|
||||
this covers is that interleaved vision input keeps the turn intact instead
|
||||
of stalling or erroring out mid-segment.
|
||||
"""
|
||||
args = demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(),
|
||||
output_dir=tmp_path / "video_input",
|
||||
)
|
||||
args.turns = 2
|
||||
args.turn_duration_ms = [args.first_turn_ms] * args.turns
|
||||
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera")
|
||||
args.video_frames_b64 = frames
|
||||
# Each unit also carries a composite of its interior sub-frames, so the
|
||||
# append exercises the official two-image frame_list that carries motion.
|
||||
args.video_stacked_frames_b64 = [concat_frames_b64([frames[index]] * 2) for index in range(len(frames))]
|
||||
|
||||
result = asyncio.run(run_demo(args))
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["video_frame_count"] == 4
|
||||
assert result["video_stacked_frame_count"] == 4
|
||||
_assert_positive_int(result["audio_delta_count"])
|
||||
assert result["done_count"] == 2
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
|
||||
@@ -18,6 +18,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
DRIVER_DIR = Path(__file__).resolve().parent
|
||||
HELPERS_DIR = DRIVER_DIR / "helpers"
|
||||
DEMO_PATH = HELPERS_DIR / "minicpmo_realtime_duplex_scenarios.py"
|
||||
EXAMPLE_DEMO_PATH = DRIVER_DIR.parents[2] / "examples/online_serving/minicpmo/realtime_duplex_demo.py"
|
||||
MULTI_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_multi_session.py"
|
||||
PAIR_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_demo_pair.py"
|
||||
SOFT_INTERRUPT_DEMO_PATH = DRIVER_DIR / "run_minicpmo_realtime_duplex_soft_interrupt.py"
|
||||
@@ -50,6 +51,18 @@ def _load_pair_demo_module():
|
||||
return module
|
||||
|
||||
|
||||
def _load_example_demo_module():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"minicpmo_realtime_duplex_example_demo_test",
|
||||
EXAMPLE_DEMO_PATH,
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _load_soft_interrupt_demo_module():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"minicpmo_realtime_duplex_soft_interrupt_test",
|
||||
@@ -1813,6 +1826,219 @@ def test_realtime_duplex_demo_waits_at_each_model_unit_and_stops_after_speak():
|
||||
assert len(ws.messages) == 10
|
||||
|
||||
|
||||
def _sent_video_frames(messages: list[dict[str, object]]) -> list[list[str]]:
|
||||
frames: list[list[str]] = []
|
||||
for message in messages:
|
||||
raw = message.get("video_frames")
|
||||
if not isinstance(raw, list):
|
||||
continue
|
||||
frames.append([frame for frame in raw if isinstance(frame, str)])
|
||||
return frames
|
||||
|
||||
|
||||
def _decode_jpeg_size(frame_b64: str) -> tuple[int, int]:
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
return Image.open(io.BytesIO(base64.b64decode(frame_b64))).size
|
||||
|
||||
|
||||
class _RecordingWebSocket:
|
||||
def __init__(self, demo):
|
||||
self._demo = demo
|
||||
self.messages: list[dict[str, object]] = []
|
||||
|
||||
async def send(self, payload):
|
||||
self.messages.append(self._demo.json.loads(payload))
|
||||
|
||||
|
||||
def _send_seconds_of_audio(demo, *, seconds: int, frames: list[str], stacked: list[str | None] | None = None):
|
||||
ws = _RecordingWebSocket(demo)
|
||||
asyncio.run(
|
||||
demo._send_pcm16(
|
||||
ws,
|
||||
b"\x01\x00" * (demo.PCM16_SAMPLE_RATE * seconds),
|
||||
chunk_ms=200,
|
||||
realtime_delay=False,
|
||||
frames_b64=frames,
|
||||
stacked_frames_b64=stacked,
|
||||
)
|
||||
)
|
||||
return ws.messages
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_streams_one_video_frame_per_model_unit():
|
||||
demo = _load_demo_module()
|
||||
|
||||
messages = _send_seconds_of_audio(demo, seconds=3, frames=["f0", "f1", "f2", "f3"])
|
||||
|
||||
# Frame k rides the append that closes model unit k, so Stage0 can bind it
|
||||
# to that unit's audio. 3 s of audio closes units at 1030 ms and 2030 ms and
|
||||
# leaves a residual too short for a third, so only two frames go out.
|
||||
assert _sent_video_frames(messages) == [["f0"], ["f1"]]
|
||||
frame_indices = [index for index, message in enumerate(messages) if "video_frames" in message]
|
||||
assert frame_indices == [5, 10]
|
||||
# The carrying appends are exactly the ones that reach a unit boundary.
|
||||
assert [messages[index]["audio_end_ms"] for index in frame_indices] == [1200, 2200]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_never_sends_a_frame_before_its_unit_can_close():
|
||||
"""Regression: frame 0 used to ride the very first 200 ms append.
|
||||
|
||||
Stage0 cannot close a unit until 1030 ms of audio has arrived, and it does
|
||||
not carry frames across appends, so a frame sent that early was silently
|
||||
dropped and every later frame ended up one unit ahead of its audio.
|
||||
"""
|
||||
demo = _load_demo_module()
|
||||
|
||||
messages = _send_seconds_of_audio(demo, seconds=3, frames=["f0", "f1", "f2"])
|
||||
|
||||
carried = [message for message in messages if "video_frames" in message]
|
||||
assert carried, "expected at least one frame to be sent"
|
||||
for message in carried:
|
||||
assert message["audio_end_ms"] >= demo.duplex_unit_boundary_ms(0)
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_holds_the_last_video_frame_when_audio_outlives_the_clip():
|
||||
demo = _load_demo_module()
|
||||
|
||||
# 4 s of audio closes three units (1030/2030/3030 ms); the two-frame clip
|
||||
# holds its last frame for the third.
|
||||
assert _sent_video_frames(_send_seconds_of_audio(demo, seconds=4, frames=["a", "b"])) == [["a"], ["b"], ["b"]]
|
||||
# A still image is a one-element clip and therefore repeats every unit.
|
||||
assert _sent_video_frames(_send_seconds_of_audio(demo, seconds=3, frames=["still"])) == [["still"], ["still"]]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_sends_each_units_stacked_composite_next_to_its_base_frame():
|
||||
demo = _load_demo_module()
|
||||
|
||||
messages = _send_seconds_of_audio(
|
||||
demo,
|
||||
seconds=4,
|
||||
frames=["f0", "f1", "f2"],
|
||||
stacked=["s0", "s1", None],
|
||||
)
|
||||
|
||||
# Official pairing is frame_list=[base, composite of that unit's interior],
|
||||
# so the composite belongs to the same unit as the base beside it -- never
|
||||
# to the previous one. A unit without interior sub-frames sends base alone.
|
||||
assert _sent_video_frames(messages) == [["f0", "s0"], ["f1", "s1"], ["f2"]]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_decodes_a_video_file_into_one_frame_per_second(tmp_path):
|
||||
pytest.importorskip("cv2")
|
||||
pytest.importorskip("imageio")
|
||||
from tests.helpers.media import generate_synthetic_video
|
||||
|
||||
demo = _load_demo_module()
|
||||
# 30 fps synthetic clip: 90 frames is 3 s of camera input.
|
||||
video = generate_synthetic_video(64, 64, 90, cache_dir=tmp_path)
|
||||
|
||||
frames, stacked = demo._resolve_video_frames(SimpleNamespace(input_video=video["file_path"]))
|
||||
|
||||
assert len(frames) == 3
|
||||
assert all(base64.b64decode(frame, validate=True)[:3] == b"\xff\xd8\xff" for frame in frames)
|
||||
assert len(set(frames)) == 3
|
||||
# stack_frames defaults to 1, so there is nothing to stack.
|
||||
assert stacked == [None, None, None]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_stacks_a_units_interior_subframes_into_one_composite(tmp_path):
|
||||
pytest.importorskip("cv2")
|
||||
pytest.importorskip("imageio")
|
||||
from tests.helpers.media import generate_synthetic_video
|
||||
|
||||
demo = _load_demo_module()
|
||||
video = generate_synthetic_video(64, 64, 90, cache_dir=tmp_path)
|
||||
|
||||
frames, stacked = demo._resolve_video_frames(SimpleNamespace(input_video=video["file_path"], stack_frames=5))
|
||||
|
||||
# stack_frames=5 raises the visual refresh rate to 5 fps, but the four
|
||||
# interior sub-frames of each second collapse into a single composite, so
|
||||
# the wire still carries 2 images per unit and the audio cadence is
|
||||
# untouched.
|
||||
assert len(frames) == 3
|
||||
assert len(stacked) == 3
|
||||
assert all(frame is not None for frame in stacked)
|
||||
assert all(base64.b64decode(frame, validate=True)[:3] == b"\xff\xd8\xff" for frame in stacked)
|
||||
composite = _decode_jpeg_size(stacked[0])
|
||||
assert composite != _decode_jpeg_size(frames[0])
|
||||
|
||||
|
||||
def test_example_demo_uses_external_wav_duration_and_skips_video_soundtrack(tmp_path, monkeypatch):
|
||||
demo = _load_example_demo_module()
|
||||
wav = tmp_path / "external.wav"
|
||||
demo.write_pcm16_wav(wav, b"\x00\x00" * (demo.PCM16_SAMPLE_RATE * 2), sample_rate_hz=demo.PCM16_SAMPLE_RATE)
|
||||
soundtrack_calls: list[object] = []
|
||||
|
||||
def fake_soundtrack(*args, **kwargs):
|
||||
soundtrack_calls.append(1)
|
||||
raise AssertionError("video soundtrack must not be loaded when --input-wav is set")
|
||||
|
||||
def fake_frames(video_path, work_dir, *, fps, max_side, duration_s, stack_frames):
|
||||
assert video_path == Path("/tmp/silent.mp4")
|
||||
assert fps == 1.0
|
||||
assert duration_s == pytest.approx(2.0)
|
||||
return ["f0", "f1"], [None, None]
|
||||
|
||||
monkeypatch.setattr(demo, "_extract_video_soundtrack", fake_soundtrack)
|
||||
monkeypatch.setattr(demo, "_extract_video_frames", fake_frames)
|
||||
|
||||
wav_out, frames, stacked = demo._resolve_duplex_av_inputs(
|
||||
input_wav=str(wav),
|
||||
input_video="/tmp/silent.mp4",
|
||||
work_dir=tmp_path / "video_input",
|
||||
fps=1.0,
|
||||
max_side=0,
|
||||
)
|
||||
|
||||
assert wav_out == str(wav)
|
||||
assert frames == ["f0", "f1"]
|
||||
assert stacked == [None, None]
|
||||
assert soundtrack_calls == []
|
||||
|
||||
|
||||
def test_example_demo_extracts_video_soundtrack_only_when_wav_is_absent(tmp_path, monkeypatch):
|
||||
demo = _load_example_demo_module()
|
||||
extracted = tmp_path / "from_video.wav"
|
||||
demo.write_pcm16_wav(extracted, b"\x00\x00" * demo.PCM16_SAMPLE_RATE, sample_rate_hz=demo.PCM16_SAMPLE_RATE)
|
||||
|
||||
def fake_extract(video_path, work_dir, *, fps, max_side, stack_frames):
|
||||
assert video_path == Path("/tmp/clip.mp4")
|
||||
return extracted, ["v0"], ["s0"]
|
||||
|
||||
monkeypatch.setattr(demo, "_extract_video_input", fake_extract)
|
||||
|
||||
wav_out, frames, stacked = demo._resolve_duplex_av_inputs(
|
||||
input_wav=None,
|
||||
input_video="/tmp/clip.mp4",
|
||||
work_dir=tmp_path / "video_input",
|
||||
fps=1.0,
|
||||
max_side=0,
|
||||
)
|
||||
|
||||
assert wav_out == str(extracted)
|
||||
assert frames == ["v0"]
|
||||
assert stacked == ["s0"]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_prefers_a_video_over_a_still_frame_image(tmp_path):
|
||||
demo = _load_demo_module()
|
||||
still = tmp_path / "frame.jpg"
|
||||
still.write_bytes(b"\xff\xd8\xff-still")
|
||||
|
||||
# A still has no interior sub-frames, so it never stacks.
|
||||
assert demo._resolve_video_frames(SimpleNamespace(frame_image=str(still))) == (
|
||||
[base64.b64encode(still.read_bytes()).decode("ascii")],
|
||||
[None],
|
||||
)
|
||||
assert demo._resolve_video_frames(
|
||||
SimpleNamespace(frame_image=str(still), video_frames_b64=["preset"]),
|
||||
) == (["preset"], [None])
|
||||
assert demo._resolve_video_frames(SimpleNamespace()) == ([], [])
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_listen_only_overlap_accepts_silence_unit_before_first_done(monkeypatch):
|
||||
demo = _load_demo_module()
|
||||
state = demo.DemoState()
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import sys
|
||||
@@ -169,6 +172,64 @@ def test_open_streaming_response_requires_post_commit_drain():
|
||||
)
|
||||
|
||||
|
||||
def _asymmetric_image():
|
||||
from PIL import Image
|
||||
|
||||
image = Image.new("RGB", (4, 2), (0, 0, 0))
|
||||
image.putpixel((0, 0), (255, 0, 0))
|
||||
return image
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rotation,expected_corner",
|
||||
[
|
||||
(0, (0, 0)),
|
||||
(90, (0, 3)),
|
||||
(180, (3, 1)),
|
||||
(-90, (1, 0)),
|
||||
(270, (1, 0)),
|
||||
(-270, (0, 3)),
|
||||
],
|
||||
)
|
||||
def test_upright_applies_the_display_matrix_quarter_turn(rotation, expected_corner):
|
||||
demo = _load_demo_module()
|
||||
|
||||
rotated = demo._upright(_asymmetric_image(), rotation)
|
||||
|
||||
assert rotated.getpixel(expected_corner) == (255, 0, 0)
|
||||
|
||||
|
||||
def test_decoded_frames_apply_the_container_display_matrix(monkeypatch, tmp_path):
|
||||
"""A -90 phone clip must reach the model upright, not on its side."""
|
||||
demo = _load_demo_module()
|
||||
source = _asymmetric_image()
|
||||
|
||||
class _Frame:
|
||||
rotation = -90
|
||||
|
||||
def to_image(self):
|
||||
return source
|
||||
|
||||
class _Container:
|
||||
streams = type("_Streams", (), {"video": [type("_Stream", (), {})()]})()
|
||||
|
||||
def decode(self, stream):
|
||||
yield _Frame()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
monkeypatch.setitem(sys.modules, "av", type("_Av", (), {"open": staticmethod(lambda path: _Container())}))
|
||||
|
||||
(image,) = demo._decode_video_frames_rgb(tmp_path / "clip.mp4", [0])
|
||||
|
||||
assert image.size == (2, 4)
|
||||
assert image.getpixel((1, 0)) == (255, 0, 0)
|
||||
|
||||
|
||||
def test_streaming_output_writer_persists_audio_deltas_as_they_arrive(tmp_path, capsys):
|
||||
demo = _load_demo_module()
|
||||
writer = demo._StreamingOutputWriter(tmp_path)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
@@ -849,6 +852,161 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
|
||||
assert result["prompt_suffix_len"] == 0
|
||||
|
||||
|
||||
def _stage0_vision_runtime():
|
||||
import torch
|
||||
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
|
||||
MiniCPMO45Stage0DuplexRuntime,
|
||||
)
|
||||
|
||||
class _StageModel(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.embed = torch.nn.Embedding(256, 2)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embed
|
||||
|
||||
def get_audio_hidden_states(self, _data):
|
||||
return [torch.tensor([[0.5, 0.5]], dtype=torch.float32)]
|
||||
|
||||
runtime = MiniCPMO45Stage0DuplexRuntime.__new__(MiniCPMO45Stage0DuplexRuntime)
|
||||
runtime.stage_model = _StageModel()
|
||||
runtime.thinker = runtime.stage_model
|
||||
runtime.tokenizer = SimpleNamespace(
|
||||
unk_token_id=0,
|
||||
convert_tokens_to_ids=lambda token: {
|
||||
"<unit>": 1,
|
||||
"</unit>": 2,
|
||||
"<|listen|>": 3,
|
||||
"<|speak|>": 4,
|
||||
"<|tts_bos|>": 5,
|
||||
"<|tts_eos|>": 6,
|
||||
"<|tts_pad|>": 7,
|
||||
"<|chunk_eos|>": 8,
|
||||
"<|chunk_tts_eos|>": 9,
|
||||
"<|turn_eos|>": 10,
|
||||
"<|audio|>": 11,
|
||||
"<image>": 12,
|
||||
"</image>": 13,
|
||||
"<slice>": 14,
|
||||
"</slice>": 15,
|
||||
}.get(token, 0),
|
||||
encode=lambda text, add_special_tokens=False: [201, 5],
|
||||
)
|
||||
runtime.processor = SimpleNamespace(get_streaming_chunk_size=lambda: 4)
|
||||
runtime.device = "cpu"
|
||||
runtime._init_token_ids()
|
||||
# One 2-row block per frame keeps the expected token_ids readable; the real
|
||||
# resampler emits VISION_EMBEDS_PER_FRAME rows per slice.
|
||||
runtime._stage_vision_embeddings = lambda frames: [
|
||||
[torch.full((2, 2), float(index), dtype=torch.float32)] for index in range(len(frames))
|
||||
]
|
||||
return runtime
|
||||
|
||||
|
||||
def test_minicpmo_stage0_puts_every_frame_of_an_append_in_one_unit():
|
||||
"""Official streaming_prefill feeds the whole frame_list into one unit.
|
||||
|
||||
Stacked pairs used to be spread one frame per unit, so a two-frame append
|
||||
that closed a single unit left a frame over, failed the prefill, and still
|
||||
advanced the session state.
|
||||
"""
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-stacked-frames")
|
||||
|
||||
result = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(4, dtype=np.float32),
|
||||
video_frames=["frame-prev", "frame-cur"],
|
||||
seq=1,
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
# <unit>, then each frame as <image> + embeds + </image>, then the audio.
|
||||
assert result["input_token_ids"] == [1, 12, 0, 0, 13, 12, 0, 0, 13, 11]
|
||||
assert state.audio_chunk_idx == 1
|
||||
|
||||
|
||||
def test_minicpmo_stage0_wraps_hd_slices_after_the_source_image():
|
||||
"""Official stacked pair is max_slice_nums=[2, 1]: source + slices, then audio."""
|
||||
import torch
|
||||
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-hd-slices")
|
||||
runtime._stage_vision_embeddings = lambda frames: [
|
||||
[torch.ones((2, 2)), torch.full((2, 2), 2.0), torch.full((2, 2), 3.0)],
|
||||
[torch.full((2, 2), 4.0)],
|
||||
]
|
||||
|
||||
result = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(4, dtype=np.float32),
|
||||
video_frames=["base", "stack"],
|
||||
seq=1,
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
# <unit> <image> src </image> <slice> p0 </slice> <slice> p1 </slice>
|
||||
# <image> stack </image> audio
|
||||
assert result["input_token_ids"] == [
|
||||
1,
|
||||
12,
|
||||
0,
|
||||
0,
|
||||
13,
|
||||
14,
|
||||
0,
|
||||
0,
|
||||
15,
|
||||
14,
|
||||
0,
|
||||
0,
|
||||
15,
|
||||
12,
|
||||
0,
|
||||
0,
|
||||
13,
|
||||
11,
|
||||
]
|
||||
|
||||
|
||||
def test_minicpmo_stage0_reports_frames_that_close_no_unit_instead_of_dropping_them():
|
||||
"""A frame only binds to the unit its append closes.
|
||||
|
||||
Frames are not carried across appends, so one arriving early used to vanish
|
||||
under a generic "audio not enough" buffering result.
|
||||
"""
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
|
||||
_MiniCPMO45Stage0SessionState,
|
||||
)
|
||||
|
||||
runtime = _stage0_vision_runtime()
|
||||
state = _MiniCPMO45Stage0SessionState(session_id="sid-stage0-early-frame")
|
||||
|
||||
assert runtime._stage_prefill_embeddings_only(state, np.zeros(4, dtype=np.float32), seq=1)["success"] is True
|
||||
|
||||
early = runtime._stage_prefill_embeddings_only(
|
||||
state,
|
||||
np.zeros(2, dtype=np.float32),
|
||||
video_frames=["frame-cur"],
|
||||
seq=2,
|
||||
)
|
||||
|
||||
assert early["success"] is False
|
||||
assert "closes no model unit" in early["reason"]
|
||||
# The half-filled append must not advance the unit counter.
|
||||
assert state.audio_chunk_idx == 1
|
||||
|
||||
|
||||
def test_minicpmo_stage0_speech_append_sets_pending_context_once():
|
||||
import torch
|
||||
|
||||
@@ -2514,7 +2672,7 @@ def _ar_runner_classdefs():
|
||||
|
||||
|
||||
def _method_calls(method):
|
||||
calls = {}
|
||||
calls: dict[str, int] = {}
|
||||
for node in ast.walk(method):
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
||||
calls.setdefault(node.func.attr, node.lineno)
|
||||
|
||||
Reference in new issue
Block a user