[Full Duplex] Feat: Support Full-Duplex realtime runtime & add MiniCPM-o 4.5 demo (#3907)

Signed-off-by: Sy03 <1370724210@qq.com>
Signed-off-by: lishunyang12 <lishunyang03@gmail.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: StormMelody <22371184@zju.edu.cn>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-07-23 16:45:18 +08:00
1 parent b55120c93b
commit 05b794f29b
142 files changed
+48389 -1013

No files matched your search

@@ -138,6 +138,112 @@ async def test_receive_config_accepts_client_legacy_aliases():
assert config.frame_filter_threshold == 0.87
@pytest.mark.asyncio
async def test_video_frame_ack_reports_receiver_buffer_state():
ws = MockWebSocket(
[
json.dumps(
{
"type": "session.config",
"model": "test",
"enable_frame_filter": False,
}
),
json.dumps(
{
"type": "video.frame",
"data": _b64(_make_jpeg()),
"frame_id": "frame-7",
"pts_ms": 700,
"capture_ts_ms": 1234.5,
}
),
json.dumps({"type": "video.done"}),
]
)
handler = QwenOmniStreamingVideoHandler(chat_service=object())
await handler.handle_session(ws)
ack = next(message for message in ws.sent if message.get("type") == "video.frame.ack")
assert ack["frame_id"] == "frame-7"
assert ack["pts_ms"] == 700
assert ack["capture_ts_ms"] == 1234.5
assert ack["accepted"] is True
assert ack["buffered_frames"] == 1
assert ack["server_receive_ts_ms"] > 0
@pytest.mark.asyncio
async def test_video_frames_consumed_is_emitted_after_engine_uses_frame_prompt():
class OneOutputEngine:
def generate(self, **_kwargs):
async def _gen():
await asyncio.sleep(0.05)
yield _text_result("visible")
return _gen()
class CapturingHandler(QwenOmniStreamingVideoHandler):
async def _preprocess_to_engine_prompt(self, request):
return {"prompt": "with-video"}
ws = MockWebSocket(
[
json.dumps(
{
"type": "session.config",
"model": "test",
"modalities": ["text"],
"num_frames": 1,
"enable_frame_filter": False,
}
),
json.dumps(
{
"type": "video.frame",
"data": _b64(_make_jpeg()),
"frame_id": "frame-9",
"pts_ms": 900,
"source_pts_ms": 880,
"quality_profile": "balanced",
}
),
json.dumps({"type": "video.query", "text": "describe"}),
json.dumps({"type": "video.done"}),
]
)
handler = CapturingHandler(
chat_service=object(),
engine_client=OneOutputEngine(),
idle_timeout=2.0,
)
await handler.handle_session(ws)
consumed = next(message for message in ws.sent if message.get("type") == "video.frames.consumed")
assert consumed["frame_ids"] == ["frame-9"]
assert consumed["latest_pts_ms"] == 900
assert consumed["request_id"].startswith("video-")
assert consumed["model_selected_ts_ms"] > 0
assert consumed["frames"] == [
{
"frame_id": "frame-9",
"pts_ms": 900,
"source_pts_ms": 880,
"quality_profile": "balanced",
"receiver_received_ts_ms": consumed["frames"][0]["receiver_received_ts_ms"],
"decoded_ready_ts_ms": consumed["frames"][0]["decoded_ready_ts_ms"],
}
]
assert consumed["frames"][0]["receiver_received_ts_ms"] > 0
assert consumed["frames"][0]["decoded_ready_ts_ms"] >= consumed["frames"][0]["receiver_received_ts_ms"]
assert consumed["model_selected_ts_ms"] >= consumed["frames"][0]["decoded_ready_ts_ms"]
assert ws.sent.index(consumed) < next(
index for index, message in enumerate(ws.sent) if message.get("type") == "response.text.delta"
)
@pytest.mark.asyncio
async def test_audio_in_video_sets_mm_processor_kwargs():
captured_requests = []