Merge branch 'main' into qwen-image-2.1
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
51 files changed
+2189
-70
No files matched your search
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Drive complete native duplex turns while Stage-0 repeatedly rebuilds KV."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
from vllm_omni.clients.duplex import DuplexClient, EventCollector, read_pcm16_wav, wait_for_condition
|
||||
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
|
||||
|
||||
|
||||
async def run_window_turn(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
mode: str,
|
||||
ref_audio: Path | None = None,
|
||||
repeats: int = 2,
|
||||
video_frames: list[str] | None = None,
|
||||
buffered_flush: bool = False,
|
||||
timeout_s: float = 240.0,
|
||||
) -> dict[str, object]:
|
||||
"""Use aggressive limits to exercise multiple windows in one audio turn."""
|
||||
pcm = read_pcm16_wav(input_wav) * repeats
|
||||
collector = EventCollector()
|
||||
reference = (
|
||||
"data:audio/wav;base64," + base64.b64encode(ref_audio.read_bytes()).decode("ascii")
|
||||
if ref_audio is not None
|
||||
else None
|
||||
)
|
||||
config = create_duplex_session_config(
|
||||
modalities=("text", "audio") if ref_audio is not None else ("text",),
|
||||
ref_audio=reference,
|
||||
temperature=0.0,
|
||||
extra_body={
|
||||
"sliding_window_mode": mode,
|
||||
"context_max_units": 1,
|
||||
"context_previous_max_tokens": 16,
|
||||
"basic_window_high_tokens": 64,
|
||||
"basic_window_low_tokens": 32,
|
||||
},
|
||||
)
|
||||
client = DuplexClient(
|
||||
url, model=model, config=config, reconnect=None, heartbeat_interval_s=None, handshake_timeout_s=timeout_s
|
||||
)
|
||||
async with client:
|
||||
reader = asyncio.create_task(collector.consume(client))
|
||||
collector.add({"type": "session.created", "session": client.session_info})
|
||||
try:
|
||||
frames_sent = await client.stream_pcm(
|
||||
pcm,
|
||||
chunk_ms=2500 if buffered_flush else 200,
|
||||
realtime=not buffered_flush,
|
||||
is_speech=True,
|
||||
video_frames=video_frames,
|
||||
)
|
||||
await client.commit(final=True)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()),
|
||||
timeout_s=timeout_s,
|
||||
label="final input commit",
|
||||
)
|
||||
await wait_for_condition(
|
||||
lambda: collector.count("response.done") > 0 or bool(collector.errors()),
|
||||
timeout_s=timeout_s,
|
||||
label="completed windowed response",
|
||||
)
|
||||
# Final execution follows the commit acknowledgement asynchronously.
|
||||
await asyncio.sleep(3.0)
|
||||
await client.close(timeout_s=timeout_s)
|
||||
await asyncio.wait_for(reader, timeout=5.0)
|
||||
finally:
|
||||
if not reader.done():
|
||||
reader.cancel()
|
||||
await asyncio.gather(reader, return_exceptions=True)
|
||||
return {
|
||||
"created": collector.count("session.created"),
|
||||
"committed": collector.count("input_audio_buffer.committed"),
|
||||
"done": collector.count("response.done"),
|
||||
"closed": collector.count("session.closed"),
|
||||
"errors": collector.errors(),
|
||||
"audio_bytes": len(collector.audio_bytes()),
|
||||
"frames_sent": frames_sent,
|
||||
"input_seconds": len(pcm) / 32000,
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Real-model regression for the scheduler-owned Thinker sliding window."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
duplex_camera_frames,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
)
|
||||
from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.advanced_model]
|
||||
|
||||
|
||||
def _assert_complete(result, *, require_audio=False):
|
||||
assert result["created"] == 1
|
||||
assert result["committed"] > 0
|
||||
assert result["done"] > 0
|
||||
assert result["closed"] == 1
|
||||
assert result["errors"] == []
|
||||
if require_audio:
|
||||
assert result["audio_bytes"] > 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_rebuild_and_next_session(omni_server, mode, reference):
|
||||
# Two independent sessions exercise admission after final-unit execution.
|
||||
for _ in range(2):
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
ref_audio=resolve_ref_audio() if reference else None,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=reference)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_continuous_input(omni_server, tmp_path, mode, camera):
|
||||
# 24 repeats cover over two minutes and many canonical KV replacements.
|
||||
frames = duplex_camera_frames(seconds=4, cache_dir=tmp_path / "camera") if camera else None
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
repeats=24,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
video_frames=frames,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=True)
|
||||
assert isinstance(result["input_seconds"], (int, float)) and result["input_seconds"] > 120
|
||||
if camera:
|
||||
assert isinstance(result["frames_sent"], int) and result["frames_sent"] >= 120
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode", ["off", "basic", "context"])
|
||||
def test_window_buffered_flush(omni_server, mode):
|
||||
# Burst appends cross processor chunk boundaries and leave a final tail.
|
||||
result = asyncio.run(
|
||||
run_window_turn(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=validated_input_wav(),
|
||||
mode=mode,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
buffered_flush=True,
|
||||
)
|
||||
)
|
||||
_assert_complete(result, require_audio=True)
|
||||
Reference in new issue
Block a user