Merge remote-tracking branch 'origin/main' into qwen-image-2.1
# Conflicts: # vllm_omni/diffusion/executor/multiproc_executor.py # vllm_omni/diffusion/offloader/sequential_backend.py
This commit is contained in:
commit
9174554e40
1332 files changed
+160051
-60504
No files matched your search
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.audio import convert_input_audio_with_rate
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_pcm16_input_conversion_is_pure_and_reports_target_rate():
|
||||
pcm16 = np.array([-32768, 0, 32767], dtype="<i2").tobytes()
|
||||
encoded = base64.b64encode(pcm16).decode("ascii")
|
||||
|
||||
converted, fmt, sample_rate = convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
|
||||
assert encoded == base64.b64encode(pcm16).decode("ascii")
|
||||
assert fmt == "pcm_f32le"
|
||||
assert sample_rate == 16_000
|
||||
samples = np.frombuffer(base64.b64decode(converted), dtype="<f4")
|
||||
assert np.allclose(samples, [-1.0, 0.0, 32767 / 32768])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz", [8_000, 192_000])
|
||||
def test_pcm16_input_conversion_accepts_supported_sample_rate_boundaries(sample_rate_hz: int):
|
||||
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
|
||||
|
||||
_, fmt, converted_rate = convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=sample_rate_hz,
|
||||
)
|
||||
|
||||
assert fmt == "pcm_f32le"
|
||||
assert converted_rate == 16_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz", [7_999, 192_001, 10**1000])
|
||||
def test_pcm16_input_conversion_rejects_sample_rates_outside_supported_range(sample_rate_hz: int):
|
||||
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
|
||||
|
||||
with pytest.raises(ValueError, match="sample_rate_hz"):
|
||||
convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=sample_rate_hz,
|
||||
)
|
||||
@@ -0,0 +1,609 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The thin websocket handler: handshake, command translation, event pump, resume/takeover."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import WebSocketDisconnect
|
||||
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex import commands
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities
|
||||
from vllm_omni.engine.duplex.events import AudioDelta, DuplexEvent, SessionClosed, SessionCreated
|
||||
from vllm_omni.engine.duplex.messages import DuplexSessionError
|
||||
from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
from vllm_omni.entrypoints.duplex.websocket import MAX_EVENT_BYTES
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_DISCONNECT = object()
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self, query: dict[str, str] | None = None) -> None:
|
||||
self.query_params = dict(query or {})
|
||||
self.sent: list[dict[str, Any]] = []
|
||||
self.accepted = False
|
||||
self.closed: list[tuple[int, str]] = []
|
||||
self._send_failure: str | None = None
|
||||
self._inbound: asyncio.Queue[Any] = asyncio.Queue()
|
||||
|
||||
async def accept(self) -> None:
|
||||
self.accepted = True
|
||||
|
||||
async def send_json(self, payload: dict[str, Any]) -> None:
|
||||
if self.closed or self._send_failure is not None:
|
||||
raise RuntimeError(
|
||||
self._send_failure or "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'."
|
||||
)
|
||||
self.sent.append(json.loads(json.dumps(payload)))
|
||||
|
||||
async def receive_text(self) -> str:
|
||||
item = await self._inbound.get()
|
||||
if item is _DISCONNECT:
|
||||
raise WebSocketDisconnect(code=1000)
|
||||
return item
|
||||
|
||||
async def close(self, code: int = 1000, reason: str = "") -> None:
|
||||
self.closed.append((code, reason))
|
||||
# A closed socket ends the reader exactly like a client disconnect.
|
||||
self._inbound.put_nowait(_DISCONNECT)
|
||||
|
||||
# ---- test helpers ----
|
||||
|
||||
def feed(self, payload: dict[str, Any] | str) -> None:
|
||||
self._inbound.put_nowait(payload if isinstance(payload, str) else json.dumps(payload))
|
||||
|
||||
def disconnect(self) -> None:
|
||||
self._inbound.put_nowait(_DISCONNECT)
|
||||
|
||||
def break_sends(self) -> None:
|
||||
"""Kill the write half only: the reader stays parked, as it does in practice."""
|
||||
self._send_failure = "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'."
|
||||
|
||||
def types(self) -> list[str]:
|
||||
return [payload["type"] for payload in self.sent]
|
||||
|
||||
async def wait_for(self, wire_type: str, *, timeout_s: float = 2.0) -> dict[str, Any]:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout_s
|
||||
while loop.time() < deadline:
|
||||
for payload in self.sent:
|
||||
if payload["type"] == wire_type:
|
||||
return payload
|
||||
await asyncio.sleep(0.005)
|
||||
raise AssertionError(f"no {wire_type!r} in {self.types()}")
|
||||
|
||||
|
||||
class FakeHandle:
|
||||
def __init__(self, session_id: str, capabilities: DuplexCapabilities, *, idle_timeout_s: float = 300) -> None:
|
||||
self.session_id = session_id
|
||||
self.capabilities = capabilities
|
||||
self.public_session: dict[str, Any] = {"id": session_id, "idle_timeout_s": idle_timeout_s}
|
||||
self.lease_generation = 0
|
||||
self.closed = False
|
||||
self.close_reasons: list[str] = []
|
||||
self.commands: list[commands.DuplexCommand] = []
|
||||
self._outbox: asyncio.Queue[DuplexEvent] = asyncio.Queue()
|
||||
|
||||
def deliver(self, event: DuplexEvent) -> None:
|
||||
self._outbox.put_nowait(event)
|
||||
|
||||
async def submit(self, command: commands.DuplexCommand) -> None:
|
||||
if self.closed:
|
||||
raise DuplexSessionError("closed", code="session_closed", session_id=self.session_id)
|
||||
self.commands.append(command)
|
||||
|
||||
async def events(self):
|
||||
while True:
|
||||
event = await self._outbox.get()
|
||||
yield event
|
||||
if event.is_terminal:
|
||||
return
|
||||
|
||||
async def close(self, *, reason: str = "client_close", timeout: float | None = None) -> None:
|
||||
self.close_reasons.append(reason)
|
||||
if self.closed:
|
||||
return
|
||||
self.closed = True
|
||||
self.deliver(SessionClosed(session_id=self.session_id, reason=reason))
|
||||
|
||||
|
||||
class FakeOmni:
|
||||
def __init__(
|
||||
self, *, resumable: bool = True, replay_max_bytes: int = 64 * 1024, idle_timeout_s: float = 300
|
||||
) -> None:
|
||||
self.duplex_session_config = DuplexSessionRuntimeConfig(
|
||||
resume_replay_ttl_s=60.0, resume_replay_max_bytes_per_session=replay_max_bytes
|
||||
)
|
||||
self.capabilities = DuplexCapabilities(supports_session_resume=resumable)
|
||||
self.idle_timeout_s = idle_timeout_s
|
||||
self.opened: list[dict[str, Any]] = []
|
||||
self.handles: dict[str, FakeHandle] = {}
|
||||
self.resumed: list[tuple[str, int]] = []
|
||||
self.detached: list[str] = []
|
||||
self.open_error: DuplexSessionError | None = None
|
||||
|
||||
async def open_session(self, config: Any) -> FakeHandle:
|
||||
self.opened.append(dict(config))
|
||||
if self.open_error is not None:
|
||||
raise self.open_error
|
||||
session_id = f"duplex-{len(self.handles) + 1:032x}"
|
||||
handle = FakeHandle(session_id, self.capabilities, idle_timeout_s=self.idle_timeout_s)
|
||||
self.handles[session_id] = handle
|
||||
handle.deliver(SessionCreated(session_id=session_id, session={"id": session_id, "model": config.get("model")}))
|
||||
return handle
|
||||
|
||||
def get_session(self, session_id: str) -> FakeHandle | None:
|
||||
return self.handles.get(session_id)
|
||||
|
||||
async def resume_session(self, session_id: str, *, expected_lease_generation: int) -> FakeHandle:
|
||||
self.resumed.append((session_id, expected_lease_generation))
|
||||
handle = self.handles[session_id]
|
||||
handle.lease_generation = expected_lease_generation + 1
|
||||
return handle
|
||||
|
||||
async def detach_session(self, session_id: str) -> None:
|
||||
self.detached.append(session_id)
|
||||
|
||||
|
||||
def _handler(omni: FakeOmni, **kwargs: Any) -> OmniDuplexSessionHandler:
|
||||
kwargs.setdefault("config_timeout_s", 1.0)
|
||||
kwargs.setdefault("idle_timeout_s", 5.0)
|
||||
return OmniDuplexSessionHandler(duplex_omni=omni, **kwargs)
|
||||
|
||||
|
||||
def _session_update(**session: Any) -> dict[str, Any]:
|
||||
return {"type": "session.update", "session": {"model": "test-model", "modalities": ["audio", "text"], **session}}
|
||||
|
||||
|
||||
async def _open(
|
||||
handler: OmniDuplexSessionHandler, omni: FakeOmni, **session: Any
|
||||
) -> tuple[FakeWebSocket, FakeHandle, asyncio.Task]:
|
||||
ws = FakeWebSocket({"duplex": "1", "autostart": "0"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed(_session_update(**session))
|
||||
created = await ws.wait_for("session.created")
|
||||
handle = omni.handles[created["session"]["id"]]
|
||||
return ws, handle, task
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Handshake and commands #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_update_opens_a_session_and_announces_the_server_id() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni, session_id="mine", instructions="hi")
|
||||
|
||||
assert ws.accepted
|
||||
created = ws.sent[0]
|
||||
assert created["type"] == "session.created"
|
||||
assert created["session"]["id"] == handle.session_id != "mine"
|
||||
assert created["attachment_generation"] == 1
|
||||
assert isinstance(created["resume_token"], str) and created["resume_token"]
|
||||
assert "incarnation" not in created
|
||||
assert "server_event_seq" not in created
|
||||
# The whole session object went to open_session; the id inside it is ignored there.
|
||||
assert omni.opened == [
|
||||
{"model": "test-model", "modalities": ["audio", "text"], "session_id": "mine", "instructions": "hi"}
|
||||
]
|
||||
|
||||
ws.feed({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 8).decode("ascii")})
|
||||
ws.feed({"type": "input_audio_buffer.commit", "event_id": "evt-c"})
|
||||
await asyncio.sleep(0.05)
|
||||
assert [type(command) for command in handle.commands] == [commands.AppendAudio, commands.Commit]
|
||||
assert handle.commands[1].event_id == "evt-c"
|
||||
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
# A resumable session is detached (engine-owned grace), not closed.
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert handle.close_reasons == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_events_are_journaled_and_sent_in_order() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
delta = await ws.wait_for("response.output_audio.delta")
|
||||
assert delta["delta"] == "aGk=" and delta["server_event_seq"] == 1
|
||||
|
||||
ws.feed({"type": "session.event_ack", "server_event_seq": 1})
|
||||
ws.feed({"type": "session.event_ack", "server_event_seq": 9, "event_id": "evt-ack"})
|
||||
error = await ws.wait_for("error")
|
||||
assert error["error"]["code"] == "invalid_event_ack" and error["error"]["event_id"] == "evt-ack"
|
||||
|
||||
await handle.close(reason="client_close")
|
||||
closed = await ws.wait_for("session.closed")
|
||||
assert closed["reason"] == "client_close"
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.closed[0][1] == "client_close"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_envelope_errors_are_answered_locally_without_touching_the_session() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
ws.feed("not json")
|
||||
ws.feed("x" * (MAX_EVENT_BYTES + 1))
|
||||
ws.feed({"type": "totally.unknown", "event_id": "evt-u"})
|
||||
ws.feed({"type": "session.resume", "event_id": "evt-r"})
|
||||
ws.feed({"type": "playback.ack", "event_id": "evt-p"}) # played_ms missing
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
codes = [payload["error"]["code"] for payload in ws.sent if payload["type"] == "error"]
|
||||
assert codes == ["invalid_json", "event_too_large", "unknown_event", "unsupported_session_resume", "bad_event"]
|
||||
errors = [payload for payload in ws.sent if payload["type"] == "error"]
|
||||
assert errors[2]["error"]["event_id"] == "evt-u"
|
||||
assert errors[4]["error"]["event_id"] == "evt-p"
|
||||
assert handle.commands == []
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_rejection_is_reported_and_the_socket_ends() -> None:
|
||||
omni = FakeOmni()
|
||||
omni.open_error = DuplexSessionError("no room", code="resource_exhausted", retryable=True)
|
||||
handler = _handler(omni)
|
||||
ws = FakeWebSocket({"duplex": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed(_session_update())
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.types() == ["error"]
|
||||
assert ws.sent[0]["error"]["code"] == "resource_exhausted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_timeout_and_first_message_validation() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni, config_timeout_s=0.05)
|
||||
ws = FakeWebSocket({"duplex": "1"})
|
||||
await asyncio.wait_for(handler.handle_realtime_session(ws), timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "config_timeout"
|
||||
|
||||
ws = FakeWebSocket({"duplex": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed("{not json")
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "invalid_json"
|
||||
assert omni.opened == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_timeout_detaches_a_resumable_session_like_a_disconnect() -> None:
|
||||
omni = FakeOmni(idle_timeout_s=0.05)
|
||||
handler = _handler(omni, idle_timeout_s=0.05)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
# Serving makes no session-lifetime decision: the engine lease decides
|
||||
# whether a silent, detached session expires.
|
||||
assert handle.close_reasons == []
|
||||
assert omni.detached == [handle.session_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_timeout_closes_a_non_resumable_session() -> None:
|
||||
omni = FakeOmni(idle_timeout_s=0.05, resumable=False)
|
||||
handler = _handler(omni, idle_timeout_s=0.05)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert handle.close_reasons == ["disconnect"]
|
||||
assert omni.detached == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transport_send_failure_detaches_the_session_instead_of_closing_it() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
# Only the write half dies, so the pump's send is the one and only report
|
||||
# of the broken socket (the reader is still parked on receive).
|
||||
ws.break_sends()
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
# The engine session is alive and resumable, so this is a disconnect of
|
||||
# the current attachment, not a session close; the pump keeps journaling.
|
||||
assert handle.close_reasons == []
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert not handler._pumps[handle.session_id].done()
|
||||
|
||||
# The reader sees the same broken socket a moment later. The attachment is
|
||||
# already gone, so this must not detach a second time: another detach would
|
||||
# restart the engine's disconnect grace window.
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert handle.close_reasons == []
|
||||
|
||||
handle.deliver(SessionClosed(session_id=handle.session_id, reason="client_close"))
|
||||
await asyncio.sleep(0.05)
|
||||
assert handle.session_id not in handler._pumps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_resumable_session_is_closed_on_disconnect() -> None:
|
||||
omni = FakeOmni(resumable=False)
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
assert "resume_token" not in ws.sent[0]
|
||||
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert handle.close_reasons == ["disconnect"]
|
||||
assert omni.detached == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_client_close_still_delivers_session_closed_before_the_socket_goes() -> None:
|
||||
"""The read loop must not outrun the pump's terminal event.
|
||||
|
||||
``DuplexSessionHandle._deliver`` queues ``SessionClosed`` and marks the
|
||||
handle closed in one synchronous step, so ``handle.closed`` is already true
|
||||
while the event is still sitting in the outbox. ``_read_loop`` loops on
|
||||
exactly that flag: it returned, the endpoint returned, and the ASGI server
|
||||
tore the socket down with ``session.closed`` unsent -- the client saw an
|
||||
abrupt close (no close frame) instead of the terminal event it was waiting
|
||||
for.
|
||||
|
||||
The teardown is what makes this observable, so it is modelled here: a real
|
||||
connection stops accepting writes the moment the endpoint returns, which is
|
||||
why a pump that is merely *still scheduled* is already too late.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws = FakeWebSocket({"duplex": "1", "autostart": "0"})
|
||||
|
||||
async def serve() -> None:
|
||||
try:
|
||||
await handler.handle_realtime_session(ws)
|
||||
finally:
|
||||
# The ASGI server drops the connection when the endpoint returns;
|
||||
# anything the pump writes after this never reaches the client.
|
||||
ws.break_sends()
|
||||
|
||||
task = asyncio.create_task(serve())
|
||||
ws.feed(_session_update())
|
||||
created = await ws.wait_for("session.created")
|
||||
handle = omni.handles[created["session"]["id"]]
|
||||
submit = handle.submit
|
||||
|
||||
async def closing_submit(command: commands.DuplexCommand) -> None:
|
||||
await submit(command)
|
||||
if isinstance(command, commands.CloseSession):
|
||||
# Same order as the real handle: queue the event, then flip the flag.
|
||||
handle.deliver(SessionClosed(session_id=handle.session_id, reason="client_close"))
|
||||
handle.closed = True
|
||||
|
||||
handle.submit = closing_submit # type: ignore[method-assign]
|
||||
|
||||
ws.feed({"type": "session.close"})
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
assert ws.types()[-1] == "session.closed", f"terminal event never reached the wire: {ws.types()}"
|
||||
assert ws.closed and ws.closed[0][0] == 1000, "the socket must close with a normal close frame"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Resume and takeover #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def _resume(
|
||||
handler: OmniDuplexSessionHandler,
|
||||
session_id: str,
|
||||
token: str,
|
||||
*,
|
||||
last_seq: int = 0,
|
||||
) -> tuple[FakeWebSocket, asyncio.Task]:
|
||||
ws = FakeWebSocket({"duplex": "1", "resume": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": session_id,
|
||||
"resume_token": token,
|
||||
"last_received_server_event_seq": last_seq,
|
||||
}
|
||||
)
|
||||
return ws, task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_after_disconnect_replays_missed_events_and_rotates_the_token() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="one"))
|
||||
await ws.wait_for("response.output_audio.delta")
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
# Emitted while detached: journaled for replay only.
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="two"))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
ws2, task2 = await _resume(handler, handle.session_id, token, last_seq=1)
|
||||
resumed = await ws2.wait_for("session.resumed")
|
||||
assert resumed["session_id"] == handle.session_id
|
||||
assert resumed["attachment_generation"] == 2
|
||||
assert resumed["resume_token"] != token
|
||||
assert "incarnation" not in resumed
|
||||
replayed = await ws2.wait_for("response.output_audio.delta")
|
||||
assert replayed["delta"] == "two" and replayed["server_event_seq"] == 2
|
||||
assert omni.resumed == [(handle.session_id, 0)]
|
||||
|
||||
# The old token is revoked by the rotation.
|
||||
ws3, task3 = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(task3, timeout=2.0)
|
||||
assert ws3.sent[0]["error"]["code"] == "invalid_resume_token"
|
||||
|
||||
ws2.feed({"type": "session.close"})
|
||||
await asyncio.sleep(0.05)
|
||||
ws2.disconnect()
|
||||
await asyncio.wait_for(task2, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_takes_over_a_live_attachment() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
|
||||
ws2, task2 = await _resume(handler, handle.session_id, token)
|
||||
await ws2.wait_for("session.resumed")
|
||||
replaced = await ws.wait_for("session.replaced")
|
||||
assert replaced["attachment_generation"] == 1
|
||||
assert ws.closed and ws.closed[0][1] == "session_replaced"
|
||||
# The replaced socket's later input is ignored; the new one drives the session.
|
||||
ws.feed({"type": "input_audio_buffer.clear"})
|
||||
ws2.feed({"type": "input_audio_buffer.clear", "event_id": "evt-new"})
|
||||
await asyncio.sleep(0.05)
|
||||
assert [command.event_id for command in handle.commands] == ["evt-new"]
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
ws2.disconnect()
|
||||
await asyncio.wait_for(task2, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_validation_errors() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws = FakeWebSocket({"duplex": "1", "resume": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed({"type": "session.resume", "session_id": "duplex-x"})
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "invalid_session_resume"
|
||||
|
||||
ws, task = await _resume(handler, "duplex-unknown", "token")
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "session_resume_expired"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_journal_overflow_degrades_to_live_delivery_with_resync_required() -> None:
|
||||
omni = FakeOmni(replay_max_bytes=256)
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="x" * 400))
|
||||
resync = await ws.wait_for("session.resync_required")
|
||||
assert resync["reason"] == "journal_overflow"
|
||||
delta = await ws.wait_for("response.output_audio.delta")
|
||||
assert "server_event_seq" not in delta
|
||||
assert ws.types().index("session.resync_required") < ws.types().index("response.output_audio.delta")
|
||||
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Envelope helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_realtime_envelope_query_rules() -> None:
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m"})
|
||||
assert envelope.initial_open_payload() == {"model": "m"}
|
||||
assert envelope.initial_open_payload() is None # autostart happens once
|
||||
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
assert envelope.resume_only is True
|
||||
assert envelope.initial_open_payload() is None
|
||||
assert RealtimeEnvelope.from_query_params({"resume": "1"}).resume_only is True
|
||||
assert RealtimeEnvelope.from_query_params({"session_id": "mine"}).default_session_payload() == {"model": None}
|
||||
|
||||
|
||||
def test_realtime_envelope_first_message_classification() -> None:
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
resume = envelope.first_message({"type": "session.resume", "session_id": "s"})
|
||||
assert resume.kind == "resume" and resume.resume_payload["session_id"] == "s"
|
||||
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
opened = envelope.first_message({"type": "session.update", "session": {"model": "x", "instructions": "hi"}})
|
||||
assert opened.kind == "open" and opened.session_payload == {"model": "x", "instructions": "hi"}
|
||||
assert opened.pending_command_payload is None
|
||||
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
autostarted = envelope.first_message({"type": "input_audio_buffer.commit"})
|
||||
assert autostarted.kind == "open" and autostarted.session_payload == {"model": "m"}
|
||||
assert autostarted.pending_command_payload == {"type": "input_audio_buffer.commit"}
|
||||
|
||||
|
||||
def test_parse_resume_request_requires_the_three_fields_only() -> None:
|
||||
request = parse_resume_request({"session_id": "s", "resume_token": "t", "last_received_server_event_seq": 3})
|
||||
assert request is not None
|
||||
assert (request.session_id, request.resume_token, request.last_received_server_event_seq) == ("s", "t", 3)
|
||||
assert parse_resume_request({"session_id": "s", "resume_token": "t"}).last_received_server_event_seq == 0
|
||||
assert parse_resume_request({"session_id": "s", "resume_token": "t", "incarnation": 1}) is not None
|
||||
assert parse_resume_request({"session_id": "s"}) is None
|
||||
assert parse_resume_request({"session_id": "s", "resume_token": "t", "last_received_server_event_seq": -1}) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_that_fails_to_activate_does_not_detach_the_live_attachment() -> None:
|
||||
"""A rejected resume must roll back only what it owns.
|
||||
|
||||
Two reconnects can both authenticate and both complete the engine resume;
|
||||
only one activates. The loser's activation raises, and the rollback used to
|
||||
detach the *session*, starting the engine's disconnect grace for the winner
|
||||
-- a grace ordinary heartbeats do not clear.
|
||||
|
||||
The activation failure is injected rather than raced for: what is under test
|
||||
is the rollback's precondition, not the window that produces it.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
try:
|
||||
assert await registry.has_attachment(handle.session_id), "the opener is attached"
|
||||
detached_before = list(omni.detached)
|
||||
|
||||
async def failing_resume(*args, **kwargs):
|
||||
raise RuntimeError("transport activation lost the race")
|
||||
|
||||
original_resume = registry.resume
|
||||
registry.resume = failing_resume # type: ignore[method-assign]
|
||||
try:
|
||||
ws_lose, task_lose = await _resume(handler, handle.session_id, token)
|
||||
error = await ws_lose.wait_for("error")
|
||||
finally:
|
||||
registry.resume = original_resume # type: ignore[method-assign]
|
||||
|
||||
assert error["error"]["code"] == "session_resume_conflict"
|
||||
assert omni.detached == detached_before, "the loser must not detach the winner"
|
||||
assert await registry.has_attachment(handle.session_id), "the winner is still attached"
|
||||
finally:
|
||||
for pending in (task, locals().get("task_lose")):
|
||||
if pending is not None:
|
||||
pending.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await pending
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.runtime_adapter import (
|
||||
validate_serving_runtime_adapter,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name",
|
||||
[
|
||||
"vllm_omni.entrypoints.duplex.session_runner",
|
||||
"vllm_omni.entrypoints.duplex.runtime_bridge",
|
||||
"vllm_omni.entrypoints.duplex.serving",
|
||||
],
|
||||
)
|
||||
def test_generic_openai_runtime_import_does_not_load_model_adapters(module_name: str) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = f"""
|
||||
import sys
|
||||
|
||||
import {module_name}
|
||||
|
||||
model_duplex_prefixes = (
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
|
||||
"vllm_omni.model_executor.models.nemotron_voicechat.duplex",
|
||||
"vllm_omni.model_executor.models.personaplex.duplex",
|
||||
)
|
||||
loaded = sorted(
|
||||
name
|
||||
for name in sys.modules
|
||||
if any(
|
||||
name == prefix or name.startswith(prefix + ".")
|
||||
for prefix in model_duplex_prefixes
|
||||
)
|
||||
)
|
||||
if loaded:
|
||||
raise SystemExit("generic OpenAI runtime loaded model duplex modules: " + ", ".join(loaded))
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_generic_handler_supports_adapterless_chat_fallback() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = """
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
|
||||
chat_service = SimpleNamespace(
|
||||
engine_client=SimpleNamespace(),
|
||||
model_config=SimpleNamespace(model="test-model"),
|
||||
)
|
||||
handler = OmniDuplexSessionHandler(chat_service=chat_service)
|
||||
if handler._serving_runtime_adapter is not None:
|
||||
raise SystemExit("generic handler silently selected a model serving adapter")
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def _valid_runtime_adapter() -> SimpleNamespace:
|
||||
data_plane = SimpleNamespace(
|
||||
begin_request=lambda request_id: None,
|
||||
is_terminal=lambda request_id: False,
|
||||
mark_terminal=lambda request_id: None,
|
||||
close_stream=lambda request_id: None,
|
||||
close_session=lambda session_id, **kwargs: None,
|
||||
project=lambda result, **kwargs: (),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
adapter_id="test-adapter",
|
||||
session_states={},
|
||||
data_plane=data_plane,
|
||||
clean_response_done_prefix="clean",
|
||||
interrupted_tts_prefix="interrupted",
|
||||
private_runtime_config_keys=frozenset({"private_key"}),
|
||||
create_session_state=lambda: object(),
|
||||
session_state=lambda session_id: object(),
|
||||
remove_session_state=lambda session_id: None,
|
||||
is_enabled=lambda config: True,
|
||||
capabilities=lambda **kwargs: object(),
|
||||
validate_client_extra_body=lambda extra_body: None,
|
||||
prepare_runtime_config=lambda config, **kwargs: {},
|
||||
runtime_config_for_update=lambda config, current: {},
|
||||
data_plane_context=lambda **kwargs: object(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "invalid_value"),
|
||||
[
|
||||
("session_states", None),
|
||||
("clean_response_done_prefix", None),
|
||||
("interrupted_tts_prefix", None),
|
||||
("private_runtime_config_keys", {"private_key"}),
|
||||
],
|
||||
)
|
||||
def test_runtime_adapter_validator_rejects_invalid_protocol_attributes(
|
||||
attribute: str,
|
||||
invalid_value: object,
|
||||
) -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
setattr(adapter, attribute, invalid_value)
|
||||
|
||||
with pytest.raises(TypeError, match=attribute):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
|
||||
|
||||
def test_runtime_adapter_validator_rejects_incomplete_data_plane() -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
adapter.data_plane.close_session = None
|
||||
|
||||
with pytest.raises(TypeError, match="data_plane.*close_session"):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Regression tests for duplex ``runtime_control`` payload redaction.
|
||||
|
||||
The redacted result is emitted on the WebSocket (for example
|
||||
``session.created.runtime_control``), so it must always survive
|
||||
``json.dumps``. Control results carry typed values straight from the engine --
|
||||
``fence`` and ``accepted_fence`` are ``DuplexFence`` instances added by
|
||||
``DuplexControlClient.execute`` -- which previously reached
|
||||
``WebSocket.send_json`` unconverted and killed the session at handshake.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.entrypoints.duplex.runtime_bridge import NativeRuntimeBridgeMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_redact = NativeRuntimeBridgeMixin._redact_runtime_control_result
|
||||
|
||||
|
||||
def _control_result() -> dict[str, object]:
|
||||
"""The shape ``DuplexControlClient.execute`` returns for an open."""
|
||||
return {
|
||||
"ok": True,
|
||||
"operation": "open",
|
||||
"session_id": "sess-1",
|
||||
"fence": DuplexFence("sess-1", epoch=0, turn_id=0, response_seq=0, incarnation=0),
|
||||
"accepted_fence": DuplexFence("sess-1", epoch=1, turn_id=2, response_seq=3, incarnation=4),
|
||||
"stage_results": [{"stage_id": -1, "replica_id": -1, "result": {"supported": True}}],
|
||||
}
|
||||
|
||||
|
||||
def test_redacted_control_result_is_json_serializable() -> None:
|
||||
"""The whole payload must serialize the way send_json does."""
|
||||
redacted = _redact(_control_result())
|
||||
# Must not raise.
|
||||
json.dumps({"type": "session.created", "runtime_control": redacted})
|
||||
|
||||
|
||||
def test_fences_are_converted_to_plain_dicts() -> None:
|
||||
redacted = _redact(_control_result())
|
||||
assert redacted["fence"] == {
|
||||
"session_id": "sess-1",
|
||||
"epoch": 0,
|
||||
"turn_id": 0,
|
||||
"response_seq": 0,
|
||||
"incarnation": 0,
|
||||
}
|
||||
# Field values are preserved, not just the type.
|
||||
assert redacted["accepted_fence"]["epoch"] == 1
|
||||
assert redacted["accepted_fence"]["turn_id"] == 2
|
||||
assert redacted["accepted_fence"]["response_seq"] == 3
|
||||
assert redacted["accepted_fence"]["incarnation"] == 4
|
||||
|
||||
|
||||
def test_nested_dataclasses_are_converted() -> None:
|
||||
"""Fences can appear inside stage results, not only at the top level."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"stage_results": [
|
||||
{"stage_id": 0, "result": {"fence": DuplexFence("sess-2", epoch=7)}},
|
||||
]
|
||||
}
|
||||
)
|
||||
json.dumps(redacted)
|
||||
assert redacted["stage_results"][0]["result"]["fence"]["epoch"] == 7
|
||||
|
||||
|
||||
def test_enums_are_reduced_to_values() -> None:
|
||||
class _Mode(str, Enum):
|
||||
APPEND = "append_audio_chunk"
|
||||
|
||||
redacted = _redact({"mode": _Mode.APPEND})
|
||||
json.dumps(redacted)
|
||||
assert redacted["mode"] == "append_audio_chunk"
|
||||
|
||||
|
||||
def test_redaction_still_drops_heavy_payload_keys() -> None:
|
||||
"""The pre-existing contract must be preserved."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"ok": True,
|
||||
"stage_handoff": object(),
|
||||
"tts_handoff": object(),
|
||||
"omni_payload": object(),
|
||||
"tts_hidden_states": object(),
|
||||
"tts_token_ids": object(),
|
||||
"traceback": "long trace",
|
||||
}
|
||||
)
|
||||
assert redacted == {"ok": True}
|
||||
|
||||
|
||||
def test_plain_values_pass_through_unchanged() -> None:
|
||||
payload = {"ok": True, "count": 3, "name": "x", "missing": None, "items": [1, "two"]}
|
||||
assert _redact(payload) == payload
|
||||
|
||||
|
||||
def test_dataclass_types_are_not_treated_as_instances() -> None:
|
||||
"""A class object is not a dataclass instance and must not be asdict'd."""
|
||||
|
||||
@dataclass
|
||||
class _Shape:
|
||||
x: int = 1
|
||||
|
||||
assert _redact(_Shape) is _Shape
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.websocket import (
|
||||
DuplexWebSocketActor,
|
||||
normalize_duplex_input_event,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self) -> None:
|
||||
self.sent: list[dict[str, object]] = []
|
||||
|
||||
async def send_json(self, payload: dict[str, object]) -> None:
|
||||
self.sent.append(dict(payload))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event", "expected"),
|
||||
[
|
||||
({"type": "signal_turn", "event": "barge_in"}, {"type": "turn.signal", "event": "barge_in"}),
|
||||
({"type": "close_session"}, {"type": "session.close"}),
|
||||
({"type": "audio.playback_ack", "played_ms": 1}, {"type": "playback.ack", "played_ms": 1}),
|
||||
({"type": "input_text.append", "text": "a"}, {"type": "input.text.append", "text": "a"}),
|
||||
({"type": "push_text", "text": "b"}, {"type": "input.text.append", "text": "b"}),
|
||||
(
|
||||
{"type": "input.audio.append", "audio": "wav"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "wav"},
|
||||
),
|
||||
(
|
||||
{"type": "push_chunk", "audio": "wav", "format": "pcm_f32le"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "pcm_f32le"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_input_aliases_normalize_once_at_mailbox_boundary(event, expected):
|
||||
assert normalize_duplex_input_event(event) == expected
|
||||
|
||||
|
||||
def test_actor_uses_one_fifo_mailbox_for_inbound_events():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
|
||||
assert isinstance(actor.mailbox, asyncio.Queue)
|
||||
assert not isinstance(actor.mailbox, asyncio.PriorityQueue)
|
||||
assert not hasattr(actor, "control_queue")
|
||||
assert not hasattr(actor, "input_queue")
|
||||
assert not hasattr(actor, "event_queue")
|
||||
assert not hasattr(actor, "lifecycle_state")
|
||||
assert not hasattr(actor, "last_response_id")
|
||||
assert not hasattr(actor, "overlap_speech_ms")
|
||||
assert not hasattr(actor, "transition")
|
||||
assert not hasattr(actor, "output_generation_in_flight")
|
||||
assert not hasattr(actor, "session")
|
||||
assert not hasattr(actor, "runtime_opened")
|
||||
assert not hasattr(actor, "runtime_closed")
|
||||
assert not hasattr(actor, "drain_input_queue")
|
||||
assert not hasattr(actor, "control_events_seen")
|
||||
assert not hasattr(actor, "input_events_seen")
|
||||
assert not hasattr(actor, "cancel_count")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_event_does_not_overtake_earlier_audio_input():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": "response.cancel"})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": "response.cancel"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("event_type", ["input.cancel", "session.close", "close_session"])
|
||||
async def test_terminal_control_preserves_wire_order(event_type: str):
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": event_type})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": event_type}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mailbox_delivers_every_enqueued_event_exactly_once():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
events = [
|
||||
{"type": "input_audio_buffer.append", "audio": str(index)}
|
||||
if index % 2
|
||||
else {"type": "response.created", "response_id": f"resp-{index}"}
|
||||
for index in range(20)
|
||||
]
|
||||
|
||||
await asyncio.gather(*(actor.enqueue_event(dict(event)) for event in events))
|
||||
received = [await actor.next_event() for _ in events]
|
||||
|
||||
assert sorted(received, key=repr) == sorted(events, key=repr)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_is_single_owner_of_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "one"})
|
||||
await actor.send_json({"type": "two"})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "one"}, {"type": "two"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_fence_payload_is_dropped_before_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 2)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "response.audio.delta", "epoch": 1})
|
||||
await actor.send_json({"type": "response.audio.delta", "epoch": 2})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "response.audio.delta", "epoch": 2}]
|
||||
assert actor.stale_output_dropped == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_does_not_revoke_accepted_terminal_after_close_starts():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 1)
|
||||
await actor.send_json({"type": "response.done", "epoch": 1, "response_id": "resp-1"})
|
||||
actor.closing = True
|
||||
await actor.close_writer()
|
||||
|
||||
await actor.writer_loop()
|
||||
|
||||
assert websocket.sent == [{"type": "response.done", "epoch": 1, "response_id": "resp-1"}]
|
||||
assert actor.stale_output_dropped == 0
|
||||
@@ -1,553 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
DuplexSessionRegistry,
|
||||
DuplexTurnController,
|
||||
DuplexTurnEventType,
|
||||
DuplexTurnState,
|
||||
ResponseCreateOptions,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.realtime_session import (
|
||||
NativeRealtimeSessionProtocol,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.realtime_state import RealtimeStateOwner
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
|
||||
minicpmo45_native_capabilities,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_realtime_state_owner_uses_explicit_bindings_not_dynamic_attribute_proxy():
|
||||
assert "__getattr__" not in RealtimeStateOwner.__dict__
|
||||
assert "__setattr__" not in RealtimeStateOwner.__dict__
|
||||
|
||||
|
||||
class _ProtocolWebSocket:
|
||||
def __init__(self, *events: dict[str, object]) -> None:
|
||||
self._events = asyncio.Queue()
|
||||
for event in events:
|
||||
self._events.put_nowait(json.dumps(event))
|
||||
|
||||
async def receive_text(self) -> str:
|
||||
return await self._events.get()
|
||||
|
||||
|
||||
def test_duplex_session_commits_text_and_audio_as_one_turn():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create(DuplexSessionConfig(model="test-model"))
|
||||
|
||||
session.append_text("hello")
|
||||
session.append_audio("YWJj", fmt="wav", sample_rate_hz=16000)
|
||||
committed = session.commit_user_input()
|
||||
|
||||
assert committed is not None
|
||||
assert committed.turn_id == 0
|
||||
assert committed.input_commit_seq == 1
|
||||
assert committed.epoch == 0
|
||||
assert len(session.history) == 1
|
||||
content = session.history[0]["content"]
|
||||
assert isinstance(content, list)
|
||||
assert content[0] == {"type": "text", "text": "hello"}
|
||||
assert content[1]["type"] == "audio_url"
|
||||
assert content[1]["audio_url"]["url"] == "data:audio/wav;base64,YWJj"
|
||||
assert session.turn_state == DuplexTurnState.USER_COMMITTED
|
||||
|
||||
|
||||
def test_response_options_apply_to_one_response_without_mutating_session_defaults():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-options",
|
||||
config=DuplexSessionConfig(instructions="base", voice="base-voice", max_tokens=64),
|
||||
)
|
||||
session.reserve_response_options(
|
||||
ResponseCreateOptions(
|
||||
instructions="one response",
|
||||
voice="override-voice",
|
||||
max_tokens=8,
|
||||
)
|
||||
)
|
||||
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
session.begin_response()
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.response_config.instructions == "one response"
|
||||
assert session.response_config.voice == "override-voice"
|
||||
assert session.response_config.max_tokens == 8
|
||||
|
||||
session.end_response()
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.config.max_tokens == 64
|
||||
|
||||
|
||||
def test_response_options_cannot_overwrite_an_unconsumed_reservation():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-options-pending",
|
||||
config=DuplexSessionConfig(instructions="base"),
|
||||
)
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="first"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="already reserved"):
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="second"))
|
||||
|
||||
session.begin_response()
|
||||
assert session.response_config.instructions == "first"
|
||||
|
||||
|
||||
def test_response_options_cannot_be_reserved_while_response_is_active():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-options-active",
|
||||
config=DuplexSessionConfig(instructions="base"),
|
||||
)
|
||||
session.begin_response()
|
||||
|
||||
with pytest.raises(RuntimeError, match="active"):
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="too late"))
|
||||
|
||||
|
||||
def test_realtime_model_name_does_not_implicitly_enable_native_duplex():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime({"model": "openbmb/MiniCPM-o-4_5"})
|
||||
|
||||
assert "native_duplex" not in event["session"]["extra_body"]
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_explicit_native_duplex_flag_is_preserved():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_body": {"native_duplex": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
|
||||
|
||||
def test_realtime_legacy_native_duplex_alias_is_normalized():
|
||||
# The deprecated model-prefixed spelling still opts in, but everything
|
||||
# downstream (and every session echo) sees only the canonical key.
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostart():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
|
||||
|
||||
assert event["type"] == "session.create"
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
|
||||
|
||||
def test_realtime_legacy_query_native_duplex_alias_seeds_canonical_key():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"minicpmo45_native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
|
||||
|
||||
assert event["type"] == "session.create"
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_resume_heartbeat_and_event_ack_translate_without_session_autostart():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
resume = await protocol._to_duplex_event(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
)
|
||||
heartbeat = await protocol._to_duplex_event(
|
||||
{
|
||||
"type": "session.heartbeat",
|
||||
"event_id": "event-heartbeat",
|
||||
}
|
||||
)
|
||||
event_ack = await protocol._to_duplex_event(
|
||||
{
|
||||
"type": "session.event_ack",
|
||||
"server_event_seq": 19,
|
||||
}
|
||||
)
|
||||
|
||||
assert resume == {
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
assert heartbeat == {
|
||||
"type": "session.heartbeat",
|
||||
"event_id": "event-heartbeat",
|
||||
}
|
||||
assert event_ack == {
|
||||
"type": "session.event_ack",
|
||||
"server_event_seq": 19,
|
||||
}
|
||||
assert protocol._opened is False
|
||||
|
||||
wire_protocol = NativeRealtimeSessionProtocol({})
|
||||
first_wire_event = json.loads(
|
||||
await wire_protocol.receive_internal_event_text(
|
||||
_ProtocolWebSocket(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert first_wire_event["type"] == "session.resume"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_resume_query_suppresses_model_autostart_on_same_url():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
"resume": "1",
|
||||
}
|
||||
)
|
||||
resume = {
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
|
||||
first_wire_event = json.loads(await protocol.receive_internal_event_text(_ProtocolWebSocket(resume)))
|
||||
|
||||
assert first_wire_event == resume
|
||||
assert protocol._autostarted_default_session is False
|
||||
|
||||
|
||||
def test_realtime_projects_resume_lifecycle_events_without_duplex_prefix():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
created = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.created",
|
||||
"session": {"id": "sid-resume"},
|
||||
"incarnation": 3,
|
||||
"attachment_generation": 1,
|
||||
"resume_token": "first-token",
|
||||
}
|
||||
)[0]
|
||||
resumed = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.resumed",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"attachment_generation": 2,
|
||||
"resume_token": "second-token",
|
||||
}
|
||||
)[0]
|
||||
heartbeat = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.heartbeat_ack",
|
||||
"session_id": "sid-resume",
|
||||
}
|
||||
)[0]
|
||||
replaced = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.replaced",
|
||||
"session_id": "sid-resume",
|
||||
"attachment_generation": 1,
|
||||
}
|
||||
)[0]
|
||||
resync = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.resync_required",
|
||||
"session_id": "sid-resume",
|
||||
"reason": "journal_gap",
|
||||
}
|
||||
)[0]
|
||||
|
||||
assert created["type"] == "session.created"
|
||||
assert created["incarnation"] == 3
|
||||
assert created["attachment_generation"] == 1
|
||||
assert created["resume_token"] == "first-token"
|
||||
assert resumed["type"] == "session.resumed"
|
||||
assert resumed["resume_token"] == "second-token"
|
||||
assert heartbeat["type"] == "session.heartbeat_ack"
|
||||
assert replaced["type"] == "session.replaced"
|
||||
assert resync == {
|
||||
"type": "session.resync_required",
|
||||
"session_id": "sid-resume",
|
||||
"reason": "journal_gap",
|
||||
"event_id": resync["event_id"],
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_projects_listen_response_identity():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
protocol.encode_outbound_event({"type": "session.created", "session": {"id": "sid-listen"}})
|
||||
|
||||
bound = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sid-listen",
|
||||
"epoch": 0,
|
||||
"reason": "model_listen",
|
||||
"model_listen": True,
|
||||
"response_id": "resp_1",
|
||||
}
|
||||
)[0]
|
||||
# A listen that terminates a precreated response keeps the response
|
||||
# identity on the Realtime dialect; clients demultiplex on response_id.
|
||||
assert bound["type"] == "response.listen"
|
||||
assert bound["response_id"] == "resp_1"
|
||||
assert bound["response"]["id"] == "resp_1"
|
||||
assert bound["response"]["status"] == "listening"
|
||||
|
||||
anonymous = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sid-listen",
|
||||
"epoch": 0,
|
||||
"reason": "silence_or_noise",
|
||||
}
|
||||
)[0]
|
||||
assert "response_id" not in anonymous
|
||||
assert "id" not in anonymous["response"]
|
||||
|
||||
|
||||
def test_duplex_session_registry_advances_incarnation_when_id_is_reused():
|
||||
registry = DuplexSessionRegistry()
|
||||
|
||||
first = registry.create(session_id="reused-session")
|
||||
registry.close(first.session_id)
|
||||
second = registry.create(session_id="reused-session")
|
||||
|
||||
assert first.incarnation == 0
|
||||
assert second.incarnation == 1
|
||||
|
||||
|
||||
def test_native_input_commit_does_not_advance_model_turn_identity():
|
||||
session = DuplexSession(
|
||||
session_id="sid-model-owned-turn",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
|
||||
first = session.commit_native_audio_input(transcript="first chunk")
|
||||
second = session.commit_native_audio_input(transcript="second chunk")
|
||||
|
||||
assert session.input_commit_seq == 2
|
||||
assert first.input_commit_seq == 1
|
||||
assert second.input_commit_seq == 2
|
||||
assert first.turn_id == second.turn_id == session.turn_id == 0
|
||||
|
||||
|
||||
def test_duplex_session_owns_response_and_overlap_identity():
|
||||
session = DuplexSession(session_id="sid-state-owner", config=DuplexSessionConfig())
|
||||
|
||||
response_id = session.begin_response()
|
||||
session.accumulate_overlap_speech(320)
|
||||
session.accumulate_overlap_speech(180)
|
||||
session.end_response()
|
||||
|
||||
assert session.active_response_id is None
|
||||
assert session.last_response_id == response_id
|
||||
assert session.overlap_speech_ms == 500
|
||||
assert session.reset_overlap_speech() == 500
|
||||
assert session.overlap_speech_ms == 0
|
||||
|
||||
|
||||
def test_duplex_session_composes_single_owner_ledgers():
|
||||
session = DuplexSession(session_id="sid-ledgers", config=DuplexSessionConfig())
|
||||
|
||||
session.append_text("hello")
|
||||
session.bind_request("req-1")
|
||||
response_id = session.begin_response(turn_id=3)
|
||||
session.mark_audio_sent(duration_ms=240)
|
||||
|
||||
assert not hasattr(session, "input")
|
||||
assert not hasattr(session, "response")
|
||||
assert not hasattr(session, "playback_ledger")
|
||||
assert not hasattr(session, "conversation")
|
||||
assert session.pending_text == ("hello",)
|
||||
assert session.active_request_id == "req-1"
|
||||
assert session.active_response_id == response_id
|
||||
assert session.active_response_turn_id == 3
|
||||
assert session.playback.sent_ms == 240
|
||||
|
||||
try:
|
||||
session.playback.sent_ms = 480
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("playback view must be immutable")
|
||||
|
||||
|
||||
def test_duplex_barge_in_advances_epoch_and_drops_uncommitted_assistant_text():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
response_id = session.begin_response()
|
||||
session.bind_request("chatcmpl-duplex-test")
|
||||
session.append_assistant_text("unplayed answer")
|
||||
|
||||
new_epoch = session.barge_in()
|
||||
|
||||
assert response_id is not None
|
||||
assert new_epoch == 1
|
||||
assert session.epoch == 1
|
||||
assert session.active_request_id is None
|
||||
assert session.active_response_id is None
|
||||
assert session.assistant_text_buffer == ()
|
||||
assert session.history == ()
|
||||
assert session.turn_state == DuplexTurnState.BARGE_IN
|
||||
|
||||
|
||||
def test_duplex_playback_ack_tracks_committed_cursor_separately():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
session.mark_audio_sent(duration_ms=10_000)
|
||||
|
||||
session.acknowledge_playback(played_ms=2_000)
|
||||
|
||||
assert session.playback.generated_ms == 10_000
|
||||
assert session.playback.sent_ms == 10_000
|
||||
assert session.playback.played_ms == 2_000
|
||||
assert session.playback.committed_ms == 2_000
|
||||
|
||||
|
||||
def test_duplex_history_commit_uses_audio_text_alignment_marks():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
session.begin_response()
|
||||
|
||||
session.append_assistant_text("hello ")
|
||||
session.mark_audio_sent(duration_ms=1_000, text_chars=6)
|
||||
session.append_assistant_text("world")
|
||||
session.mark_audio_sent(duration_ms=2_000, text_chars=11)
|
||||
session.acknowledge_playback(played_ms=1_200, committed_ms=1_200)
|
||||
|
||||
committed = session.end_response(commit_text=True)
|
||||
|
||||
assert committed == {"role": "assistant", "content": "hello w"}
|
||||
|
||||
|
||||
def test_turn_controller_accepts_external_signal_source():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
controller = DuplexTurnController()
|
||||
|
||||
event = controller.signal(session, DuplexTurnEventType.USER_STARTED.value)
|
||||
|
||||
assert event["type"] == "turn.event"
|
||||
assert event["event"] == "user_started"
|
||||
assert event["turn_state"] == "user_speaking"
|
||||
assert session.turn_state == DuplexTurnState.USER_SPEAKING
|
||||
|
||||
|
||||
def test_duplex_capabilities_do_not_claim_core_kv_or_input_append():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
caps = session.capabilities.as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "serving_session_adapter"
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_input_append"] is False
|
||||
assert caps["supports_reencode_context"] is True
|
||||
assert caps["adapter_patterns"] == ["chunk_group_append"]
|
||||
assert "turn_commit_only" in caps["input_modes"]
|
||||
assert "client_event" in caps["signal_sources"]
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
|
||||
caps = minicpmo45_native_capabilities(max_sessions=2).as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "model_native_duplex"
|
||||
assert caps["supports_input_append"] is True
|
||||
assert caps["input_modes"] == ["append_audio_chunk"]
|
||||
assert caps["adapter_patterns"] == ["scheduler_data_plane"]
|
||||
assert caps["supports_model_internal_state"] is True
|
||||
assert caps["requires_model_runner_kv"] is True
|
||||
assert caps["requires_native_stage_role"] is True
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_core_kv_lease"] is False
|
||||
assert caps["supports_stage_resumption"] is True
|
||||
assert caps["supports_scheduler_native_append"] is False
|
||||
assert caps["supports_core_resumable_request"] is True
|
||||
assert caps["supports_stage_connector_handoff"] is True
|
||||
assert caps["supports_audio_truncate"] is True
|
||||
assert caps["supports_barge_in"] is True
|
||||
assert caps["target_barge_in_latency_ms"] is None
|
||||
assert caps["supports_multi_session"] is True
|
||||
assert caps["supports_multi_session_same_replica"] is True
|
||||
assert caps["supports_session_lease"] is True
|
||||
assert caps["supports_session_resume"] is True
|
||||
assert caps["session_admission_mode"] == "engine_managed"
|
||||
assert caps["stage_handoff_transport"] == "scheduler_data_plane"
|
||||
|
||||
|
||||
def test_duplex_overlap_policy_auto_falls_back_to_listen_only():
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSessionConfig,
|
||||
)
|
||||
|
||||
assert DuplexSessionConfig._normalize_overlap_policy("auto") == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment():
|
||||
caps = minicpmo45_native_capabilities(max_sessions=1).as_dict()
|
||||
|
||||
assert caps["supports_multi_session"] is False
|
||||
assert caps["supports_multi_session_same_replica"] is False
|
||||
|
||||
|
||||
def test_duplex_overlap_policy_defaults_and_invalid_values_to_listen_only():
|
||||
assert DuplexSessionConfig().overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert DuplexSessionConfig._normalize_overlap_policy("not-a-policy") == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
@@ -78,7 +78,7 @@ def test_event_journal_sequences_acknowledges_and_replays_exact_payloads() -> No
|
||||
clock = _Clock()
|
||||
journal = DuplexEventJournal(max_bytes=4096, ttl_s=60.0, clock=clock)
|
||||
|
||||
first = journal.record({"type": "response.audio.delta", "delta": "AAAA"})
|
||||
first = journal.record({"type": "response.output_audio.delta", "delta": "AAAA"})
|
||||
second = journal.record({"type": "response.done", "response_id": "resp-1"})
|
||||
|
||||
assert first.sequence == 1
|
||||
@@ -122,6 +122,90 @@ def test_event_journal_overflow_is_explicit_and_does_not_evict_silently() -> Non
|
||||
journal.replay_after(0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("journal", [False, True])
|
||||
async def test_registry_failed_send_is_accepted_only_when_journaled(mocker, journal: bool) -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
accepted = mocker.Mock()
|
||||
expected_calls = [mocker.call()] if journal else []
|
||||
|
||||
async def failing_send(payload):
|
||||
assert accepted.call_args_list == expected_calls
|
||||
raise RuntimeError("transport lost")
|
||||
|
||||
await registry.create("sid-send-failure", send=failing_send, close=mocker.AsyncMock())
|
||||
|
||||
with pytest.raises(RuntimeError, match="transport lost"):
|
||||
await registry.send_event(
|
||||
"sid-send-failure",
|
||||
{"type": "response.audio.delta", "delta": "AAAA"},
|
||||
journal=journal,
|
||||
on_accepted=accepted,
|
||||
)
|
||||
|
||||
assert accepted.call_args_list == expected_calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("journal", [False, True])
|
||||
async def test_registry_detached_event_is_accepted_only_when_journaled(mocker, journal: bool) -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
send = mocker.AsyncMock()
|
||||
accepted = mocker.Mock()
|
||||
created = await registry.create("sid-detached-acceptance", send=send, close=mocker.AsyncMock())
|
||||
await registry.detach("sid-detached-acceptance", attachment_generation=created.attachment_generation)
|
||||
|
||||
entry = await registry.send_event(
|
||||
"sid-detached-acceptance",
|
||||
{"type": "response.audio.delta", "delta": "AAAA"},
|
||||
journal=journal,
|
||||
on_accepted=accepted,
|
||||
)
|
||||
|
||||
send.assert_not_awaited()
|
||||
assert (entry is not None) is journal
|
||||
assert accepted.call_count == int(journal)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_unjournaled_event_is_accepted_after_successful_send(mocker) -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
accepted = mocker.Mock()
|
||||
|
||||
async def send(payload):
|
||||
accepted.assert_not_called()
|
||||
|
||||
await registry.create("sid-live-acceptance", send=send, close=mocker.AsyncMock())
|
||||
|
||||
entry = await registry.send_event(
|
||||
"sid-live-acceptance",
|
||||
{"type": "response.audio.delta", "delta": "AAAA"},
|
||||
journal=False,
|
||||
on_accepted=accepted,
|
||||
)
|
||||
|
||||
assert entry is None
|
||||
accepted.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_overflow_does_not_accept_or_send_event(mocker) -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=100)
|
||||
accepted = mocker.Mock()
|
||||
send = mocker.AsyncMock()
|
||||
await registry.create("sid-overflow-acceptance", send=send, close=mocker.AsyncMock())
|
||||
|
||||
with pytest.raises(DuplexJournalOverflowError, match="byte limit"):
|
||||
await registry.send_event(
|
||||
"sid-overflow-acceptance",
|
||||
{"type": "response.audio.delta", "delta": "A" * 200},
|
||||
on_accepted=accepted,
|
||||
)
|
||||
|
||||
accepted.assert_not_called()
|
||||
send.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_resume_rotates_token_replays_and_atomically_replaces_attachment() -> None:
|
||||
clock = _Clock()
|
||||
@@ -147,13 +231,12 @@ async def test_registry_resume_rotates_token_replays_and_atomically_replaces_att
|
||||
async def close_b(reason):
|
||||
closes_b.append(reason)
|
||||
|
||||
created = await registry.create("sid", incarnation=2, send=send_a, close=close_a)
|
||||
created = await registry.create("sid", send=send_a, close=close_a)
|
||||
await registry.send_event("sid", {"type": "event-1"})
|
||||
await registry.send_event("sid", {"type": "event-2"})
|
||||
|
||||
resumed = await registry.resume(
|
||||
"sid",
|
||||
incarnation=2,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=1,
|
||||
send=send_b,
|
||||
@@ -176,6 +259,34 @@ async def test_registry_resume_rotates_token_replays_and_atomically_replaces_att
|
||||
assert closes_a == [] and sends_b == [] and closes_b == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_detach_without_a_generation_targets_the_current_attachment_once() -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
sends: list[dict] = []
|
||||
|
||||
async def send(payload):
|
||||
sends.append(payload)
|
||||
|
||||
async def close(reason):
|
||||
return None
|
||||
|
||||
await registry.create("sid-current", send=send, close=close)
|
||||
|
||||
# The outbound pump outlives any one connection and only knows the session:
|
||||
# ``None`` means "whichever socket is attached right now".
|
||||
assert await registry.detach("sid-current", attachment_generation=None) is True
|
||||
# A second report of the same dead socket is not a new detach, so it cannot
|
||||
# restart the engine's disconnect grace window.
|
||||
assert await registry.detach("sid-current", attachment_generation=None) is False
|
||||
assert await registry.detach("sid-current", attachment_generation=1) is False
|
||||
assert await registry.is_current_attachment("sid-current", 1) is False
|
||||
|
||||
# A detached session still journals for a later resume.
|
||||
await registry.send_event("sid-current", {"type": "event-1"})
|
||||
assert sends == []
|
||||
assert await registry.detach("missing-session", attachment_generation=None) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_resume_sends_activation_then_replay_before_new_live_events() -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(
|
||||
@@ -198,13 +309,12 @@ async def test_registry_resume_sends_activation_then_replay_before_new_live_even
|
||||
activation_started.set()
|
||||
await release_activation.wait()
|
||||
|
||||
created = await registry.create("sid-order", incarnation=0, send=old_send, close=close)
|
||||
created = await registry.create("sid-order", send=old_send, close=close)
|
||||
await registry.detach("sid-order", attachment_generation=1)
|
||||
await registry.send_event("sid-order", {"type": "replayed"})
|
||||
resume_task = asyncio.create_task(
|
||||
registry.resume(
|
||||
"sid-order",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=new_send,
|
||||
@@ -238,13 +348,12 @@ async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recover
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-recovery", incarnation=0, send=send, close=close)
|
||||
created = await registry.create("sid-recovery", send=send, close=close)
|
||||
await registry.detach("sid-recovery", attachment_generation=1)
|
||||
|
||||
with pytest.raises(RuntimeError, match="transport lost"):
|
||||
await registry.resume(
|
||||
"sid-recovery",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=failing_send,
|
||||
@@ -258,7 +367,6 @@ async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recover
|
||||
|
||||
recovered = await registry.resume(
|
||||
"sid-recovery",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -269,7 +377,6 @@ async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recover
|
||||
with pytest.raises(InvalidResumeTokenError):
|
||||
await registry.resume(
|
||||
"sid-recovery",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -290,13 +397,12 @@ async def test_registry_concurrent_resume_allows_exactly_one_rotated_token_winne
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-race", incarnation=0, send=send, close=close)
|
||||
created = await registry.create("sid-race", send=send, close=close)
|
||||
|
||||
async def attempt():
|
||||
try:
|
||||
return await registry.resume(
|
||||
"sid-race",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -312,7 +418,7 @@ async def test_registry_concurrent_resume_allows_exactly_one_rotated_token_winne
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> None:
|
||||
async def test_registry_keeps_sessions_journals_and_tokens_isolated() -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(
|
||||
replay_ttl_s=60.0,
|
||||
replay_max_bytes_per_session=4096,
|
||||
@@ -324,8 +430,8 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created_a = await registry.create("sid-a", incarnation=1, send=send, close=close)
|
||||
created_b = await registry.create("sid-b", incarnation=4, send=send, close=close)
|
||||
created_a = await registry.create("sid-a", send=send, close=close)
|
||||
created_b = await registry.create("sid-b", send=send, close=close)
|
||||
await registry.detach("sid-a", attachment_generation=1)
|
||||
await registry.detach("sid-b", attachment_generation=1)
|
||||
await registry.send_event("sid-a", {"type": "a-only"})
|
||||
@@ -333,7 +439,6 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
|
||||
resumed_a = await registry.resume(
|
||||
"sid-a",
|
||||
incarnation=1,
|
||||
resume_token=created_a.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -341,7 +446,6 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
)
|
||||
resumed_b = await registry.resume(
|
||||
"sid-b",
|
||||
incarnation=4,
|
||||
resume_token=created_b.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -350,11 +454,10 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
|
||||
assert [entry.payload["type"] for entry in resumed_a.replay_entries] == ["a-only"]
|
||||
assert [entry.payload["type"] for entry in resumed_b.replay_entries] == ["b-only"]
|
||||
with pytest.raises(ValueError, match="incarnation mismatch"):
|
||||
with pytest.raises(InvalidResumeTokenError):
|
||||
await registry.resume(
|
||||
"sid-a",
|
||||
incarnation=2,
|
||||
resume_token=resumed_a.resume_token.plaintext,
|
||||
resume_token=resumed_b.resume_token.plaintext,
|
||||
last_received_server_event_seq=1,
|
||||
send=send,
|
||||
close=close,
|
||||
@@ -374,7 +477,7 @@ async def test_registry_repr_never_contains_plaintext_tokens() -> None:
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-repr", incarnation=0, send=send, close=close)
|
||||
created = await registry.create("sid-repr", send=send, close=close)
|
||||
await registry.detach("sid-repr", attachment_generation=1)
|
||||
entry = await registry.send_event(
|
||||
"sid-repr",
|
||||
@@ -385,3 +488,67 @@ async def test_registry_repr_never_contains_plaintext_tokens() -> None:
|
||||
assert created.resume_token.plaintext not in repr(created)
|
||||
assert created.resume_token.plaintext not in repr(entry)
|
||||
assert created.resume_token.plaintext not in repr(registry)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cancel_on", ["activation", "replay"])
|
||||
async def test_registry_resume_cancelled_mid_delivery_rolls_back_like_a_failure(cancel_on: str) -> None:
|
||||
"""Cancellation must take the same rollback as an ordinary delivery failure.
|
||||
|
||||
``CancelledError`` derives from ``BaseException``, so an ``except
|
||||
Exception`` around the activation and replay sends let a cancelled resume
|
||||
keep its attachment as the current generation while the token it rotated
|
||||
past was already gone — the session left attached to an unusable transport
|
||||
with no way back. Both send points are covered because either can be the
|
||||
one that gets cancelled.
|
||||
"""
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
|
||||
async def send(payload):
|
||||
del payload
|
||||
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-cancel", send=send, close=close)
|
||||
# One journaled event, so the replay loop has something to send.
|
||||
await registry.send_event("sid-cancel", {"type": "response.output_text.delta", "delta": "a"})
|
||||
await registry.detach("sid-cancel", attachment_generation=1)
|
||||
|
||||
sends = 0
|
||||
cancel_at = 1 if cancel_on == "activation" else 2
|
||||
|
||||
async def cancelling_send(payload):
|
||||
del payload
|
||||
nonlocal sends
|
||||
sends += 1
|
||||
if sends == cancel_at:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await registry.resume(
|
||||
"sid-cancel",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=cancelling_send,
|
||||
close=close,
|
||||
activation_payload_factory=lambda token, generation: {
|
||||
"type": "session.resumed",
|
||||
"resume_token": token.plaintext,
|
||||
"attachment_generation": generation,
|
||||
},
|
||||
)
|
||||
|
||||
assert sends == cancel_at, "the send that was cancelled is the one under test"
|
||||
# The cancelled attachment must not remain current...
|
||||
assert not await registry.has_attachment("sid-cancel")
|
||||
# ...and the token the caller presented has to work again, or the session
|
||||
# is unreachable: resume() had already rotated past it.
|
||||
recovered = await registry.resume(
|
||||
"sid-cancel",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
close=close,
|
||||
)
|
||||
assert recovered.attachment_generation == 3
|
||||
@@ -42,8 +42,12 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import signal
|
||||
from argparse import Namespace
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException
|
||||
@@ -59,11 +63,78 @@ from vllm_omni.entrypoints.serve.utils import errors as serve_errors
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeParallelConfig:
|
||||
_api_process_rank: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeVllmConfig:
|
||||
parallel_config: _FakeParallelConfig
|
||||
|
||||
|
||||
def test_omni_api_worker_sets_parent_death_signal_before_validation(mocker):
|
||||
args = _minimal_args(_omni_stage_client_configs=[{"stage_addresses": {}}])
|
||||
death_signal = mocker.patch.object(api_server, "set_death_signal")
|
||||
|
||||
with pytest.raises(RuntimeError, match="client configuration"):
|
||||
api_server.run_omni_api_server_worker_proc(
|
||||
"127.0.0.1:8000",
|
||||
mocker.Mock(),
|
||||
args,
|
||||
{"client_count": 2, "client_index": -1},
|
||||
)
|
||||
|
||||
death_signal.assert_called_once_with(signal.SIGTERM)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_omni_run_server_worker_forwards_client_config(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_build_async_omni(*_args, **kwargs):
|
||||
captured.update(kwargs)
|
||||
yield _FakeEngineClient(vllm_config=_FakeVllmConfig(_FakeParallelConfig(_api_process_rank=1)))
|
||||
|
||||
monkeypatch.setattr(api_server, "build_async_omni", fake_build_async_omni)
|
||||
monkeypatch.setattr(api_server, "build_openai_app", lambda *_args: FastAPI())
|
||||
monkeypatch.setattr(api_server, "omni_init_app_state", lambda *_args: asyncio.sleep(0))
|
||||
monkeypatch.setattr(api_server, "shutdown_unsupported_routes", lambda *_args: None)
|
||||
|
||||
async def fake_storage_start() -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(api_server.STORAGE_MANAGER, "start", fake_storage_start)
|
||||
|
||||
async def fake_serve_http(app, **_kwargs):
|
||||
assert app.state.api_server_count == 2
|
||||
task = asyncio.create_task(asyncio.sleep(0))
|
||||
await asyncio.sleep(0)
|
||||
return task
|
||||
|
||||
monkeypatch.setattr(api_server, "serve_http", fake_serve_http)
|
||||
|
||||
args = _minimal_args(model="dummy")
|
||||
config = {"client_count": 2, "client_index": 1, "stage_addresses": {}}
|
||||
|
||||
await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), args, config)
|
||||
assert captured["client_config"] is config
|
||||
|
||||
|
||||
# FastAPI/Starlette auto-adds HEAD on every GET route. Including HEAD here
|
||||
# would invent ("HEAD", path) entries for every GET and fail the current census.
|
||||
# TODO: extend this set and the expected census if @router.head / @router.options appear.
|
||||
_HTTP_METHODS = {"GET", "POST", "DELETE", "PUT", "PATCH"}
|
||||
|
||||
|
||||
def test_profiler_endpoints_detect_typed_stage_config():
|
||||
stage = SimpleNamespace(profiler_config=SimpleNamespace(profiler="torch"))
|
||||
|
||||
assert api_server._should_enable_profiler_endpoints([stage])
|
||||
|
||||
|
||||
# Exact Omni router census on main. Update intentionally when routes change.
|
||||
_EXPECTED_ROUTER_ROUTES = {
|
||||
("POST", "/v1/chat/completions"),
|
||||
@@ -80,6 +151,11 @@ _EXPECTED_ROUTER_ROUTES = {
|
||||
("WEBSOCKET", "/v1/realtime"),
|
||||
("WEBSOCKET", "/v1/realtime/robot/openpi"),
|
||||
("WEBSOCKET", "/v1/duplex"),
|
||||
("POST", "/v1/realtime/sessions"),
|
||||
("POST", "/v1/realtime/sessions/{session_id}/step"),
|
||||
("POST", "/v1/realtime/sessions/{session_id}/reset"),
|
||||
("POST", "/v1/realtime/sessions/{session_id}/close"),
|
||||
("GET", "/v1/realtime/sessions/{session_id}/status"),
|
||||
("GET", "/health"),
|
||||
("GET", "/v1/models"),
|
||||
("POST", "/v1/images/generations"),
|
||||
@@ -120,6 +196,7 @@ _DIFFUSION_APP_STATE_KEYS = {
|
||||
"openai_streaming_speech",
|
||||
"openai_streaming_video",
|
||||
"openai_serving_realtime_robot",
|
||||
"rl_rollout_serving",
|
||||
"enable_server_load_tracking",
|
||||
"server_load_metrics",
|
||||
}
|
||||
@@ -151,12 +228,14 @@ _MULTISTAGE_APP_STATE_KEYS = {
|
||||
"openai_serving_realtime",
|
||||
"openai_serving_video",
|
||||
"openai_serving_realtime_robot",
|
||||
"rl_rollout_serving",
|
||||
"enable_server_load_tracking",
|
||||
"server_load_metrics",
|
||||
}
|
||||
_MULTISTAGE_MUST_BE_NONE = {
|
||||
"openai_serving_duplex",
|
||||
"openai_serving_realtime_robot",
|
||||
"rl_rollout_serving",
|
||||
}
|
||||
_MULTISTAGE_MUST_BE_WIRED = _MULTISTAGE_APP_STATE_KEYS - _MULTISTAGE_MUST_BE_NONE
|
||||
|
||||
@@ -486,7 +565,7 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
|
||||
Fails if HTTP requests lose ``request_timestamp``, or WebSocket scopes
|
||||
start getting stamped / mutated incorrectly.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_build_openai_app(args, supported_tasks):
|
||||
return FastAPI()
|
||||
@@ -514,8 +593,8 @@ async def test_timestamp_middleware_stamps_http_and_passes_websocket(monkeypatch
|
||||
await api_server.omni_run_server_worker("127.0.0.1:8000", _FakeSocket(), _minimal_args())
|
||||
middleware = captured["served_app"]
|
||||
|
||||
http_scope = {"type": "http", "state": {}}
|
||||
ws_scope = {"type": "websocket"}
|
||||
http_scope: dict[str, Any] = {"type": "http", "state": {}}
|
||||
ws_scope: dict[str, Any] = {"type": "websocket"}
|
||||
|
||||
async def _receive():
|
||||
return {"type": "http.disconnect"}
|
||||
@@ -611,6 +690,48 @@ async def test_realtime_route_defaults_to_configured_duplex_handler(
|
||||
assert calls == [expected_handler]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/duplex", "/v1/realtime", "/v1/realtime?duplex=1"])
|
||||
def test_multi_api_duplex_reconnect_rejected_on_each_worker(path: str) -> None:
|
||||
class SessionHandler:
|
||||
async def handle_realtime_session(self, _websocket) -> None:
|
||||
pytest.fail("Multi-API requests must not access process-local sessions")
|
||||
|
||||
class PendingWarmup:
|
||||
def is_set(self) -> bool:
|
||||
raise AssertionError("Multi-API rejection must not wait for duplex warmup")
|
||||
|
||||
# Separate apps represent workers with independent frontend registries.
|
||||
# Opening on A then reconnecting to B must reject at the same boundary.
|
||||
for worker_index in (0, 1):
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 2
|
||||
app.state.api_server_index = worker_index
|
||||
app.state.openai_serving_duplex = SessionHandler()
|
||||
app.state.duplex_warmup_done = PendingWarmup()
|
||||
app.include_router(api_server.router)
|
||||
with TestClient(app) as client, client.websocket_connect(path) as websocket:
|
||||
assert websocket.receive_json()["code"] == "multi_api_duplex_unsupported"
|
||||
with pytest.raises(WebSocketDisconnect) as exc:
|
||||
websocket.receive_text()
|
||||
assert exc.value.code == 1008
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/duplex", "/v1/realtime?duplex=1"])
|
||||
def test_single_api_duplex_session_still_reaches_handler(path: str) -> None:
|
||||
class SessionHandler:
|
||||
async def handle_realtime_session(self, websocket) -> None:
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "session.created"})
|
||||
await websocket.close()
|
||||
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 1
|
||||
app.state.openai_serving_duplex = SessionHandler()
|
||||
app.include_router(api_server.router)
|
||||
with TestClient(app) as client, client.websocket_connect(path) as websocket:
|
||||
assert websocket.receive_json() == {"type": "session.created"}
|
||||
|
||||
|
||||
def test_health_without_engine_returns_stable_unhealthy_response() -> None:
|
||||
"""Lock ``/health`` with no engine initialized.
|
||||
|
||||
@@ -700,6 +821,38 @@ def test_speech_without_handler_preserves_not_found_http_error() -> None:
|
||||
assert exc_info.value.detail == "The model does not support Speech API"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_api_rejects_runtime_voice_upload() -> None:
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 2
|
||||
raw_request = _request_for(app, method="POST", path="/v1/audio/voices")
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await api_server.upload_voice(
|
||||
raw_request,
|
||||
audio_sample=None,
|
||||
speaker_embedding=None,
|
||||
consent="consent-id",
|
||||
name="probe",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 409
|
||||
assert "process-local frontend state" in exc_info.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_api_rejects_sleep_route() -> None:
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 2
|
||||
raw_request = _request_for(app, method="POST", path="/v1/omni/sleep")
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await api_server.omni_sleep(api_server.OmniSleepRequest(stage_ids=[0]), raw_request)
|
||||
|
||||
assert exc_info.value.status_code == 409
|
||||
assert "process-local frontend state" in exc_info.value.detail
|
||||
|
||||
|
||||
def test_engine_error_json_response_includes_request_and_stage_fields(monkeypatch) -> None:
|
||||
"""Lock shared ``EngineGenerateError`` JSON fields used by Omni handlers.
|
||||
|
||||
@@ -768,11 +921,10 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
|
||||
engine = _FakeEngineClient(stage_configs=[stage])
|
||||
|
||||
def _for_diffusion_factory(label: str):
|
||||
@classmethod
|
||||
def _factory(cls, *args, **kwargs):
|
||||
return _marker(label)
|
||||
|
||||
return _factory
|
||||
return classmethod(_factory)
|
||||
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingChat, "for_diffusion", _for_diffusion_factory("chat"))
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingChatBatch, "for_diffusion", _for_diffusion_factory("chat_batch"))
|
||||
@@ -813,13 +965,11 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
|
||||
speech_kwargs = {}
|
||||
|
||||
def _for_diffusion_factory(label: str):
|
||||
@classmethod
|
||||
def _factory(cls, *args, **kwargs):
|
||||
return _marker(label)
|
||||
|
||||
return _factory
|
||||
return classmethod(_factory)
|
||||
|
||||
@classmethod
|
||||
def _speech_factory(cls, *args, **kwargs):
|
||||
speech_kwargs.update(kwargs)
|
||||
return _marker("speech")
|
||||
@@ -833,7 +983,7 @@ async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) ->
|
||||
)
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingVideo, "for_diffusion", _for_diffusion_factory("video"))
|
||||
monkeypatch.setattr(api_server.OmniStreamingVideoOutputHandler, "__init__", lambda self, *a, **k: None)
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", _speech_factory)
|
||||
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", classmethod(_speech_factory))
|
||||
monkeypatch.setattr(
|
||||
api_server.ServingRealtimeRobotOpenPI,
|
||||
"create_policy_server",
|
||||
@@ -912,7 +1062,6 @@ async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
|
||||
monkeypatch.setattr(api_server, "create_streaming_video_handler", lambda **_k: _marker("streaming_video"))
|
||||
monkeypatch.setattr(api_server, "OpenAIServingRealtime", _FakeCtor)
|
||||
monkeypatch.setattr(api_server, "OmniOpenAIServingVideo", _FakeCtor)
|
||||
monkeypatch.setattr(api_server, "should_enable_duplex_endpoint", lambda *_a, **_k: False)
|
||||
|
||||
state = State()
|
||||
await api_server.omni_init_app_state(engine, state, _minimal_args())
|
||||
@@ -923,3 +1072,57 @@ async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
|
||||
must_be_wired=_MULTISTAGE_MUST_BE_WIRED,
|
||||
must_be_none=_MULTISTAGE_MUST_BE_NONE,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("count", [None, 0, -1, "2", True])
|
||||
def test_process_local_guard_rejects_unknown_topology(count):
|
||||
app = FastAPI()
|
||||
if count is not None:
|
||||
app.state.api_server_count = count
|
||||
# CLI args cannot substitute for missing/invalid actual runtime state.
|
||||
app.state.args = Namespace(api_server_count=1)
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
api_server._reject_process_local_state_with_multiple_api_workers(_request_for(app), "Sleep")
|
||||
assert exc.value.status_code == 503
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested,config,expected",
|
||||
[(None, None, 1), (1, None, 1), (2, {"client_count": 2}, 2), (None, {"client_count": 2}, 2)],
|
||||
)
|
||||
def test_api_topology_resolved_from_launch(requested, config, expected):
|
||||
assert api_server._resolve_api_server_count(Namespace(api_server_count=requested), config) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested,config", [(2, None), (2, {}), (2, {"client_count": 1}), (2, {"client_count": "2"})])
|
||||
def test_api_topology_rejects_inconsistent_launch(requested, config):
|
||||
with pytest.raises(ValueError):
|
||||
api_server._resolve_api_server_count(Namespace(api_server_count=requested), config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("count,status", [(2, 409), (None, 503)])
|
||||
@pytest.mark.parametrize(
|
||||
"method,path",
|
||||
[
|
||||
("POST", "/v1/videos"),
|
||||
("GET", "/v1/videos"),
|
||||
("GET", "/v1/videos/job"),
|
||||
("GET", "/v1/videos/job/content"),
|
||||
("DELETE", "/v1/videos/job"),
|
||||
],
|
||||
)
|
||||
def test_video_lifecycle_guard_precedes_processing(monkeypatch, count, status, method, path):
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
app = FastAPI()
|
||||
if count is not None:
|
||||
app.state.api_server_count = count
|
||||
app.include_router(api_server.router)
|
||||
|
||||
async def forbidden_form():
|
||||
pytest.fail("video form processing must not run before topology rejection")
|
||||
|
||||
app.dependency_overrides[api_server._parse_video_form] = forbidden_form
|
||||
with TestClient(app) as client:
|
||||
response = client.request(method, path)
|
||||
assert response.status_code == status
|
||||
@@ -15,6 +15,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES,
|
||||
OmniOpenAIServingSpeech,
|
||||
@@ -251,9 +252,28 @@ class TestAudexStageDetection:
|
||||
stage = serving._find_tts_stage()
|
||||
assert stage is not None and stage.engine_args.model_stage == "audex_thinker"
|
||||
|
||||
def test_typed_pipeline_detected_without_legacy_engine_args(self):
|
||||
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||
stages = [
|
||||
SimpleNamespace(
|
||||
stage_pipeline_config=StagePipelineConfig(stage_id=index, model_stage=model_stage),
|
||||
model_config=SimpleNamespace(model_arch=None),
|
||||
worker_type="generation",
|
||||
)
|
||||
for index, model_stage in enumerate(("audex_thinker", "audex_code2wav"))
|
||||
]
|
||||
serving.engine_client = SimpleNamespace(stage_configs=stages)
|
||||
|
||||
stage = serving._find_tts_stage()
|
||||
serving._tts_stage = stage
|
||||
|
||||
assert stage is stages[0]
|
||||
assert serving._detect_tts_model_type() == "audex"
|
||||
|
||||
|
||||
def test_guided_injection_sets_guided_temperature():
|
||||
"""Guided requests decode at the measured CFG temperature (0.05)."""
|
||||
|
||||
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
|
||||
cond = TestAudexCfgInjection()._cond_prompt()
|
||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
|
||||
@@ -265,6 +285,31 @@ def test_guided_injection_sets_guided_temperature():
|
||||
assert out.temperature == 0.1
|
||||
|
||||
|
||||
def test_typed_full_pipeline_uses_full_checkpoint_tokenizer(mocker):
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
|
||||
|
||||
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||
serving._tts_stage = SimpleNamespace(
|
||||
stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="audex_omni"),
|
||||
model_config=SimpleNamespace(model_arch=None),
|
||||
worker_type="ar",
|
||||
)
|
||||
serving.engine_client = SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="/models/audex"),
|
||||
)
|
||||
ensure_snapshot = mocker.patch(
|
||||
"vllm_omni.model_executor.models.audex.checkpoint.ensure_audex_snapshot",
|
||||
return_value="/snapshots/audex",
|
||||
)
|
||||
from_pretrained = mocker.patch("transformers.AutoTokenizer.from_pretrained", return_value=object())
|
||||
|
||||
adapter = AudexAdapter(SpeechServingContext(server=serving, engine_client=serving.engine_client))
|
||||
adapter._get_tokenizer()
|
||||
|
||||
ensure_snapshot.assert_called_once_with("/models/audex", profile="full")
|
||||
from_pretrained.assert_called_once_with("/snapshots/audex/checkpoint_folder_full")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- TTA serving surface
|
||||
|
||||
from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter # noqa: E402
|
||||
|
||||
@@ -12,6 +12,8 @@ Covers:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from io import BytesIO
|
||||
|
||||
import numpy as np
|
||||
@@ -27,6 +29,8 @@ from vllm_omni.entrypoints.openai.protocol.audio import (
|
||||
SUPPORTED_CHAT_AUDIO_FORMATS,
|
||||
CreateAudio,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.outputs.mm_outputs import MultimodalCompletionOutput, MultimodalPayload
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -65,6 +69,13 @@ class TestCreateAudio:
|
||||
)
|
||||
response = mixin.create_audio(audio_obj)
|
||||
assert len(response.audio_data) > 0
|
||||
assert response.audio_metadata is not None
|
||||
assert response.audio_metadata.model_dump() == {
|
||||
"format": fmt,
|
||||
"sample_rate_hz": 24000,
|
||||
"frame_count": 24000,
|
||||
"channels": 1,
|
||||
}
|
||||
|
||||
def test_wav_magic_bytes(self, mixin, audio_tensor):
|
||||
audio_obj = CreateAudio(
|
||||
@@ -113,10 +124,9 @@ class TestCreateAudio:
|
||||
response = mixin.create_audio(audio_obj)
|
||||
assert response.audio_data[:4] == b"RIFF"
|
||||
assert response.media_type == "audio/wav"
|
||||
assert response.audio_metadata.format == "wav"
|
||||
|
||||
def test_base64_encoding(self, mixin, audio_tensor):
|
||||
import base64
|
||||
|
||||
audio_obj = CreateAudio(
|
||||
audio_tensor=audio_tensor,
|
||||
sample_rate=24000,
|
||||
@@ -128,21 +138,33 @@ class TestCreateAudio:
|
||||
decoded = base64.b64decode(response.audio_data)
|
||||
assert decoded[:4] == b"RIFF"
|
||||
|
||||
def test_resamples_wav_to_requested_output_rate(self, mixin, audio_tensor):
|
||||
@pytest.mark.parametrize(
|
||||
"channels,speed,frame_count",
|
||||
[(1, 1.0, 8000), (2, 2.0, 4000)],
|
||||
ids=["mono-resampled", "stereo-resampled-faster"],
|
||||
)
|
||||
def test_metadata_matches_transformed_waveform(self, mixin, audio_tensor, channels, speed, frame_count):
|
||||
response = mixin.create_audio(
|
||||
CreateAudio(
|
||||
audio_tensor=audio_tensor,
|
||||
audio_tensor=audio_tensor if channels == 1 else np.stack((audio_tensor, audio_tensor)),
|
||||
sample_rate=24000,
|
||||
output_sample_rate=8000,
|
||||
response_format="wav",
|
||||
speed=1.0,
|
||||
speed=speed,
|
||||
base64_encode=False,
|
||||
)
|
||||
)
|
||||
|
||||
with soundfile.SoundFile(BytesIO(response.audio_data)) as audio_file:
|
||||
assert audio_file.samplerate == 8000
|
||||
assert audio_file.frames == 8000
|
||||
assert audio_file.frames == frame_count
|
||||
assert audio_file.channels == channels
|
||||
assert response.audio_metadata.model_dump() == {
|
||||
"format": "wav",
|
||||
"sample_rate_hz": audio_file.samplerate,
|
||||
"frame_count": audio_file.frames,
|
||||
"channels": audio_file.channels,
|
||||
}
|
||||
|
||||
|
||||
class TestStreamingAudioResampler:
|
||||
@@ -262,3 +284,65 @@ class TestResolveAudioFormat:
|
||||
request = self._make_request({"format": fmt, "voice": "alloy"})
|
||||
result = serving_chat._resolve_audio_format(request)
|
||||
assert not isinstance(result, ErrorResponse), f"Format {fmt} should be accepted"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("fmt", ["pcm", "wav"])
|
||||
def test_chat_audio_metadata_survives_response_serialization(stream, fmt):
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.chat_completion import (
|
||||
OmniChatCompletionResponse,
|
||||
OmniChatCompletionStreamResponse,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
serving_chat = object.__new__(OmniOpenAIServingChat)
|
||||
output = MultimodalCompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
multimodal_output=MultimodalPayload.from_dict(
|
||||
{"audio": [torch.zeros(17), torch.zeros(23)], "sr": torch.tensor(22050)}
|
||||
),
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
audio={"format": fmt, "voice": "alloy"},
|
||||
)
|
||||
choices = serving_chat._create_audio_choice(
|
||||
OmniRequestOutput(outputs=[output], final_output_type="audio"), "assistant", request, stream=stream
|
||||
)
|
||||
assert not isinstance(choices, ErrorResponse)
|
||||
response_type = OmniChatCompletionStreamResponse if stream else OmniChatCompletionResponse
|
||||
response = response_type(
|
||||
id="chatcmpl-audio-metadata",
|
||||
created=0,
|
||||
model="test-model",
|
||||
choices=choices,
|
||||
usage=UsageInfo(prompt_tokens=0, completion_tokens=0, total_tokens=0),
|
||||
)
|
||||
|
||||
serialized = json.loads(response.model_dump_json(exclude_unset=True))
|
||||
choice = serialized["choices"][0]
|
||||
frame_count = 23 if stream else 40
|
||||
assert choice["audio_metadata"] == {
|
||||
"format": fmt,
|
||||
"sample_rate_hz": 22050,
|
||||
"frame_count": frame_count,
|
||||
"channels": 1,
|
||||
}
|
||||
audio = choice["delta"]["content"] if stream else choice["message"]["audio"]["data"]
|
||||
raw = base64.b64decode(audio)
|
||||
if fmt == "wav":
|
||||
with soundfile.SoundFile(BytesIO(raw)) as audio_file:
|
||||
assert audio_file.frames == frame_count
|
||||
assert audio_file.samplerate == 22050
|
||||
else:
|
||||
assert len(raw) == frame_count * 2
|
||||
@@ -0,0 +1,520 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU tests for serving-mode selection, duplex API wiring and warmup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from argparse import Namespace
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.datastructures import State
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from vllm_omni.config import stage_config
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig, PipelineConfig, StagePipelineConfig
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
from vllm_omni.entrypoints.openai import api_server
|
||||
from vllm_omni.utils.tracking_parser import TrackingNamespace
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# ``app.state`` of a duplex-only server. Presence is not enough: a route whose
|
||||
# handler is missing answers "not available", so "not wired" is written as None.
|
||||
_DUPLEX_APP_STATE_KEYS = {
|
||||
"engine_client",
|
||||
"log_stats",
|
||||
"args",
|
||||
"sleeping_stages",
|
||||
"stage_configs",
|
||||
"vllm_config",
|
||||
"diffusion_engine",
|
||||
"openai_serving_models",
|
||||
"serving_tokenization",
|
||||
"serving_tokens",
|
||||
"online_renderer",
|
||||
"openai_serving_chat",
|
||||
"openai_serving_chat_batch",
|
||||
"openai_serving_completion",
|
||||
"openai_serving_responses",
|
||||
"openai_serving_embedding",
|
||||
"openai_serving_pooling",
|
||||
"openai_serving_classification",
|
||||
"openai_serving_scores",
|
||||
"openai_serving_transcription",
|
||||
"openai_serving_translation",
|
||||
"openai_serving_speech",
|
||||
"openai_serving_audio_generate",
|
||||
"openai_serving_video",
|
||||
"openai_streaming_speech",
|
||||
"openai_streaming_video",
|
||||
"openai_streaming_video_output",
|
||||
"openai_serving_realtime",
|
||||
"openai_serving_realtime_robot",
|
||||
"anthropic_serving_messages",
|
||||
"openai_serving_duplex",
|
||||
"enable_server_load_tracking",
|
||||
"server_load_metrics",
|
||||
}
|
||||
#: Every turn-based service, plus the Realtime route that is not the duplex one.
|
||||
#: ``openai_serving_chat`` is absent: a duplex engine also serves chat, through
|
||||
#: the ordinary turn-based service, because DuplexOmni extends AsyncOmni.
|
||||
#: ``online_renderer`` is absent for the same reason -- the chat service needs it.
|
||||
_DUPLEX_MUST_BE_NONE = _DUPLEX_APP_STATE_KEYS - {
|
||||
"engine_client",
|
||||
"log_stats",
|
||||
"args",
|
||||
"sleeping_stages",
|
||||
"stage_configs",
|
||||
"vllm_config",
|
||||
"openai_serving_models",
|
||||
"openai_serving_duplex",
|
||||
"openai_serving_chat",
|
||||
"online_renderer",
|
||||
"enable_server_load_tracking",
|
||||
"server_load_metrics",
|
||||
}
|
||||
_DUPLEX_MUST_BE_WIRED = {
|
||||
"engine_client",
|
||||
"args",
|
||||
"stage_configs",
|
||||
"vllm_config",
|
||||
"openai_serving_models",
|
||||
"openai_serving_duplex",
|
||||
"openai_serving_chat",
|
||||
}
|
||||
|
||||
|
||||
class _FakeDuplexOmni(DuplexOmni):
|
||||
"""A ``DuplexOmni`` without an engine: only what the app-state wiring reads."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.model = "demo-duplex-model"
|
||||
self._stage_configs = [object(), object(), object()]
|
||||
self._vllm_config = SimpleNamespace(
|
||||
lora_config=None,
|
||||
model_config=SimpleNamespace(),
|
||||
parallel_config=SimpleNamespace(_api_process_rank=0),
|
||||
)
|
||||
self._duplex_session_config = DuplexSessionRuntimeConfig()
|
||||
self._capabilities = DuplexCapabilities(supports_chat_completions=True)
|
||||
|
||||
async def get_vllm_config(self):
|
||||
return self._vllm_config
|
||||
|
||||
async def get_supported_tasks(self) -> tuple[str, ...]:
|
||||
return ("generate",)
|
||||
|
||||
async def get_tokenizer(self):
|
||||
return SimpleNamespace(chat_template="{{ messages }}")
|
||||
|
||||
@property
|
||||
def stage_configs(self) -> list[object]:
|
||||
return self._stage_configs
|
||||
|
||||
@property
|
||||
def duplex_session_config(self) -> DuplexSessionRuntimeConfig:
|
||||
return self._duplex_session_config
|
||||
|
||||
@property
|
||||
def duplex_capabilities(self) -> DuplexCapabilities:
|
||||
return self._capabilities
|
||||
|
||||
# Properties on AsyncOmni, so the fake overrides rather than assigns them.
|
||||
@property
|
||||
def model_config(self) -> SimpleNamespace:
|
||||
return SimpleNamespace()
|
||||
|
||||
@property
|
||||
def renderer(self) -> SimpleNamespace:
|
||||
return SimpleNamespace()
|
||||
|
||||
|
||||
class _FakeModels:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.base_model_paths = kwargs.get("base_model_paths") or []
|
||||
|
||||
|
||||
def _minimal_args(**overrides) -> SimpleNamespace:
|
||||
args = SimpleNamespace(
|
||||
model="demo-duplex-model",
|
||||
served_model_name=None,
|
||||
disable_log_stats=True,
|
||||
enable_log_requests=False,
|
||||
max_log_len=None,
|
||||
enable_server_load_tracking=False,
|
||||
trust_remote_code=True,
|
||||
deploy_config=None,
|
||||
# What the chat service reads when a duplex model also serves chat.
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
trust_request_chat_template=False,
|
||||
default_chat_template_kwargs=None,
|
||||
response_role="assistant",
|
||||
return_tokens_as_token_ids=False,
|
||||
enable_auto_tool_choice=False,
|
||||
exclude_tools_when_tool_choice_none=False,
|
||||
tool_call_parser=None,
|
||||
structured_outputs_config=SimpleNamespace(reasoning_parser=None),
|
||||
enable_prompt_tokens_details=False,
|
||||
enable_force_include_usage=False,
|
||||
enable_log_outputs=False,
|
||||
enable_log_deltas=False,
|
||||
)
|
||||
for key, value in overrides.items():
|
||||
setattr(args, key, value)
|
||||
return args
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Startup: which entrypoint class serves the model #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pipeline_config", "expected"),
|
||||
[
|
||||
(SimpleNamespace(duplex_plugin="pkg.mod.Plugin"), True),
|
||||
(SimpleNamespace(duplex_plugin=None), False),
|
||||
(SimpleNamespace(duplex_plugin=""), False),
|
||||
(SimpleNamespace(), False),
|
||||
(None, False),
|
||||
],
|
||||
)
|
||||
def test_duplex_serving_requires_a_pipeline_plugin(monkeypatch, tmp_path, pipeline_config, expected: bool) -> None:
|
||||
seen: dict[str, object] = {}
|
||||
deploy_path = tmp_path / "deploy.yaml"
|
||||
deploy_path.write_text("session_mode: duplex\n")
|
||||
|
||||
def fake_get_pipeline_config(*, model, trust_remote_code, deploy_config_path=None, **_kwargs):
|
||||
seen.update(model=model, trust_remote_code=trust_remote_code, deploy_config_path=deploy_config_path)
|
||||
return pipeline_config
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", fake_get_pipeline_config)
|
||||
|
||||
kwargs = {"trust_remote_code": True, "deploy_config": str(deploy_path)}
|
||||
assert api_server._should_serve_duplex("demo-duplex-model", kwargs) is expected
|
||||
assert seen == {
|
||||
"model": "demo-duplex-model",
|
||||
"trust_remote_code": True,
|
||||
"deploy_config_path": str(deploy_path),
|
||||
}
|
||||
|
||||
|
||||
def test_duplex_model_probe_propagates_resolution_errors(monkeypatch) -> None:
|
||||
"""A pipeline that cannot be resolved must fail startup, not fall back to turn-based."""
|
||||
|
||||
def fake_get_pipeline_config(**_kwargs):
|
||||
raise ValueError("needs trust_remote_code=True")
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", fake_get_pipeline_config)
|
||||
|
||||
with pytest.raises(ValueError, match="trust_remote_code"):
|
||||
api_server._should_serve_duplex("demo-duplex-model", {})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def duplex_pipeline():
|
||||
return PipelineConfig(
|
||||
model_type="demo",
|
||||
stages=(StagePipelineConfig(stage_id=0, model_stage="demo", final_output=True),),
|
||||
duplex_plugin="pkg.mod.Plugin",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("session_mode", ["turn", "duplex"])
|
||||
@pytest.mark.parametrize("config_source", ["explicit", "default", "basename", "inherited"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_serving_mode_selects_engine_and_preserves_stage_config(
|
||||
monkeypatch, tmp_path, duplex_pipeline, session_mode, config_source
|
||||
):
|
||||
base_path = tmp_path / "base.yaml"
|
||||
base_path.write_text("session_mode: duplex\nstages:\n - stage_id: 0\n max_num_seqs: 4\n")
|
||||
deploy_path = tmp_path / "deploy.yaml"
|
||||
deploy_path.write_text(f"base_config: base.yaml\nsession_mode: {session_mode}\n")
|
||||
if config_source == "inherited":
|
||||
deploy_path = tmp_path / "overlay.yaml"
|
||||
deploy_path.write_text("base_config: deploy.yaml\n")
|
||||
pipeline = replace(duplex_pipeline, default_deploy_config_name=deploy_path.name)
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: pipeline)
|
||||
monkeypatch.setattr(stage_config, "_DEPLOY_DIR", tmp_path)
|
||||
kwargs = {}
|
||||
if config_source != "default":
|
||||
kwargs["deploy_config"] = deploy_path.name if config_source == "basename" else str(deploy_path)
|
||||
|
||||
created = []
|
||||
|
||||
class TurnEngine:
|
||||
def __init__(self, model, **engine_kwargs):
|
||||
self.model = model
|
||||
self.kwargs = engine_kwargs
|
||||
self.closed = False
|
||||
created.append(self)
|
||||
|
||||
def shutdown(self):
|
||||
self.closed = True
|
||||
|
||||
class DuplexEngine(TurnEngine):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(api_server, "AsyncOmni", TurnEngine)
|
||||
monkeypatch.setattr(api_server, "DuplexOmni", DuplexEngine)
|
||||
args = TrackingNamespace(
|
||||
Namespace(model="demo-model", disable_log_stats=True, **kwargs), frozenset({"model", *kwargs})
|
||||
)
|
||||
async with api_server.build_async_omni_from_stage_config(args) as engine:
|
||||
assert type(engine) is (DuplexEngine if session_mode == "duplex" else TurnEngine)
|
||||
assert engine.model == "demo-model"
|
||||
assert engine.kwargs == {**kwargs, "log_stats": False}
|
||||
assert not engine.closed
|
||||
assert len(created) == 1
|
||||
assert engine.closed
|
||||
deploy = stage_config.load_deploy_config(deploy_path)
|
||||
assert deploy.session_mode == session_mode
|
||||
assert deploy.stages[0].max_num_seqs == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("contents", ["{}\n", "session_mode: null\n", "session_mode: typo\n"])
|
||||
def test_duplex_serving_rejects_missing_or_invalid_mode(monkeypatch, tmp_path, duplex_pipeline, contents):
|
||||
deploy_path = tmp_path / "deploy.yaml"
|
||||
deploy_path.write_text(contents)
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: duplex_pipeline)
|
||||
with pytest.raises(ValueError, match="session_mode: turn or duplex"):
|
||||
api_server._should_serve_duplex("demo-model", {"deploy_config": str(deploy_path)})
|
||||
|
||||
|
||||
def test_duplex_serving_requires_a_deploy_config(monkeypatch, duplex_pipeline):
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: duplex_pipeline)
|
||||
with pytest.raises(ValueError, match="requires a deploy config"):
|
||||
api_server._should_serve_duplex("demo-model", {})
|
||||
|
||||
|
||||
def test_duplex_serving_propagates_deploy_load_errors(monkeypatch, tmp_path, duplex_pipeline):
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: duplex_pipeline)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
api_server._should_serve_duplex("demo-model", {"deploy_config": str(tmp_path / "missing.yaml")})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deploy_name,expected", [("minicpmo_4_5.yaml", True), ("minicpmo_4_5_turn.yaml", False)])
|
||||
def test_minicpmo_serving_profiles(monkeypatch, deploy_name, expected):
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **_: MINICPMO_4_5_PIPELINE)
|
||||
assert api_server._should_serve_duplex("openbmb/MiniCPM-o-4_5", {"deploy_config": deploy_name}) is expected
|
||||
assert api_server._should_serve_duplex("openbmb/MiniCPM-o-4_5", {}) is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# app.state of a duplex-only server #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_app_state_wires_only_the_session_surfaces(monkeypatch) -> None:
|
||||
"""Lock the duplex ``app.state``: the two session-backed surfaces are live, every turn-based service is None.
|
||||
|
||||
Fails if a turn-based service is wired into a duplex server (its route
|
||||
would answer instead of reporting "not available"), or if a key the routes
|
||||
read disappears. ``openai_serving_chat`` is the one shared name: the route
|
||||
is served, but by the adapter that runs it on a duplex session, never by
|
||||
the turn-based chat service.
|
||||
"""
|
||||
monkeypatch.setattr(api_server, "OpenAIServingModels", _FakeModels)
|
||||
monkeypatch.setattr(api_server, "OnlineRenderer", lambda **kwargs: SimpleNamespace(**kwargs))
|
||||
monkeypatch.setattr(api_server, "OmniOpenAIServingChat", lambda **kwargs: SimpleNamespace(**kwargs))
|
||||
engine = _FakeDuplexOmni()
|
||||
state = State()
|
||||
|
||||
await api_server.omni_init_app_state(engine, state, _minimal_args())
|
||||
|
||||
present = {key for key in _DUPLEX_APP_STATE_KEYS if hasattr(state, key)}
|
||||
assert present == _DUPLEX_APP_STATE_KEYS
|
||||
not_wired = sorted(key for key in _DUPLEX_MUST_BE_WIRED if getattr(state, key) is None)
|
||||
assert not not_wired, f"duplex app.state keys registered but not wired: {not_wired}"
|
||||
unexpectedly_set = sorted(key for key in _DUPLEX_MUST_BE_NONE if getattr(state, key) is not None)
|
||||
assert not unexpectedly_set, f"turn-based services wired into a duplex server: {unexpectedly_set}"
|
||||
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
|
||||
assert state.openai_serving_chat is not None
|
||||
assert state.engine_client is engine
|
||||
assert state.vllm_config is engine._vllm_config
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Routes #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _RecordingHandler:
|
||||
"""Stands in for ``OmniDuplexSessionHandler`` on the app state."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.queries: list[dict[str, str]] = []
|
||||
|
||||
async def handle_realtime_session(self, websocket) -> None:
|
||||
await websocket.accept()
|
||||
self.queries.append(dict(websocket.query_params))
|
||||
await websocket.send_json({"type": "session.created", "session": {"id": "duplex-test"}})
|
||||
await websocket.close()
|
||||
|
||||
|
||||
def _duplex_app(handler: object) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.include_router(api_server.router)
|
||||
app.state.openai_serving_duplex = handler
|
||||
# A duplex server serves no turn-based Realtime route.
|
||||
app.state.openai_serving_realtime = None
|
||||
return app
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/duplex", "/v1/realtime?duplex=1"])
|
||||
def test_both_duplex_routes_reach_the_session_handler(path: str) -> None:
|
||||
"""``/v1/duplex`` is an alias of ``/v1/realtime?duplex=1``: same handler, same protocol."""
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(path) as websocket:
|
||||
assert websocket.receive_json()["type"] == "session.created"
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert len(handler.queries) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["1", "true", "on"])
|
||||
def test_realtime_route_switches_to_duplex_only_for_the_documented_flag_values(flag: str) -> None:
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(f"/v1/realtime?duplex={flag}") as websocket:
|
||||
assert websocket.receive_json()["type"] == "session.created"
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert handler.queries == [{"duplex": flag}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query", ["", "?model=openbmb%2FMiniCPM-o-4_5"])
|
||||
def test_realtime_without_a_duplex_flag_reaches_the_session_handler(query: str) -> None:
|
||||
"""A bare connection to a duplex deployment is a duplex session.
|
||||
|
||||
``?duplex=1`` is the spelling the server advertises, but it is not a
|
||||
requirement: a deployment that declares a duplex plugin answers plain
|
||||
``/v1/realtime`` too, so a stock Realtime client needs no vendor query
|
||||
parameter.
|
||||
"""
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(f"/v1/realtime{query}") as websocket:
|
||||
assert websocket.receive_json()["type"] == "session.created"
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert len(handler.queries) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query", ["?duplex=0", "?duplex=false", "?duplex=off"])
|
||||
def test_realtime_opted_out_of_duplex_falls_through_to_the_turn_based_route(query: str) -> None:
|
||||
"""An explicit opt-out selects the legacy handler, which a duplex server does not mount."""
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(f"/v1/realtime{query}") as websocket:
|
||||
assert websocket.receive_json() == {
|
||||
"type": "error",
|
||||
"error": "Realtime API is not available",
|
||||
"code": "unsupported",
|
||||
}
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert handler.queries == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Startup warmup gate #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _warmup_websocket(warmup_done: asyncio.Event | None, query: dict[str, str]) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
app=SimpleNamespace(state=SimpleNamespace(duplex_warmup_done=warmup_done)),
|
||||
query_params=query,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_warmup_gate_holds_clients_until_the_warmup_session_finishes() -> None:
|
||||
warmup_done = asyncio.Event()
|
||||
websocket = _warmup_websocket(warmup_done, {})
|
||||
|
||||
waiting = asyncio.create_task(api_server._wait_for_duplex_warmup(websocket))
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(asyncio.shield(waiting), timeout=0.05)
|
||||
|
||||
warmup_done.set()
|
||||
await asyncio.wait_for(waiting, timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("warmup_done", "query"),
|
||||
[
|
||||
(None, {}),
|
||||
(asyncio.Event(), {"vllm_omni_warmup": "1"}),
|
||||
],
|
||||
ids=["no-warmup-configured", "the-warmup-connection-itself"],
|
||||
)
|
||||
async def test_warmup_gate_lets_the_warmup_connection_and_plain_servers_through(warmup_done, query) -> None:
|
||||
websocket = _warmup_websocket(warmup_done, query)
|
||||
|
||||
await asyncio.wait_for(api_server._wait_for_duplex_warmup(websocket), timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_model_that_does_not_declare_chat_completions_does_not_get_the_route(monkeypatch) -> None:
|
||||
"""The decision stays the model's: no capability, no chat service, and the route says so."""
|
||||
monkeypatch.setattr(api_server, "OpenAIServingModels", _FakeModels)
|
||||
engine = _FakeDuplexOmni()
|
||||
engine._capabilities = DuplexCapabilities(supports_chat_completions=False)
|
||||
state = State()
|
||||
|
||||
await api_server.omni_init_app_state(engine, state, _minimal_args())
|
||||
|
||||
assert state.openai_serving_chat is None
|
||||
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
|
||||
|
||||
|
||||
def test_qwen_plugin_preserves_custom_turn_deployment_default(monkeypatch, tmp_path):
|
||||
from vllm_omni.model_executor.models.qwen3_omni.pipeline import QWEN3_OMNI_PIPELINE
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", lambda **kwargs: QWEN3_OMNI_PIPELINE)
|
||||
deploy = tmp_path / "qwen.yaml"
|
||||
deploy.write_text("stages: []\n")
|
||||
assert not api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
|
||||
deploy.write_text("session_mode: duplex\nstages: []\n")
|
||||
assert api_server._should_serve_duplex("qwen", {"deploy_config": str(deploy)})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["1", "true", "on"])
|
||||
def test_turn_deployment_rejects_explicit_duplex_without_falling_back_to_stt(flag: str) -> None:
|
||||
app = _duplex_app(None)
|
||||
# Even an available STT service must not silently accept a duplex request.
|
||||
app.state.openai_serving_realtime = object()
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect(f"/v1/realtime?duplex={flag}") as websocket:
|
||||
assert websocket.receive_json() == {
|
||||
"type": "error",
|
||||
"code": "unsupported",
|
||||
"error": "VAD realtime is not enabled",
|
||||
}
|
||||
with pytest.raises(WebSocketDisconnect) as exc:
|
||||
websocket.receive_text()
|
||||
assert exc.value.code == 1008
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,276 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU tests for the Gepard-1.0 speech adapter.
|
||||
|
||||
Covers detection, the reject matrix, voice default, empty input, extra_params,
|
||||
max_new_tokens frame bounds, and build() parity with the offline prompt helper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
from vllm_omni.entrypoints.openai.tts_adapters import detect_tts_model_type, resolve_adapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.gepard import GepardAdapter
|
||||
from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig
|
||||
from vllm_omni.model_executor.models.gepard.prompt import build_gepard_prompt_ids
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_MODEL = "nineninesix/gepard-1.0"
|
||||
|
||||
|
||||
def _adapter(*, revision: str | None = None) -> GepardAdapter:
|
||||
engine = SimpleNamespace(model_config=SimpleNamespace(model=_MODEL, revision=revision))
|
||||
return GepardAdapter(SpeechServingContext(server=SimpleNamespace(engine_client=engine), engine_client=engine))
|
||||
|
||||
|
||||
def test_gepard_detection_and_registration() -> None:
|
||||
assert resolve_adapter("gepard") is GepardAdapter
|
||||
assert GepardAdapter.stage_keys == {"gepard"}
|
||||
assert GepardAdapter.model_archs == {"GepardTalkerForConditionalGeneration"}
|
||||
assert detect_tts_model_type("gepard", None) == "gepard"
|
||||
assert detect_tts_model_type("gepard", "GepardTalkerForConditionalGeneration") == "gepard"
|
||||
assert detect_tts_model_type(None, "GepardTalkerForConditionalGeneration") == "gepard"
|
||||
|
||||
|
||||
def test_gepard_accepts_omitted_and_default_voice() -> None:
|
||||
adapter = _adapter()
|
||||
assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.")) is None
|
||||
assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", voice="default")) is None
|
||||
assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", voice="Default")) is None
|
||||
assert (
|
||||
adapter.validate(
|
||||
OpenAICreateSpeechRequest.model_validate({"input": "Hello from Gepard.", "speaker": "default"})
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "field"),
|
||||
[
|
||||
({"voice": "vivian"}, "voice"),
|
||||
({"speed": 1.2}, "speed"),
|
||||
({"extra_params": {"temperature": 0.3}}, "extra_params"),
|
||||
({"extra_params": {"top_p": 0.9}}, "extra_params"),
|
||||
({"extra_params": {"top_k": 20}}, "extra_params"),
|
||||
({"extra_params": {"foo": 1}}, "extra_params"),
|
||||
({"ref_audio": "https://example.com/ref.wav"}, "ref_audio"),
|
||||
({"ref_text": "transcript"}, "ref_text"),
|
||||
({"ref_audio_2": "https://example.com/ref2.wav"}, "ref_audio_2"),
|
||||
({"speaker_embedding": [0.1, 0.2]}, "speaker_embedding"),
|
||||
({"x_vector_only_mode": True}, "x_vector_only_mode"),
|
||||
({"task_type": "Base"}, "task_type"),
|
||||
({"non_streaming_mode": True}, "non_streaming_mode"),
|
||||
({"instructions": "speak happily"}, "instructions"),
|
||||
({"language": "English"}, "language"),
|
||||
({"ambient_sound": "rain"}, "ambient_sound"),
|
||||
({"duration_seconds": 2.0}, "duration_seconds"),
|
||||
({"initial_codec_chunk_frames": 4}, "initial_codec_chunk_frames"),
|
||||
({"word_timestamps": True}, "word_timestamps"),
|
||||
({"response_format": "opus"}, "opus"),
|
||||
],
|
||||
)
|
||||
def test_gepard_rejects_unsupported_fields(kwargs: dict, field: str) -> None:
|
||||
adapter = _adapter()
|
||||
err = adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", **kwargs))
|
||||
assert err is not None
|
||||
assert field in err
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["", " ", "\n\t"])
|
||||
def test_gepard_rejects_empty_input(text: str) -> None:
|
||||
adapter = _adapter()
|
||||
err = adapter.validate(OpenAICreateSpeechRequest(input=text))
|
||||
assert err is not None
|
||||
assert "empty" in err.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_new_tokens", [0, -1])
|
||||
def test_gepard_schema_rejects_non_positive_max_new_tokens(max_new_tokens: int) -> None:
|
||||
with pytest.raises(ValidationError, match="greater than or equal to 1"):
|
||||
OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=max_new_tokens)
|
||||
|
||||
|
||||
def test_gepard_rejects_max_new_tokens_above_frame_budget() -> None:
|
||||
adapter = _adapter()
|
||||
err = adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=4097))
|
||||
assert err is not None
|
||||
assert "max_new_tokens" in err
|
||||
assert "frames" in err
|
||||
|
||||
|
||||
def test_gepard_accepts_max_new_tokens_bounds() -> None:
|
||||
adapter = _adapter()
|
||||
assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=1)) is None
|
||||
assert adapter.validate(OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=4096)) is None
|
||||
|
||||
|
||||
def test_gepard_apply_sampling_overrides_uses_frames() -> None:
|
||||
adapter = _adapter()
|
||||
stage_defaults = [SimpleNamespace(max_tokens=1000)]
|
||||
overridden = adapter.apply_sampling_overrides(
|
||||
stage_defaults,
|
||||
OpenAICreateSpeechRequest(input="Hello from Gepard.", max_new_tokens=32),
|
||||
)
|
||||
assert overridden[0].max_tokens == 32
|
||||
assert stage_defaults[0].max_tokens == 1000
|
||||
|
||||
|
||||
def test_gepard_build_matches_offline_prompt_builder(monkeypatch) -> None:
|
||||
adapter = _adapter()
|
||||
cfg = GepardConfig()
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.models.gepard.configuration_gepard.GepardConfig.from_checkpoint",
|
||||
staticmethod(lambda *_a, **_k: cfg),
|
||||
)
|
||||
|
||||
class _Tok:
|
||||
def __call__(self, text, add_special_tokens=False):
|
||||
return {"input_ids": [11, 12, 13]}
|
||||
|
||||
adapter._tokenizer = _Tok()
|
||||
adapter._gepard_config = cfg
|
||||
text = "Hello from Gepard."
|
||||
prepared = asyncio.run(adapter.build(OpenAICreateSpeechRequest(input=text, seed=7), [], False))
|
||||
|
||||
expected = build_gepard_prompt_ids([11, 12, 13], config=cfg)
|
||||
assert prepared.prompt["prompt_token_ids"] == expected
|
||||
assert prepared.prompt["additional_information"] == {"text": [text]}
|
||||
assert prepared.model_type == "gepard"
|
||||
assert prepared.output_policy.accumulate_nonstreaming is True
|
||||
|
||||
|
||||
def test_gepard_loads_tokenizer_and_config_at_served_revision(monkeypatch) -> None:
|
||||
adapter = _adapter(revision="abc123")
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_from_checkpoint(model, backbone_config=None, revision=None):
|
||||
seen["config"] = (model, revision)
|
||||
return GepardConfig()
|
||||
|
||||
class _Tok:
|
||||
def __init__(self, *args, **kwargs):
|
||||
seen["tok"] = (args[0] if args else None, kwargs.get("revision"))
|
||||
|
||||
def __call__(self, text, add_special_tokens=False):
|
||||
return {"input_ids": [11, 12, 13]}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.models.gepard.configuration_gepard.GepardConfig.from_checkpoint",
|
||||
staticmethod(fake_from_checkpoint),
|
||||
)
|
||||
monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", _Tok)
|
||||
prepared = asyncio.run(adapter.build(OpenAICreateSpeechRequest(input="Hello from Gepard."), [], False))
|
||||
assert seen["config"] == (_MODEL, "abc123")
|
||||
assert seen["tok"] == (_MODEL, "abc123")
|
||||
assert prepared.prompt["prompt_token_ids"] == build_gepard_prompt_ids([11, 12, 13], config=GepardConfig())
|
||||
|
||||
|
||||
def _gepard_mock_output(sample_rate: int = 22050, num_samples: int = 2048) -> OmniRequestOutput:
|
||||
class MockCompletionOutput:
|
||||
def __init__(self) -> None:
|
||||
self.index = 0
|
||||
self.text = ""
|
||||
self.token_ids: list[int] = []
|
||||
self.finish_reason = "stop"
|
||||
self.stop_reason = None
|
||||
self.logprobs = None
|
||||
|
||||
class MockRequestOutput:
|
||||
def __init__(self) -> None:
|
||||
self.request_id = "speech-gepard-mock"
|
||||
self.outputs = [MockCompletionOutput()]
|
||||
self.multimodal_output = {
|
||||
"audio": torch.zeros(num_samples),
|
||||
"sr": torch.tensor(sample_rate),
|
||||
}
|
||||
self.finished = True
|
||||
self.prompt_token_ids = None
|
||||
self.encoder_prompt_token_ids = None
|
||||
self.num_cached_tokens = None
|
||||
self.prompt_logprobs = None
|
||||
self.kv_transfer_params = None
|
||||
|
||||
output = OmniRequestOutput.from_stage_output(MockRequestOutput(), stage_id=0, final_output_type="audio")
|
||||
output.metrics = {"stage_metrics": {"0": {"num_tokens_in": 0, "num_tokens_out": 2}}}
|
||||
return output
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gepard_server(mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value={"default"},
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=22050.0,
|
||||
)
|
||||
|
||||
async def _gen(*_args, **_kwargs):
|
||||
yield _gepard_mock_output()
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model=_MODEL, async_chunk=False, revision=None)
|
||||
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=1000, seed=42, extra_args=None)]
|
||||
mock_engine_client.tts_batch_max_items = 32
|
||||
mock_engine_client.generate = mocker.MagicMock(side_effect=lambda **_k: _gen())
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(model_stage="gepard", model_arch="GepardTalkerForConditionalGeneration"),
|
||||
tts_args={},
|
||||
)
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
cfg = GepardConfig()
|
||||
server._adapter._gepard_config = cfg
|
||||
server._adapter._tokenizer = lambda text, add_special_tokens=False: {"input_ids": [11, 12, 13]}
|
||||
yield server
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_gepard_prepare_uses_prompt_builder(gepard_server) -> None:
|
||||
request_id, _generator, _ = asyncio.run(
|
||||
gepard_server._prepare_speech_generation(OpenAICreateSpeechRequest(input="Hello from Gepard."))
|
||||
)
|
||||
assert request_id.startswith("speech-")
|
||||
prompt = gepard_server.engine_client.generate.call_args.kwargs["prompt"]
|
||||
assert prompt["prompt_token_ids"] == build_gepard_prompt_ids([11, 12, 13], config=GepardConfig())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", ["wav", "pcm", "flac", "mp3"])
|
||||
def test_gepard_mocked_engine_encodes_response_formats(gepard_server, response_format: str) -> None:
|
||||
audio, media_type = asyncio.run(
|
||||
gepard_server._generate_audio_bytes(
|
||||
OpenAICreateSpeechRequest(input="Hello from Gepard.", response_format=response_format)
|
||||
)
|
||||
)
|
||||
assert isinstance(audio, (bytes, str))
|
||||
assert len(audio) > 0
|
||||
expected_media_type = {"wav": "audio/wav", "pcm": "audio/pcm", "flac": "audio/flac", "mp3": "audio/mpeg"}
|
||||
assert media_type == expected_media_type[response_format]
|
||||
|
||||
|
||||
def test_gepard_streaming_rejects_non_pcm_wav() -> None:
|
||||
with pytest.raises(ValidationError, match="requires response_format='pcm' or 'wav'"):
|
||||
OpenAICreateSpeechRequest(input="Hello from Gepard.", stream=True, response_format="mp3")
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""L1 tests for the image response encoder's compression contract."""
|
||||
|
||||
import base64
|
||||
import io
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.entrypoints.openai.image_api_utils import (
|
||||
encode_image_base64_with_compression,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image():
|
||||
# Photographic-content proxy: noise-plus-gradient content compresses very
|
||||
# differently across PNG compress levels, which is what the mapping under
|
||||
# test controls.
|
||||
import numpy as np
|
||||
|
||||
rng = np.random.default_rng(0)
|
||||
grad = np.linspace(0, 255, 256, dtype=np.float32)
|
||||
arr = np.clip(grad[None, :, None] + rng.integers(-24, 25, (256, 256, 3)), 0, 255).astype(np.uint8)
|
||||
return Image.fromarray(arr, mode="RGB")
|
||||
|
||||
|
||||
def test_png_output_compression_maps_to_compress_level(image):
|
||||
"""output_compression 100 -> compress_level 0, 1 -> compress_level 9."""
|
||||
|
||||
def encode(compression):
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG", compress_level=max(0, min(9, 9 - compression // 11)))
|
||||
return len(base64.b64encode(buffer.getvalue()))
|
||||
|
||||
fast = encode_image_base64_with_compression(image, format="png", output_compression=100)
|
||||
small = encode_image_base64_with_compression(image, format="png", output_compression=1)
|
||||
|
||||
# Level-0 PNG barely compresses; the mapped level-9 encode is smaller.
|
||||
assert len(small) < len(fast)
|
||||
# The mapping must produce the same sizes as the equivalent compress_level.
|
||||
assert len(fast) == encode(100)
|
||||
assert len(small) == encode(1)
|
||||
|
||||
|
||||
def test_jpeg_output_compression_maps_to_quality(image):
|
||||
high = base64.b64decode(encode_image_base64_with_compression(image, format="jpeg", output_compression=100))
|
||||
low = base64.b64decode(encode_image_base64_with_compression(image, format="jpeg", output_compression=10))
|
||||
|
||||
assert len(high) > len(low)
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.images import (
|
||||
_FILE_RESPONSE_CHUNK_SIZE,
|
||||
ImageData,
|
||||
ImageGenerationResponse,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_NEWLINE_HEAVY_PAYLOAD = (b"\x89PNG\n\x1a\n" * 64) + bytes(range(256)) * 8
|
||||
|
||||
|
||||
def _response(*payloads: bytes) -> ImageGenerationResponse:
|
||||
return ImageGenerationResponse(
|
||||
created=0,
|
||||
data=[ImageData(b64_json=base64.b64encode(payload).decode()) for payload in payloads],
|
||||
output_format="png",
|
||||
size="1x1",
|
||||
)
|
||||
|
||||
|
||||
async def _collect(response: StreamingResponse) -> list[bytes]:
|
||||
return [bytes(chunk) async for chunk in response.body_iterator]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_file_response_chunks_ignore_newlines() -> None:
|
||||
payload = _NEWLINE_HEAVY_PAYLOAD
|
||||
assert b"\n" in payload
|
||||
|
||||
response = _response(payload).stream_response()
|
||||
chunks = await _collect(response)
|
||||
|
||||
assert len(chunks) == 1
|
||||
assert b"".join(chunks) == payload
|
||||
assert response.headers["content-length"] == str(len(payload))
|
||||
assert response.headers["content-type"] == "image/png"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_file_response_chunks_are_bounded() -> None:
|
||||
payload = _NEWLINE_HEAVY_PAYLOAD * 200
|
||||
expected_chunks = -(-len(payload) // _FILE_RESPONSE_CHUNK_SIZE)
|
||||
assert expected_chunks > 1
|
||||
|
||||
chunks = await _collect(_response(payload).stream_response())
|
||||
|
||||
assert len(chunks) == expected_chunks
|
||||
assert max(len(chunk) for chunk in chunks) <= _FILE_RESPONSE_CHUNK_SIZE
|
||||
assert b"".join(chunks) == payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zip_file_response_streams_whole_archive() -> None:
|
||||
payloads = (_NEWLINE_HEAVY_PAYLOAD, b"second image\nwith a newline")
|
||||
|
||||
response = _response(*payloads).stream_response()
|
||||
chunks = await _collect(response)
|
||||
|
||||
zip_bytes = b"".join(chunks)
|
||||
assert len(chunks) == -(-len(zip_bytes) // _FILE_RESPONSE_CHUNK_SIZE)
|
||||
assert response.headers["content-length"] == str(len(zip_bytes))
|
||||
assert response.headers["content-type"] == "application/zip"
|
||||
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as archive:
|
||||
assert archive.read("image_0.png") == payloads[0]
|
||||
assert archive.read("image_1.png") == payloads[1]
|
||||
@@ -2314,3 +2314,72 @@ def test_image_edits_size_auto_preserves_bridge_size(async_omni_stage_configs_on
|
||||
assert captured_prompt["prompt"].count("<img>") == 2, (
|
||||
f"N=2 reference images must emit 2 <img> placeholders in AR prompt; got {captured_prompt[KEY].count(IMG)} -- prompt: {captured_prompt[KEY]!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason=(
|
||||
"Pre-existing gap, not introduced by this change: the /v1/images/edits route in "
|
||||
"api_server.py never calls resolve_stop_token_ids at all (grep: api_server.py has "
|
||||
"no stop_token_ids reference), so the AR stage keeps SamplingParams' default empty "
|
||||
"list. The omitted-bot_task fix landed in serving_chat.py, which is the only "
|
||||
"production caller; that path is covered by "
|
||||
"test_serving_chat_multistage_generation.py::"
|
||||
"test_build_multistage_generation_inputs_omitted_bot_task_matches_prompt_default. "
|
||||
"Wiring the images/edits route through the same seam needs a tokenizer in that "
|
||||
"scope and a decision on whether HunyuanImage3 it2i should resolve AR stop tokens "
|
||||
"unconditionally (today the whole AR block is gated on an explicit bot_task / "
|
||||
"use_system_prompt / system_prompt), so it is tracked separately."
|
||||
),
|
||||
)
|
||||
def test_image_edits_omitted_bot_task_stop_tokens_match_prompt_default(
|
||||
async_omni_stage_configs_only_client,
|
||||
):
|
||||
"""Regression: an omitted bot_task must resolve identically for the AR
|
||||
prompt and its stop_token_ids.
|
||||
|
||||
build_prompt/build_prompt_tokens default an omitted bot_task per-task
|
||||
(e.g. "think" for the base "it2i" task, since it isn't itself a key in
|
||||
_TASK_PRESETS). resolve_stop_token_ids must land on that same default
|
||||
to compute the matching stop set. Passing the raw (still-None) outer
|
||||
bot_task variable to resolve_stop_token_ids -- instead of mirroring
|
||||
build_kwargs's own omitted-or-not "bot_task" entry -- made it normalize
|
||||
bot_task=None instead of "think", so it fell through to the full
|
||||
<img_ratio_*> stop range instead of the think/recaption-only pair.
|
||||
|
||||
An explicit (non-"auto") size is required to reach this: with
|
||||
need_ratio=True (size="auto"), resolve_stop_token_ids returns the full
|
||||
ratio range regardless of bot_task, so the two code paths only visibly
|
||||
disagree once a concrete size selects the narrower think/recaption stop
|
||||
set.
|
||||
"""
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
||||
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS,
|
||||
)
|
||||
|
||||
img = make_test_image_bytes((64, 64))
|
||||
response = async_omni_stage_configs_only_client.post(
|
||||
"/v1/images/edits",
|
||||
files=[("image", img)],
|
||||
data={
|
||||
"prompt": "make it neon",
|
||||
"size": "512x512",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
engine = async_omni_stage_configs_only_client.app.state.engine_client
|
||||
captured = engine.captured_sampling_params_list
|
||||
assert captured is not None
|
||||
|
||||
ar_params = captured[0]
|
||||
expected_stop_token_ids = [
|
||||
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["</think>"],
|
||||
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["</recaption>"],
|
||||
]
|
||||
assert ar_params.stop_token_ids == expected_stop_token_ids, (
|
||||
f"omitted bot_task with an explicit size must resolve stop_token_ids "
|
||||
f"for the default 'think' bot_task ({expected_stop_token_ids}); got "
|
||||
f"{ar_params.stop_token_ids} -- this is the full ratio range, meaning "
|
||||
"resolve_stop_token_ids disagreed with build_prompt_tokens's default."
|
||||
)
|
||||
@@ -0,0 +1,302 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.tts_adapters import moss_tts as adapter_module
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import MossTTSAdapter
|
||||
from vllm_omni.model_executor.models.moss_tts import reference_encoder
|
||||
from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("hf_config", "expected_codec_path"),
|
||||
[
|
||||
(SimpleNamespace(), "OpenMOSS-Team/MOSS-Audio-Tokenizer"),
|
||||
(
|
||||
SimpleNamespace(codec_model_name_or_path="/models/custom-moss-codec"),
|
||||
"/models/custom-moss-codec",
|
||||
),
|
||||
],
|
||||
ids=["default-codec", "configured-codec"],
|
||||
)
|
||||
def test_realtime_components_use_the_realtime_model_and_codec(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
hf_config: SimpleNamespace,
|
||||
expected_codec_path: str,
|
||||
) -> None:
|
||||
engine_client = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
model="OpenMOSS-Team/MOSS-TTS-Realtime",
|
||||
hf_config=hf_config,
|
||||
)
|
||||
)
|
||||
|
||||
server = MossTTSAdapter(SpeechServingContext(server=object(), engine_client=engine_client))
|
||||
|
||||
tokenizer = object()
|
||||
codec = type("Codec", (), {"to": lambda self, device: self, "eval": lambda self: self})()
|
||||
processor_calls = []
|
||||
|
||||
class Processor:
|
||||
def __init__(self, *, tokenizer):
|
||||
processor_calls.append(tokenizer)
|
||||
|
||||
class_calls = []
|
||||
tokenizer_calls = []
|
||||
codec_calls = []
|
||||
|
||||
def fake_get_class(class_reference, model_id):
|
||||
class_calls.append((class_reference, model_id))
|
||||
return Processor
|
||||
|
||||
def fake_load_tokenizer(model_id, *, trust_remote_code):
|
||||
tokenizer_calls.append((model_id, trust_remote_code))
|
||||
return tokenizer
|
||||
|
||||
def fake_load_codec(model_id, *, trust_remote_code):
|
||||
codec_calls.append((model_id, trust_remote_code))
|
||||
return codec
|
||||
|
||||
monkeypatch.setattr(adapter_module, "get_class_from_dynamic_module", fake_get_class)
|
||||
monkeypatch.setattr(adapter_module.AutoTokenizer, "from_pretrained", fake_load_tokenizer)
|
||||
monkeypatch.setattr(adapter_module.AutoModel, "from_pretrained", fake_load_codec)
|
||||
|
||||
components = server._get_moss_realtime_components()
|
||||
|
||||
assert components[0] is tokenizer
|
||||
assert components[2] is codec
|
||||
assert server._get_moss_realtime_components() is components
|
||||
assert class_calls == [
|
||||
(
|
||||
"processing_mossttsrealtime.MossTTSRealtimeProcessor",
|
||||
"OpenMOSS-Team/MOSS-TTS-Realtime",
|
||||
)
|
||||
]
|
||||
assert tokenizer_calls == [("OpenMOSS-Team/MOSS-TTS-Realtime", True)]
|
||||
assert codec_calls == [(expected_codec_path, True)]
|
||||
assert processor_calls == [tokenizer]
|
||||
|
||||
|
||||
def test_realtime_serving_builds_the_talker_prompt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
server = MossTTSAdapter(SpeechServingContext(server=SimpleNamespace(uploaded_speakers={})))
|
||||
server._moss_variant = "realtime"
|
||||
tokenizer = object()
|
||||
processor = object()
|
||||
codec = object()
|
||||
request_thread = threading.get_ident()
|
||||
|
||||
def load_components():
|
||||
assert threading.get_ident() != request_thread
|
||||
return tokenizer, processor, codec
|
||||
|
||||
server._get_moss_realtime_components = load_components
|
||||
|
||||
reference_codes = torch.arange(64, dtype=torch.int64).reshape(4, 16)
|
||||
encode = AsyncMock(return_value=([reference_codes], {0: "reference-cache-key"}))
|
||||
monkeypatch.setattr(server, "_encode_moss_references", encode)
|
||||
|
||||
build_call = None
|
||||
|
||||
def fake_build(actual_tokenizer, actual_processor, text, actual_codes):
|
||||
nonlocal build_call
|
||||
build_call = (actual_tokenizer, actual_processor, text, actual_codes)
|
||||
return {
|
||||
"prompt_token_ids": [10, 11],
|
||||
"codes": {"ref": actual_codes},
|
||||
"ids": {"all": [12]},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(adapter_module, "build_realtime_prompt", fake_build)
|
||||
|
||||
params = asyncio.run(
|
||||
server._build_moss_tts_params(
|
||||
OpenAICreateSpeechRequest(
|
||||
input="speak this text",
|
||||
ref_audio="data:audio/wav;base64,AAAA",
|
||||
max_new_tokens=50,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
encode.assert_awaited_once()
|
||||
assert encode.call_args.kwargs == {"has_inline_ref_audio": False, "two_speaker": False}
|
||||
assert encode.call_args.args[0].ref_audio == "data:audio/wav;base64,AAAA"
|
||||
assert build_call == (tokenizer, processor, "speak this text", reference_codes)
|
||||
assert params == {
|
||||
"prompt_token_ids": [10, 11],
|
||||
"codes": {"ref": reference_codes},
|
||||
"ids": {"all": [12]},
|
||||
"max_new_frames": [50],
|
||||
"ref_audio_cache_key": "reference-cache-key",
|
||||
}
|
||||
assert "prompt_audio_array" not in params
|
||||
|
||||
|
||||
def test_cancelled_component_waiter_does_not_duplicate_codec_load(monkeypatch):
|
||||
adapter = MossTTSAdapter(
|
||||
SpeechServingContext(
|
||||
server=object(),
|
||||
engine_client=SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="OpenMOSS-Team/MOSS-TTS-Realtime", hf_config=SimpleNamespace())
|
||||
),
|
||||
)
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
second_started = threading.Event()
|
||||
loads = []
|
||||
codec = type("Codec", (), {"to": lambda self, device: self, "eval": lambda self: self})()
|
||||
|
||||
def load_codec(*args, **kwargs):
|
||||
loads.append(args)
|
||||
started.set()
|
||||
assert release.wait(5), "codec loader was not released"
|
||||
return codec
|
||||
|
||||
monkeypatch.setattr(adapter_module, "get_class_from_dynamic_module", lambda *args: lambda **kwargs: object())
|
||||
monkeypatch.setattr(adapter_module.AutoTokenizer, "from_pretrained", lambda *args, **kwargs: object())
|
||||
monkeypatch.setattr(adapter_module.AutoModel, "from_pretrained", load_codec)
|
||||
|
||||
def second_load():
|
||||
second_started.set()
|
||||
return adapter._get_moss_realtime_components()
|
||||
|
||||
async def run():
|
||||
first = asyncio.create_task(asyncio.to_thread(adapter._get_moss_realtime_components))
|
||||
try:
|
||||
assert await asyncio.to_thread(started.wait, 5)
|
||||
first.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await first
|
||||
second = asyncio.create_task(asyncio.to_thread(second_load))
|
||||
try:
|
||||
assert await asyncio.to_thread(second_started.wait, 5)
|
||||
finally:
|
||||
release.set()
|
||||
components = await asyncio.wait_for(second, 5)
|
||||
assert components[2] is codec
|
||||
assert adapter._get_moss_realtime_components() is components
|
||||
assert len(loads) == 1
|
||||
finally:
|
||||
release.set()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("voice_name", [None, "speaker"])
|
||||
def test_realtime_reference_cache_reuses_codes(voice_name):
|
||||
cache = SpeakerEmbeddingCache()
|
||||
resolve = AsyncMock(return_value=([0.0], 24000, "audio-key"))
|
||||
request_thread = threading.get_ident()
|
||||
encode_calls = []
|
||||
expected = torch.arange(64).reshape(4, 16)
|
||||
|
||||
def batch_encode(wavs, *, num_quantizers):
|
||||
assert threading.get_ident() != request_thread
|
||||
assert num_quantizers == 16
|
||||
encode_calls.append(wavs)
|
||||
return SimpleNamespace(audio_codes=expected.T.unsqueeze(1), audio_codes_lengths=torch.tensor([4]))
|
||||
|
||||
encoder = reference_encoder.build_reference_encoder(
|
||||
SimpleNamespace(batch_encode=batch_encode), variant="realtime", speaker_cache=cache
|
||||
)
|
||||
|
||||
async def run():
|
||||
kwargs = dict(
|
||||
resolve_ref_audio=resolve,
|
||||
get_artifact_key=lambda key: "content-key",
|
||||
voice_name=voice_name,
|
||||
voice_created_at=1,
|
||||
)
|
||||
try:
|
||||
first, first_key = await encoder.encode("reference", **kwargs)
|
||||
second, second_key = await encoder.encode("reference", **kwargs)
|
||||
assert torch.equal(first, expected)
|
||||
assert torch.equal(second, expected)
|
||||
assert first.dtype == torch.int64
|
||||
assert first_key == "audio-key"
|
||||
assert second_key == (None if voice_name else "audio-key")
|
||||
second.zero_()
|
||||
third, _ = await encoder.encode("another-reference", **kwargs)
|
||||
assert torch.equal(third, expected)
|
||||
assert len(encode_calls) == 1
|
||||
assert resolve.await_count == (1 if voice_name else 3)
|
||||
key = cache.make_cache_key(
|
||||
voice_name or "ref:content-key", "moss_tts_realtime_nq16", 1 if voice_name else 0
|
||||
)
|
||||
assert cache.get(key)["codes"].dtype == torch.int32
|
||||
if voice_name:
|
||||
kwargs["voice_created_at"] = 2
|
||||
await encoder.encode("reference", **kwargs)
|
||||
assert len(encode_calls) == 2
|
||||
assert resolve.await_count == 2
|
||||
finally:
|
||||
await encoder.aclose()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_realtime_reference_batch_trims_each_item():
|
||||
cache = SpeakerEmbeddingCache()
|
||||
codes = torch.arange(16 * 2 * 5).reshape(16, 2, 5)
|
||||
calls = []
|
||||
|
||||
def batch_encode(wavs, *, num_quantizers):
|
||||
calls.append(wavs)
|
||||
assert num_quantizers == 16
|
||||
assert [wav.shape for wav in wavs] == [torch.Size([8]), torch.Size([12])]
|
||||
return SimpleNamespace(audio_codes=codes, audio_codes_lengths=torch.tensor([3, 5]))
|
||||
|
||||
encoder = reference_encoder.build_reference_encoder(
|
||||
SimpleNamespace(batch_encode=batch_encode), variant="realtime", speaker_cache=cache
|
||||
)
|
||||
|
||||
async def resolve(ref):
|
||||
return [0.0] * (8 if ref == "short" else 12), 24000, ref
|
||||
|
||||
async def run():
|
||||
try:
|
||||
short, long = await asyncio.gather(
|
||||
encoder.encode("short", resolve_ref_audio=resolve, get_artifact_key=lambda key: key),
|
||||
encoder.encode("long", resolve_ref_audio=resolve, get_artifact_key=lambda key: key),
|
||||
)
|
||||
torch.testing.assert_close(short[0], codes[:, 0, :3].T)
|
||||
torch.testing.assert_close(long[0], codes[:, 1, :5].T)
|
||||
assert len(calls) == 1
|
||||
finally:
|
||||
await encoder.aclose()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_failed_realtime_reference_encoding_does_not_populate_cache():
|
||||
cache = SpeakerEmbeddingCache()
|
||||
resolve = AsyncMock(return_value=([0.0], 24000, "audio-key"))
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise ValueError("invalid reference")
|
||||
|
||||
encoder = reference_encoder.build_reference_encoder(
|
||||
SimpleNamespace(batch_encode=fail), variant="realtime", speaker_cache=cache
|
||||
)
|
||||
|
||||
async def run():
|
||||
try:
|
||||
with pytest.raises(ValueError, match="invalid reference"):
|
||||
await encoder.encode("reference", resolve_ref_audio=resolve, get_artifact_key=lambda key: None)
|
||||
assert cache.memory_bytes() == 0
|
||||
finally:
|
||||
await encoder.aclose()
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -20,6 +20,7 @@ def _make_app(engine_client):
|
||||
from vllm_omni.entrypoints.openai.api_server import router
|
||||
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 1
|
||||
app.include_router(router)
|
||||
app.state.engine_client = engine_client
|
||||
app.state.sleeping_stages = set()
|
||||
@@ -253,6 +254,7 @@ def pure_diffusion_app(pure_diffusion_engine, mocker):
|
||||
)
|
||||
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 1
|
||||
app.include_router(router)
|
||||
|
||||
mocker.patch(
|
||||
|
||||
@@ -310,6 +310,29 @@ def test_handle_connection_closes_websocket_on_idle_timeout(monkeypatch):
|
||||
serving.infer.assert_not_called()
|
||||
|
||||
|
||||
def test_handle_connection_can_disable_idle_timeout(monkeypatch):
|
||||
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
|
||||
|
||||
async def fail_wait_for(*_args, **_kwargs):
|
||||
pytest.fail("asyncio.wait_for should not be used when the idle timeout is disabled")
|
||||
|
||||
monkeypatch.setattr(openpi_connection.asyncio, "wait_for", fail_wait_for)
|
||||
websocket = FakeWebSocket([{"type": "websocket.disconnect"}])
|
||||
serving = _serving_mock()
|
||||
|
||||
asyncio.run(
|
||||
openpi_connection.RobotRealtimeConnection(
|
||||
websocket,
|
||||
serving,
|
||||
idle_timeout=None,
|
||||
).handle_connection()
|
||||
)
|
||||
|
||||
assert websocket.accepted is True
|
||||
assert websocket.closed is False
|
||||
serving.infer.assert_not_called()
|
||||
|
||||
|
||||
def test_handle_connection_keeps_session_state_per_websocket(monkeypatch):
|
||||
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
|
||||
requests = {
|
||||
@@ -343,6 +366,73 @@ def test_handle_connection_keeps_session_state_per_websocket(monkeypatch):
|
||||
assert calls[2].kwargs == {"session_id": "session-b", "reset": True}
|
||||
|
||||
|
||||
def test_handle_connection_keeps_interleaved_parallel_session_state(monkeypatch):
|
||||
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
|
||||
requests = {
|
||||
b"a1": {"prompt": "first-a", "session_id": "session-a"},
|
||||
b"b1": {"prompt": "first-b", "session_id": "session-b"},
|
||||
b"a2": {"prompt": "second-a", "session_id": "session-a"},
|
||||
b"b2": {"prompt": "second-b", "session_id": "session-b"},
|
||||
}
|
||||
monkeypatch.setattr(openpi_connection, "_unpack", lambda data: dict(requests[data]))
|
||||
serving = _serving_mock()
|
||||
websocket = FakeWebSocket(
|
||||
[{"type": "websocket.receive", "bytes": key} for key in (b"a1", b"b1", b"a2", b"b2")]
|
||||
+ [{"type": "websocket.disconnect"}]
|
||||
)
|
||||
|
||||
asyncio.run(openpi_connection.RobotRealtimeConnection(websocket, serving).handle_connection())
|
||||
|
||||
assert [call.kwargs for call in serving.infer.await_args_list] == [
|
||||
{"session_id": "session-a", "reset": True},
|
||||
{"session_id": "session-b", "reset": True},
|
||||
{"session_id": "session-a", "reset": False},
|
||||
{"session_id": "session-b", "reset": False},
|
||||
]
|
||||
|
||||
|
||||
def test_handle_connection_bounds_tracked_sessions_without_resetting_live_ones(monkeypatch):
|
||||
"""A client may hold one socket across many episodes, minting a fresh
|
||||
session id per episode and never sending ``reset`` (RoboLab does exactly
|
||||
this). Session tracking must stay bounded, and evicting finished episodes
|
||||
must not disturb the sessions that are still running.
|
||||
"""
|
||||
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
|
||||
monkeypatch.setattr(openpi_connection, "MAX_TRACKED_SESSIONS", 8)
|
||||
|
||||
# One long-lived session interleaved with far more short episodes than the cap.
|
||||
episodes = 40
|
||||
keys = []
|
||||
requests = {}
|
||||
for episode in range(episodes):
|
||||
for session_id in (f"episode-{episode}", "live-env"):
|
||||
key = f"{session_id}@{episode}".encode()
|
||||
keys.append(key)
|
||||
requests[key] = {"prompt": "pick", "session_id": session_id}
|
||||
|
||||
monkeypatch.setattr(openpi_connection, "_unpack", lambda data: dict(requests[data]))
|
||||
serving = _serving_mock()
|
||||
websocket = FakeWebSocket(
|
||||
[{"type": "websocket.receive", "bytes": key} for key in keys] + [{"type": "websocket.disconnect"}]
|
||||
)
|
||||
connection = openpi_connection.RobotRealtimeConnection(websocket, serving)
|
||||
|
||||
asyncio.run(connection.handle_connection())
|
||||
|
||||
assert len(connection._seen_sessions) <= 8
|
||||
|
||||
resets_by_session: dict[str, list[bool]] = {}
|
||||
for call in serving.infer.await_args_list:
|
||||
resets_by_session.setdefault(call.kwargs["session_id"], []).append(call.kwargs["reset"])
|
||||
|
||||
# Each new episode resets exactly once, on its first observation.
|
||||
for episode in range(episodes):
|
||||
assert resets_by_session[f"episode-{episode}"] == [True]
|
||||
# The continuously active session is never evicted, so it resets only once
|
||||
# despite ``episodes`` other sessions churning past the cap.
|
||||
assert resets_by_session["live-env"] == [True] + [False] * (episodes - 1)
|
||||
|
||||
|
||||
def test_handle_connection_reset_endpoint_resets_next_infer(monkeypatch):
|
||||
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
|
||||
requests = {
|
||||
|
||||
@@ -15,6 +15,7 @@ from starlette.testclient import TestClient
|
||||
|
||||
from vllm_omni.entrypoints.openpi import connection as openpi_connection
|
||||
from vllm_omni.entrypoints.openpi import serving as openpi_serving
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -137,6 +138,36 @@ def test_policy_server_config_reads_stage_config_model_config():
|
||||
assert serving.policy_server_config.to_dict() == policy_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("inline", [False, True], ids=["out_of_process", "inline"])
|
||||
def test_policy_server_config_reads_typed_runtime(inline):
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||
from vllm_omni.diffusion.models.pi0_pipeline_config import PI0_PIPELINE
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
typed_stage = VllmOmniConfig.from_pipeline_config(
|
||||
PI0_PIPELINE, deploy_config_path=get_deploy_config_path("pi0.yaml")
|
||||
).stage_configs[0]
|
||||
policy_config = typed_stage.diffusion_config.model_config["policy_server_config"]
|
||||
stage_client = SimpleNamespace(stage_type="diffusion")
|
||||
if inline:
|
||||
# Worker-enriched inline metadata remains authoritative over the
|
||||
# startup stage config; only the process client needs the fallback.
|
||||
policy_config = {**policy_config, "worker_metadata": "loaded"}
|
||||
stage_client.od_config = SimpleNamespace(model_config={"policy_server_config": policy_config})
|
||||
engine_client = AsyncOmni.__new__(AsyncOmni)
|
||||
engine_client.engine = SimpleNamespace(
|
||||
stage_clients=[stage_client],
|
||||
stage_configs=[typed_stage],
|
||||
stage_vllm_configs=[None],
|
||||
get_diffusion_od_config=lambda: SimpleNamespace(model_class_name="Pi0Pipeline"),
|
||||
)
|
||||
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=engine_client)
|
||||
|
||||
assert serving.policy_server_config.to_dict() == policy_config
|
||||
|
||||
|
||||
def test_policy_server_config_reads_omegaconf_stage_config():
|
||||
engine_client = SimpleNamespace(
|
||||
get_diffusion_od_config=lambda: None,
|
||||
@@ -177,6 +208,22 @@ def test_create_policy_server_returns_none_without_policy_config():
|
||||
assert serving is None
|
||||
|
||||
|
||||
def test_legacy_policy_request_defaults_key_no_longer_controls_serving():
|
||||
od_config = SimpleNamespace(
|
||||
model_config={
|
||||
"policy_server_config": {},
|
||||
"policy_request_defaults": "malformed legacy value",
|
||||
}
|
||||
)
|
||||
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI.create_policy_server(
|
||||
engine_client=SimpleNamespace(get_diffusion_od_config=lambda: od_config),
|
||||
model_name="generic-policy",
|
||||
)
|
||||
|
||||
assert serving is not None
|
||||
|
||||
|
||||
def test_policy_server_config_allows_explicit_empty_config():
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(
|
||||
engine_client=_engine_with_policy_config(policy_config={}),
|
||||
@@ -221,6 +268,54 @@ def test_build_request_uses_unique_engine_request_id_per_inference():
|
||||
assert request_a.request_id != request_b.request_id
|
||||
|
||||
|
||||
def test_build_request_clones_stage_defaults_before_protocol_fields():
|
||||
default_params = OmniDiffusionSamplingParams(
|
||||
num_inference_steps=12,
|
||||
output_type="latent",
|
||||
extra_args={
|
||||
"format_prompt_as_json": True,
|
||||
"session_id": "configured-session-must-not-win",
|
||||
"reset": False,
|
||||
"nested": {"values": []},
|
||||
},
|
||||
)
|
||||
od_config = SimpleNamespace(model_config={"policy_server_config": {}})
|
||||
engine_client = SimpleNamespace(
|
||||
get_diffusion_od_config=lambda: od_config,
|
||||
default_sampling_params_list=[default_params],
|
||||
)
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(
|
||||
engine_client=engine_client,
|
||||
model_name="custom-policy",
|
||||
)
|
||||
|
||||
request_a = serving._build_request(
|
||||
{"prompt": "pick up the object"},
|
||||
session_id="wire-session",
|
||||
reset=True,
|
||||
)
|
||||
request_b = serving._build_request(
|
||||
{"prompt": "pick up the object"},
|
||||
session_id="wire-session",
|
||||
reset=False,
|
||||
)
|
||||
request_c = serving._build_request(
|
||||
{"prompt": "pick up the object", "sampling_params": {"num_inference_steps": 4}},
|
||||
session_id="wire-session",
|
||||
reset=False,
|
||||
)
|
||||
|
||||
assert request_a.sampling_params.extra_args["format_prompt_as_json"] is True
|
||||
assert request_a.sampling_params.extra_args["session_id"] == "wire-session"
|
||||
assert request_a.sampling_params.extra_args["reset"] is True
|
||||
assert request_a.sampling_params.num_inference_steps == 12
|
||||
assert request_a.sampling_params.output_type == "latent"
|
||||
assert request_c.sampling_params.num_inference_steps == 4
|
||||
request_a.sampling_params.extra_args["nested"]["values"].append("request-a")
|
||||
assert request_b.sampling_params.extra_args["nested"] == {"values": []}
|
||||
assert default_params.extra_args["nested"] == {"values": []}
|
||||
|
||||
|
||||
def test_build_request_forwards_seed_to_sampling_params():
|
||||
"""``seed`` in the inference message is the engine-level seed; omitted, the request auto-seeds."""
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=_engine_with_policy_config())
|
||||
@@ -233,6 +328,46 @@ def test_build_request_forwards_seed_to_sampling_params():
|
||||
assert isinstance(unseeded.sampling_params.seed, int)
|
||||
|
||||
|
||||
# Per-request inference parameters live in their own namespace so robot feature
|
||||
# names stay open-ended. Unknown keys inside it are ignored rather than rejected.
|
||||
def test_build_request_forwards_num_inference_steps():
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=_engine_with_policy_config())
|
||||
|
||||
request = serving._build_request(
|
||||
{"prompt": "pick up the object", "sampling_params": {"num_inference_steps": 4}},
|
||||
session_id="session-a",
|
||||
reset=True,
|
||||
)
|
||||
|
||||
assert request.sampling_params.num_inference_steps == 4
|
||||
# The namespace is an engine concern; the pipeline must not see it as a feature.
|
||||
assert "sampling_params" not in request.sampling_params.extra_args["robot_obs"]
|
||||
|
||||
|
||||
def test_build_request_omits_num_inference_steps_when_not_sent():
|
||||
"""Absent must stay absent: the pipeline reads None as "use the configured
|
||||
default", so materialising a null here would be a different request."""
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=_engine_with_policy_config())
|
||||
|
||||
request = serving._build_request({"prompt": "x"}, session_id="s", reset=True)
|
||||
|
||||
assert request.sampling_params.num_inference_steps is None
|
||||
|
||||
|
||||
def test_build_request_does_not_forward_arbitrary_observation_keys():
|
||||
"""A robot observation may carry any feature name; none of them become
|
||||
engine parameters."""
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=_engine_with_policy_config())
|
||||
|
||||
request = serving._build_request(
|
||||
{"prompt": "x", "state": np.zeros(32), "guidance_scale": 7.5}, session_id="s", reset=True
|
||||
)
|
||||
|
||||
extra_args = request.sampling_params.extra_args
|
||||
assert set(extra_args) == {"reset", "session_id", "robot_obs"}
|
||||
assert extra_args["robot_obs"]["guidance_scale"] == 7.5
|
||||
|
||||
|
||||
def test_infer_keeps_session_state_but_uses_unique_engine_request_ids():
|
||||
engine = RecordingEngine()
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=engine)
|
||||
@@ -257,6 +392,29 @@ def test_infer_keeps_session_state_but_uses_unique_engine_request_ids():
|
||||
assert sampling_params_b.extra_args["reset"] is False
|
||||
|
||||
|
||||
def test_infer_yields_event_loop_while_engine_is_running():
|
||||
class DelayedEngine(RecordingEngine):
|
||||
def generate(self, *, prompt, request_id, sampling_params_list):
|
||||
async def _generate():
|
||||
await asyncio.sleep(0.03)
|
||||
yield SimpleNamespace(multimodal_output={"actions": [0.0]})
|
||||
|
||||
return _generate()
|
||||
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=DelayedEngine())
|
||||
|
||||
async def run_test():
|
||||
infer_task = asyncio.create_task(serving.infer({"prompt": "pick"}, session_id="session-a", reset=True))
|
||||
ticks = 0
|
||||
while not infer_task.done():
|
||||
await asyncio.sleep(0.005)
|
||||
ticks += 1
|
||||
await infer_task
|
||||
return ticks
|
||||
|
||||
assert asyncio.run(run_test()) >= 2
|
||||
|
||||
|
||||
def test_two_websocket_clients_without_session_id_do_not_conflict(monkeypatch):
|
||||
monkeypatch.setattr(openpi_connection, "_pack", _json_pack)
|
||||
monkeypatch.setattr(openpi_connection, "_unpack", _json_unpack)
|
||||
@@ -362,3 +520,17 @@ def test_extract_actions_does_not_iterate_result_object():
|
||||
actions = serving._extract_actions(IterableResult())
|
||||
|
||||
np.testing.assert_allclose(actions, np.array([[1.0, 2.0, 3.0]], dtype=np.float32))
|
||||
|
||||
|
||||
def test_drop_session_calls_dreamzero_close_hook():
|
||||
closed: list[str] = []
|
||||
pipeline = SimpleNamespace(close_ar_diffusion_session=lambda session_id: closed.append(session_id))
|
||||
engine_client = SimpleNamespace(
|
||||
model_config={"policy_server_config": TEST_POLICY_SERVER_CONFIG},
|
||||
model_runner=SimpleNamespace(pipeline=pipeline),
|
||||
)
|
||||
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=engine_client)
|
||||
|
||||
serving.drop_session("rollout-1")
|
||||
|
||||
assert closed == ["rollout-1"]
|
||||
@@ -18,21 +18,27 @@ import io
|
||||
import json
|
||||
import os
|
||||
import wave
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import websockets
|
||||
import yaml
|
||||
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import validated_input_wav
|
||||
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import validated_input_wav, validated_soft_interrupt_wav
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import (
|
||||
convert_audio_bytes_to_text,
|
||||
cosine_similarity_text,
|
||||
generate_synthetic_audio,
|
||||
)
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, get_model_prefix
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.config.stage_config import load_deploy_config
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
from vllm_omni.engine.duplex.contracts import duplex_resource_request_belongs_to_session
|
||||
from vllm_omni.engine.duplex.vad import (
|
||||
SILERO_VAD_FILENAME,
|
||||
SILERO_VAD_REPO_ID,
|
||||
SILERO_VAD_REVISION,
|
||||
@@ -175,7 +181,7 @@ async def _run_realtime_audio_roundtrip(
|
||||
if event_type == "session.created":
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.delta":
|
||||
if event_type == "response.output_audio.delta":
|
||||
delta_events += 1
|
||||
sr = event.get("sample_rate_hz")
|
||||
if isinstance(sr, int) and sr > 0:
|
||||
@@ -195,7 +201,7 @@ async def _run_realtime_audio_roundtrip(
|
||||
final_text = event.get("text", "") or "".join(text_chunks)
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.done":
|
||||
if event_type == "response.output_audio.done":
|
||||
break
|
||||
|
||||
if event_type == "error":
|
||||
@@ -315,7 +321,7 @@ def _server_vad_pcm16_input() -> bytes:
|
||||
def _assert_realtime_smoke(result: dict) -> None:
|
||||
out_pcm = result["output_pcm"]
|
||||
assert result["delta_events"] >= 1
|
||||
assert out_pcm, "No output PCM from response.audio.delta"
|
||||
assert out_pcm, "No output PCM from response.output_audio.delta"
|
||||
assert len(out_pcm) % 2 == 0
|
||||
assert len(out_pcm) >= 4096, "Output audio unexpectedly small"
|
||||
assert result["output_sample_rate"] > 0
|
||||
@@ -361,6 +367,15 @@ def _assert_realtime_accuracy(
|
||||
|
||||
|
||||
class TestQwen3OmniRealtimeWebSocket:
|
||||
@pytest.fixture(scope="class")
|
||||
def omni_server(self, request, run_level):
|
||||
# Release the legacy turn deployment before the duplex class below
|
||||
# starts another model server on the same pair of GPUs.
|
||||
from tests.helpers.fixtures.runtime import omni_fixture_lock
|
||||
from tests.helpers.runtime import iter_omni_server
|
||||
|
||||
yield from iter_omni_server(request, run_level, omni_fixture_lock)
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@@ -388,7 +403,7 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", realtime_async_chunk_server_params, indirect=True)
|
||||
def test_long_audio_response_completes_async_chunk(self, omni_server) -> None:
|
||||
"""Regression for #6474: long async-chunk audio must emit response.audio.done."""
|
||||
"""Regression for #6474: long async-chunk audio must emit response.output_audio.done."""
|
||||
issue_6474_pcm16 = _synthetic_pcm16_input(
|
||||
phrase_text=ISSUE_6474_SYNTH_PHRASE_TEXT,
|
||||
)
|
||||
@@ -450,8 +465,8 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
"input_audio_buffer.speech_stopped",
|
||||
"input_audio_buffer.committed",
|
||||
"response.created",
|
||||
"response.audio.delta",
|
||||
"response.audio.done",
|
||||
"response.output_audio.delta",
|
||||
"response.output_audio.done",
|
||||
"response.done",
|
||||
]
|
||||
input_item_ids: list[str] = []
|
||||
@@ -459,7 +474,7 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
for events in turns:
|
||||
event_types = [event["type"] for event in events]
|
||||
for event_type in required_sequence:
|
||||
if event_type != "response.audio.delta":
|
||||
if event_type != "response.output_audio.delta":
|
||||
assert event_types.count(event_type) == 1, event_types
|
||||
positions = [event_types.index(event_type) for event_type in required_sequence]
|
||||
assert positions == sorted(positions)
|
||||
@@ -486,7 +501,7 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
output_pcm = b"".join(
|
||||
base64.b64decode(event["delta"])
|
||||
for event in events
|
||||
if event["type"] == "response.audio.delta" and event.get("delta")
|
||||
if event["type"] == "response.output_audio.delta" and event.get("delta")
|
||||
)
|
||||
assert output_pcm
|
||||
assert created["id"] == done["id"]
|
||||
@@ -518,3 +533,394 @@ class TestQwen3OmniRealtimeWebSocket:
|
||||
|
||||
_assert_realtime_smoke(result)
|
||||
_assert_realtime_accuracy(result)
|
||||
|
||||
|
||||
# These regressions use the engine-owned Qwen duplex plugin. The older class
|
||||
# above intentionally exercises the separate turn/STT deployment.
|
||||
_DUPLEX_REPEATS = int(os.environ.get("VLLM_OMNI_TEST_QWEN_DUPLEX_REPEATS", "20"))
|
||||
_DUPLEX_LONG_INSTRUCTIONS = "Answer the user's question in English using at least eight complete sentences."
|
||||
_DUPLEX_SHORT_INSTRUCTIONS = "Answer the user's question in one short English sentence."
|
||||
|
||||
|
||||
class _DuplexTestRuntime(NamedTuple):
|
||||
server: OmniServer
|
||||
processor: Any
|
||||
artifacts: Path
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def qwen_duplex_server(tmp_path_factory, cached_silero_vad_artifact, run_level):
|
||||
"""Load real weights once for all interruption repetitions and history checks."""
|
||||
from transformers import AutoProcessor
|
||||
|
||||
assert run_level in {"advanced_model", "full_model"}, "Duplex playback acceptance requires real weights"
|
||||
artifacts = tmp_path_factory.mktemp("qwen_duplex_playback")
|
||||
deploy = artifacts / "deploy.yaml"
|
||||
# An absolute base keeps the production YAML's relative inheritance intact
|
||||
# when the test's Silero override lives in a temporary directory.
|
||||
deploy.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"base_config": get_deploy_config_path("qwen3_omni_duplex.yaml"),
|
||||
"duplex_session": {"server_vad_model_path": cached_silero_vad_artifact},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert load_deploy_config(deploy).session_mode == "duplex"
|
||||
model = get_model_prefix() + MODEL
|
||||
processor = AutoProcessor.from_pretrained(model, trust_remote_code=True)
|
||||
with OmniServer(
|
||||
model,
|
||||
["--deploy-config", str(deploy), "--init-timeout", "900", "--stage-init-timeout", "600"],
|
||||
env_dict={"VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR": str(artifacts)},
|
||||
) as server:
|
||||
yield _DuplexTestRuntime(server=server, processor=processor, artifacts=artifacts)
|
||||
|
||||
|
||||
def _duplex_fixture_pcm(path: Path, *, duration_s: float | None = None) -> bytes:
|
||||
"""Replay checked-in speech at the web client's negotiated 24 kHz PCM16 rate."""
|
||||
pcm = np.frombuffer(_pcm16_mono_16k_from_wav_bytes(path.read_bytes()), dtype="<i2")
|
||||
if duration_s is not None:
|
||||
pcm = pcm[: int(duration_s * 16000)]
|
||||
positions = np.arange(len(pcm) * 3 // 2) * (2.0 / 3.0)
|
||||
return np.interp(positions, np.arange(len(pcm)), pcm).astype("<i2").tobytes()
|
||||
|
||||
|
||||
def _duplex_response_id(event: dict) -> str | None:
|
||||
return event.get("response_id") or event.get("response", {}).get("id")
|
||||
|
||||
|
||||
def _duplex_response_events(events: list[dict], response_id: str, event_type: str) -> list[dict]:
|
||||
return [event for event in events if event["type"] == event_type and _duplex_response_id(event) == response_id]
|
||||
|
||||
|
||||
async def _duplex_receive_until(ws, events: list[dict], predicate, *, timeout_s: float = 180) -> None:
|
||||
# One deadline covers the whole operation, including an endless stream that
|
||||
# would otherwise reset a per-recv timeout forever.
|
||||
async def receive() -> None:
|
||||
while not predicate():
|
||||
raw = await ws.recv()
|
||||
assert isinstance(raw, str), "Expected JSON Realtime events"
|
||||
event = json.loads(raw)
|
||||
events.append(event)
|
||||
assert event.get("type") != "error", event
|
||||
|
||||
await asyncio.wait_for(receive(), timeout=timeout_s)
|
||||
|
||||
|
||||
async def _duplex_send_pcm(ws, pcm: bytes, *, paced: bool = False, trailing_silence: bool = True) -> None:
|
||||
if trailing_silence:
|
||||
pcm += bytes(24000 * 2) # One second exceeds the configured 500 ms endpoint.
|
||||
chunk_bytes = 24000 * 2 // 5
|
||||
for offset in range(0, len(pcm), chunk_bytes):
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(pcm[offset : offset + chunk_bytes]).decode(),
|
||||
}
|
||||
)
|
||||
)
|
||||
if paced:
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
|
||||
async def _duplex_add_image(ws) -> None:
|
||||
from PIL import Image
|
||||
|
||||
image = io.BytesIO()
|
||||
Image.new("RGB", (448, 336), (32, 96, 160)).save(image, format="JPEG")
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"id": "camera_1",
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/jpeg;base64," + base64.b64encode(image.getvalue()).decode(),
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _duplex_recorded_session(runtime, log_path: Path, *, instructions: str, vad: bool = True):
|
||||
events: list[dict] = []
|
||||
server = runtime.server
|
||||
async with websockets.connect(
|
||||
f"ws://{server.host}:{server.port}/v1/realtime?duplex=1", max_size=64 * 1024 * 1024
|
||||
) as ws:
|
||||
failed = True
|
||||
try:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"model": server.model,
|
||||
"instructions": instructions,
|
||||
"temperature": 0,
|
||||
"max_output_tokens": 384 if vad else 64,
|
||||
"audio": {
|
||||
"input": {
|
||||
"format": {"type": "audio/pcm", "rate": 24000},
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
"threshold": 0.5,
|
||||
"prefix_padding_ms": 300,
|
||||
"silence_duration_ms": 500,
|
||||
"create_response": True,
|
||||
"interrupt_response": True,
|
||||
}
|
||||
if vad
|
||||
else None,
|
||||
},
|
||||
"output": {"format": {"type": "audio/pcm", "rate": 24000}},
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "session.updated" for e in events))
|
||||
session = next(e["session"] for e in events if e["type"] == "session.updated")
|
||||
if vad:
|
||||
accepted = session["audio"]["input"]["turn_detection"]
|
||||
assert accepted["interrupt_response"] is True
|
||||
assert accepted["create_response"] is True
|
||||
await _duplex_add_image(ws)
|
||||
yield ws, events, session["id"]
|
||||
failed = False
|
||||
finally:
|
||||
# Closing the transport alone leaves a resumable session occupying
|
||||
# the deployment's single session slot. Always release the session.
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
await _duplex_receive_until(
|
||||
ws, events, lambda: any(e["type"] == "session.closed" for e in events), timeout_s=15
|
||||
)
|
||||
except Exception:
|
||||
if not failed:
|
||||
raise
|
||||
finally:
|
||||
log_path.write_text(json.dumps(events, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def _duplex_assert_prompt(
|
||||
runtime,
|
||||
session_id: str,
|
||||
*,
|
||||
audio_count: int,
|
||||
instructions: str,
|
||||
request_count: int,
|
||||
assistant_texts: list[str] | None = None,
|
||||
):
|
||||
captures = []
|
||||
for path in runtime.artifacts.glob("*/input.json"):
|
||||
capture = json.loads(path.read_text())
|
||||
if duplex_resource_request_belongs_to_session(capture["request_id"], session_id):
|
||||
captures.append((path.stat().st_mtime_ns, capture))
|
||||
assert len(captures) == request_count, captures
|
||||
capture = max(captures, key=lambda item: item[0])[1]
|
||||
assert capture["audio_count"] == audio_count
|
||||
# The camera belongs to the first user turn and leaves with its audio.
|
||||
image_retained = request_count <= 4
|
||||
assert len(capture["images"]) == int(image_retained)
|
||||
messages: list[dict[str, Any]] = [{"role": "system", "content": instructions}]
|
||||
if assistant_texts is None:
|
||||
assistant_texts = [""] * (audio_count - 1)
|
||||
assert len(assistant_texts) == audio_count - 1
|
||||
for index in range(audio_count):
|
||||
if index:
|
||||
messages.append({"role": "assistant", "content": assistant_texts[index - 1]})
|
||||
parts = [{"type": "image"}] if index == 0 and image_retained else []
|
||||
parts.append({"type": "audio"})
|
||||
messages.append({"role": "user", "content": parts})
|
||||
expected = runtime.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
assert capture["prompt"] == expected
|
||||
# Use the checkpoint tokenizer too: a repr(messages) processor stub cannot
|
||||
# establish that an empty assistant survives as a real turn separator.
|
||||
tokenizer = runtime.processor.tokenizer
|
||||
actual_ids = tokenizer.encode(capture["prompt"], add_special_tokens=False)
|
||||
separator_ids = tokenizer.encode("<|im_start|>assistant\n<|im_end|>\n", add_special_tokens=False)
|
||||
assert sum(
|
||||
actual_ids[index : index + len(separator_ids)] == separator_ids for index in range(len(actual_ids))
|
||||
) == assistant_texts.count("")
|
||||
|
||||
|
||||
def _duplex_assert_completed_audio(events: list[dict], response_id: str) -> None:
|
||||
terminal = _duplex_response_events(events, response_id, "response.done")
|
||||
assert len(terminal) == 1, terminal
|
||||
assert terminal[0]["response"]["status"] == "completed", terminal
|
||||
audio = _duplex_response_events(events, response_id, "response.output_audio.delta")
|
||||
assert sum(len(base64.b64decode(event["delta"])) for event in audio) > 4800
|
||||
assert any(
|
||||
event.get("delta")
|
||||
for event in events
|
||||
if _duplex_response_id(event) == response_id
|
||||
and event["type"] in {"response.output_text.delta", "response.output_audio_transcript.delta"}
|
||||
), "The follow-up must produce text as well as audio"
|
||||
|
||||
|
||||
async def _run_duplex_playback_interruption(runtime, log_path: Path, *, generation_finished: bool) -> None:
|
||||
async with _duplex_recorded_session(runtime, log_path, instructions=_DUPLEX_LONG_INSTRUCTIONS) as (
|
||||
ws,
|
||||
events,
|
||||
session_id,
|
||||
):
|
||||
await _duplex_send_pcm(ws, _duplex_fixture_pcm(validated_input_wav()))
|
||||
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "response.created" for e in events))
|
||||
old_id = next(_duplex_response_id(e) for e in events if e["type"] == "response.created")
|
||||
assert old_id
|
||||
await _duplex_receive_until(
|
||||
ws,
|
||||
events,
|
||||
lambda: (
|
||||
sum(
|
||||
len(base64.b64decode(e["delta"]))
|
||||
for e in _duplex_response_events(events, old_id, "response.output_audio.delta")
|
||||
)
|
||||
> 4800
|
||||
),
|
||||
)
|
||||
if generation_finished:
|
||||
await _duplex_receive_until(ws, events, lambda: _duplex_response_events(events, old_id, "response.done"))
|
||||
await ws.send(
|
||||
json.dumps({"type": "playback.ack", "response_id": old_id, "played_ms": 100, "committed_ms": 100})
|
||||
)
|
||||
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "playback.acknowledged" for e in events))
|
||||
ack = next(e["event"] for e in events if e["type"] == "playback.acknowledged")
|
||||
assert ack["item_id"] == f"item_{old_id}"
|
||||
assert ack["committed_ms"] == 100
|
||||
assert ack["playback"]["sent_ms"] > 100
|
||||
cursor = len(events)
|
||||
sender = asyncio.create_task(
|
||||
_duplex_send_pcm(ws, _duplex_fixture_pcm(validated_soft_interrupt_wav(), duration_s=4.7), paced=True)
|
||||
)
|
||||
try:
|
||||
await _duplex_receive_until(
|
||||
ws, events, lambda: any(e["type"] == "input_audio_buffer.speech_started" for e in events[cursor:])
|
||||
)
|
||||
speech_index = next(
|
||||
i for i in range(cursor, len(events)) if events[i]["type"] == "input_audio_buffer.speech_started"
|
||||
)
|
||||
if not generation_finished:
|
||||
assert not _duplex_response_events(events[:speech_index], old_id, "response.done"), (
|
||||
"Generation completed before speech interruption; the in-progress scenario was not exercised"
|
||||
)
|
||||
await _duplex_receive_until(
|
||||
ws,
|
||||
events,
|
||||
lambda: any(e["type"] == "response.done" and _duplex_response_id(e) != old_id for e in events[cursor:]),
|
||||
)
|
||||
await sender
|
||||
finally:
|
||||
sender.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await sender
|
||||
new_id = next(
|
||||
_duplex_response_id(e)
|
||||
for e in events[cursor:]
|
||||
if e["type"] == "response.created" and _duplex_response_id(e) != old_id
|
||||
)
|
||||
# Include everything observed through session.closed when checking duplicates.
|
||||
terminals = _duplex_response_events(events, old_id, "response.done")
|
||||
assert len(terminals) == 1, terminals
|
||||
assert terminals[0]["response"]["status"] == ("completed" if generation_finished else "cancelled")
|
||||
if generation_finished:
|
||||
cleared = _duplex_response_events(events, old_id, "output_audio_buffer.cleared")
|
||||
assert len(cleared) == 1, cleared
|
||||
boundary = events.index(cleared[0])
|
||||
else:
|
||||
boundary = events.index(terminals[0])
|
||||
assert not _duplex_response_events(events[boundary + 1 :], old_id, "response.output_audio.delta")
|
||||
assert new_id and new_id != old_id
|
||||
_duplex_assert_completed_audio(events, new_id)
|
||||
_duplex_assert_prompt(runtime, session_id, audio_count=2, instructions=_DUPLEX_LONG_INSTRUCTIONS, request_count=2)
|
||||
|
||||
|
||||
async def _run_duplex_history_pruning(runtime, log_path: Path) -> None:
|
||||
async with _duplex_recorded_session(runtime, log_path, instructions=_DUPLEX_SHORT_INSTRUCTIONS, vad=False) as (
|
||||
ws,
|
||||
events,
|
||||
session_id,
|
||||
):
|
||||
response_ids = []
|
||||
heard_texts: list[str] = []
|
||||
pcm = _duplex_fixture_pcm(validated_input_wav())
|
||||
for turn in range(6):
|
||||
cursor = len(events)
|
||||
await _duplex_send_pcm(ws, pcm, trailing_silence=False)
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
|
||||
await ws.send(json.dumps({"type": "response.create"}))
|
||||
await _duplex_receive_until(ws, events, lambda: any(e["type"] == "response.done" for e in events[cursor:]))
|
||||
response_id = next(_duplex_response_id(e) for e in events[cursor:] if e["type"] == "response.done")
|
||||
assert response_id and response_id not in response_ids
|
||||
response_ids.append(response_id)
|
||||
_duplex_assert_completed_audio(events, response_id)
|
||||
_duplex_assert_prompt(
|
||||
runtime,
|
||||
session_id,
|
||||
audio_count=min(turn + 1, 4),
|
||||
instructions=_DUPLEX_SHORT_INSTRUCTIONS,
|
||||
request_count=turn + 1,
|
||||
assistant_texts=heard_texts[-3:],
|
||||
)
|
||||
# Complete playback so pruning is checked against real assistant
|
||||
# answers paired with retained user audio. Repeated empty assistant
|
||||
# turns can make Qwen emit EOS; interruption tests cover those slots.
|
||||
done = _duplex_response_events(events, response_id, "response.done")[0]["response"]
|
||||
played_ms = done["metadata"]["playback"]["sent_ms"]
|
||||
heard_texts.append(
|
||||
"".join(
|
||||
e["delta"]
|
||||
for e in _duplex_response_events(events, response_id, "response.output_audio_transcript.delta")
|
||||
).strip()
|
||||
)
|
||||
assert heard_texts[-1]
|
||||
cursor = len(events)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "playback.ack",
|
||||
"response_id": response_id,
|
||||
"played_ms": played_ms,
|
||||
"committed_ms": played_ms,
|
||||
}
|
||||
)
|
||||
)
|
||||
await _duplex_receive_until(
|
||||
ws, events, lambda: any(e["type"] == "playback.acknowledged" for e in events[cursor:])
|
||||
)
|
||||
ack = next(e["event"] for e in events[cursor:] if e["type"] == "playback.acknowledged")
|
||||
assert ack["item_id"] == f"item_{response_id}"
|
||||
assert ack["committed_ms"] == ack["playback"]["sent_ms"] == played_ms
|
||||
assert ack["history_committed"] is True
|
||||
for response_id in response_ids:
|
||||
_duplex_assert_completed_audio(events, response_id)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
class TestQwen3OmniDuplexPlayback:
|
||||
@pytest.mark.parametrize("generation_finished", [False, True], ids=["generating", "queued-playback"])
|
||||
@pytest.mark.parametrize("repetition", range(_DUPLEX_REPEATS))
|
||||
def test_speech_interrupts_playback(
|
||||
self, qwen_duplex_server, tmp_path: Path, generation_finished: bool, repetition: int
|
||||
):
|
||||
"""Reuse one live deployment; every repetition opens an independent call."""
|
||||
asyncio.run(
|
||||
_run_duplex_playback_interruption(
|
||||
qwen_duplex_server, tmp_path / f"events-{repetition}.json", generation_finished=generation_finished
|
||||
)
|
||||
)
|
||||
|
||||
def test_mixed_image_audio_history_pruning(self, qwen_duplex_server, tmp_path: Path):
|
||||
asyncio.run(_run_duplex_history_pruning(qwen_duplex_server, tmp_path / "events.json"))
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from collections import OrderedDict
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||
_REF_AUDIO_RESOLVE_CACHE_MAX_BYTES,
|
||||
_REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES,
|
||||
OmniOpenAIServingSpeech,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache():
|
||||
server = object.__new__(OmniOpenAIServingSpeech)
|
||||
server._diffusion_mode = False
|
||||
server.model_config = None
|
||||
server._ref_audio_resolve_cache = OrderedDict()
|
||||
server._ref_audio_resolve_cache_bytes = 0
|
||||
server._ref_audio_resolve_cache_max_entries = _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES
|
||||
server._ref_audio_resolve_cache_max_bytes = _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES
|
||||
server._ref_audio_model_artifact_ready = set()
|
||||
server._request_ref_audio_artifact_keys = {}
|
||||
return server
|
||||
|
||||
|
||||
def _put(cache, key, samples, artifact=None):
|
||||
cache._put_resolved_ref_audio(key, samples, 24000, artifact or key)
|
||||
|
||||
|
||||
def test_compact_storage_and_default_capacity(cache):
|
||||
assert cache._ref_audio_resolve_cache_max_entries == 1024
|
||||
assert cache._ref_audio_resolve_cache_max_bytes == 512 * 1024 * 1024
|
||||
_put(cache, "a", [0.0, 0.25, -0.5, 1.0])
|
||||
value = cache._ref_audio_resolve_cache["a"]
|
||||
assert value[0].dtype == np.float32
|
||||
assert value[2] == cache._ref_audio_resolve_cache_bytes == 16
|
||||
|
||||
|
||||
def test_cached_float32_roundtrip_and_mutation_isolation(cache):
|
||||
source = "data:audio/wav;base64,already-cached"
|
||||
key = hashlib.sha1(source.encode()).hexdigest()
|
||||
values = np.random.default_rng(42).standard_normal(48000).astype(np.float32).tolist()
|
||||
expected = values.copy()
|
||||
_put(cache, key, values)
|
||||
values[0] = 99.0
|
||||
first, sample_rate, _ = asyncio.run(cache._resolve_ref_audio(source))
|
||||
assert first == expected
|
||||
first[0] = 100.0
|
||||
second, _, _ = asyncio.run(cache._resolve_ref_audio(source))
|
||||
assert sample_rate == 24000
|
||||
assert second == expected
|
||||
|
||||
|
||||
def test_entry_eviction_invalidates_artifact_readiness(cache):
|
||||
cache._ref_audio_resolve_cache_max_entries = 2
|
||||
_put(cache, "a", [0.0])
|
||||
cache._ref_audio_model_artifact_ready.add(("a", False))
|
||||
_put(cache, "b", [0.0])
|
||||
_put(cache, "c", [0.0])
|
||||
assert list(cache._ref_audio_resolve_cache) == ["b", "c"]
|
||||
assert ("a", False) not in cache._ref_audio_model_artifact_ready
|
||||
|
||||
|
||||
def test_byte_budget_evicts_and_rejects_oversized_entries(cache):
|
||||
cache._ref_audio_resolve_cache_max_bytes = 16
|
||||
_put(cache, "a", [0.0] * 3)
|
||||
_put(cache, "b", [0.0] * 3)
|
||||
assert list(cache._ref_audio_resolve_cache) == ["b"]
|
||||
assert cache._ref_audio_resolve_cache_bytes == 12
|
||||
_put(cache, "too-large", [0.0] * 5)
|
||||
assert "too-large" not in cache._ref_audio_resolve_cache
|
||||
|
||||
|
||||
def test_replacement_accounts_bytes_and_preserves_referenced_artifact(cache):
|
||||
_put(cache, "a", [0.0] * 3, "same")
|
||||
_put(cache, "alias", [0.0] * 2, "same")
|
||||
cache._ref_audio_model_artifact_ready.add(("same", False))
|
||||
_put(cache, "a", [1.0] * 4, "new")
|
||||
assert cache._ref_audio_resolve_cache_bytes == 24
|
||||
assert ("same", False) in cache._ref_audio_model_artifact_ready
|
||||
|
||||
|
||||
def test_pool_accepts_666_references(cache):
|
||||
for index in range(666):
|
||||
_put(cache, str(index), [0.0] * 8)
|
||||
assert len(cache._ref_audio_resolve_cache) == 666
|
||||
assert cache._ref_audio_resolve_cache_bytes == 666 * 8 * 4
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path))
|
||||
return OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stereo", [False, True])
|
||||
def test_finalized_waveform_owns_contiguous_numeric_storage(server, stereo):
|
||||
import gc
|
||||
|
||||
source = np.linspace(-1, 1, 96000, dtype=np.float64).reshape(48000, 2)
|
||||
decoded = source if stereo else source[:, 0]
|
||||
expected = np.asarray(decoded, dtype=np.float32)
|
||||
if stereo:
|
||||
expected = expected.mean(axis=-1)
|
||||
waveform, sr, artifact, duration = server._finalize_fetched_ref_audio(decoded, 24000)
|
||||
assert waveform.dtype == np.float32
|
||||
assert waveform.flags.c_contiguous and waveform.flags.owndata
|
||||
assert not np.shares_memory(waveform, decoded)
|
||||
assert not gc.is_tracked(waveform)
|
||||
np.testing.assert_array_equal(waveform, expected)
|
||||
assert artifact == server._make_ref_audio_artifact_cache_key(expected, sr)
|
||||
assert duration == 2.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_array_resolver_reuses_buffer_and_legacy_lists_are_temporary(server, mocker):
|
||||
server._init_speaker_storage()
|
||||
server._diffusion_mode = True
|
||||
server._allowed_local_media_path = ""
|
||||
server._ref_audio_cache_key = mocker.AsyncMock(return_value="locator")
|
||||
server._media_connector = mocker.Mock()
|
||||
server._media_connector.fetch_audio_async = mocker.AsyncMock(
|
||||
return_value=(np.linspace(-1, 1, 24000, dtype=np.float32), 24000)
|
||||
)
|
||||
first, sr, key = await server._resolve_ref_audio_array("reference")
|
||||
second, _, _ = await server._resolve_ref_audio_array("reference")
|
||||
legacy, _, _ = await server._resolve_ref_audio("reference")
|
||||
assert first is second
|
||||
assert isinstance(legacy, list)
|
||||
np.testing.assert_array_equal(first, legacy)
|
||||
legacy[0] = 100
|
||||
assert first[0] == -1
|
||||
assert server._ref_audio_resolve_cache[key][0] is first
|
||||
assert server._ref_audio_resolve_cache_bytes == first.nbytes == 96000
|
||||
server._media_connector.fetch_audio_async.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", [None, "local", "delay", "realtime"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_moss_array_resolution_keeps_nano_list_transport(mocker, variant):
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.moss_tts import _MossTTSAdapterBase
|
||||
|
||||
server = mocker.Mock()
|
||||
server._resolve_ref_audio = mocker.AsyncMock(return_value=([0.0], 24000, "key"))
|
||||
waveform = np.zeros(1, dtype=np.float32)
|
||||
server._resolve_ref_audio_array = mocker.AsyncMock(return_value=(waveform, 24000, "key"))
|
||||
adapter = _MossTTSAdapterBase.__new__(_MossTTSAdapterBase)
|
||||
adapter.ctx = SpeechServingContext(server=server)
|
||||
adapter._moss_variant = variant
|
||||
result, _, _ = await adapter._resolve_ref_audio("reference")
|
||||
if variant is None:
|
||||
assert isinstance(result, list)
|
||||
server._resolve_ref_audio_array.assert_not_awaited()
|
||||
else:
|
||||
assert result is waveform
|
||||
server._resolve_ref_audio.assert_not_awaited()
|
||||
|
||||
|
||||
def test_waveform_cache_eviction_releases_owned_buffer(server):
|
||||
import weakref
|
||||
|
||||
server._init_speaker_storage()
|
||||
server._ref_audio_resolve_cache_max_entries = 1
|
||||
waveform = np.zeros(24000, dtype=np.float32)
|
||||
ref = weakref.ref(waveform)
|
||||
server._put_resolved_ref_audio("first", waveform, 24000, "artifact-first")
|
||||
del waveform
|
||||
assert ref() is not None
|
||||
server._put_resolved_ref_audio("second", np.zeros(24000, dtype=np.float32), 24000, "artifact-second")
|
||||
assert ref() is None
|
||||
assert server._ref_audio_resolve_cache_bytes == 96000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entries, budget", [(1, 1000), (10, 8)])
|
||||
def test_numeric_waveform_cache_limits(server, entries, budget):
|
||||
server._init_speaker_storage()
|
||||
server._ref_audio_resolve_cache_max_entries = entries
|
||||
server._ref_audio_resolve_cache_max_bytes = budget
|
||||
server._put_resolved_ref_audio("a", np.zeros(2, dtype=np.float32), 24000, "artifact-a")
|
||||
server._put_resolved_ref_audio("b", np.ones(2, dtype=np.float32), 24000, "artifact-b")
|
||||
assert list(server._ref_audio_resolve_cache) == ["b"]
|
||||
assert server._ref_audio_resolve_cache_bytes == 8
|
||||
@@ -1,429 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
import threading
|
||||
import wave
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
ServerVADConfig,
|
||||
ServerVADFrame,
|
||||
ServerVADPipeline,
|
||||
SileroVADBackend,
|
||||
SileroVADBackendProvider,
|
||||
SpeechEndpointDecision,
|
||||
ThresholdEndpointPolicy,
|
||||
parse_session_turn_detection,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class SequenceDetector:
|
||||
frame_samples = 160
|
||||
|
||||
def __init__(self, probabilities: list[float]) -> None:
|
||||
self.probabilities = probabilities
|
||||
|
||||
def new_state(self) -> int:
|
||||
return 0
|
||||
|
||||
def infer(self, frame: np.ndarray, state: object) -> tuple[float, object]:
|
||||
index = int(state)
|
||||
probability = self.probabilities[index] if index < len(self.probabilities) else 0.0
|
||||
return probability, index + 1
|
||||
|
||||
|
||||
def test_server_vad_config_defaults_and_validation():
|
||||
config = ServerVADConfig.from_value({"type": "server_vad"})
|
||||
|
||||
assert config.type == "server_vad"
|
||||
assert config.threshold == 0.5
|
||||
assert config.prefix_padding_ms == 300
|
||||
assert config.silence_duration_ms == 500
|
||||
assert config.create_response is True
|
||||
assert config.interrupt_response is None
|
||||
assert "interrupt_response" not in config.as_dict()
|
||||
assert config.min_speech_duration_ms is None
|
||||
assert config.as_dict()["type"] == config.type
|
||||
|
||||
interrupting = ServerVADConfig.from_value(
|
||||
{
|
||||
"type": "server_vad",
|
||||
"interrupt_response": True,
|
||||
"min_speech_duration_ms": 96,
|
||||
}
|
||||
)
|
||||
assert interrupting.interrupt_response is True
|
||||
assert interrupting.min_speech_duration_ms == 96
|
||||
for value in (None, 0, "true"):
|
||||
with pytest.raises(ValueError, match="interrupt_response must be a boolean"):
|
||||
ServerVADConfig.from_value({"type": "server_vad", "interrupt_response": value})
|
||||
for enabled in (False, True):
|
||||
_, aliased = parse_session_turn_detection(
|
||||
{
|
||||
"turn_detection": {"type": "server_vad"},
|
||||
"audio": {"input": {"turn_detection": {"type": "server_vad", "interrupt_response": enabled}}},
|
||||
}
|
||||
)
|
||||
assert aliased is not None and aliased.interrupt_response is enabled
|
||||
with pytest.raises(ValueError, match="Unknown"):
|
||||
ServerVADConfig.from_value({"type": "server_vad", "semantic_eagerness": "high"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prefix_padding_ms", [0, 10, 30])
|
||||
async def test_server_vad_pipeline_handles_arbitrary_chunk_boundaries(prefix_padding_ms: int):
|
||||
detector = SequenceDetector([0.0, 0.9, 0.8, 0.0, 0.0])
|
||||
pipeline = ServerVADPipeline(
|
||||
detector,
|
||||
ServerVADConfig(prefix_padding_ms=prefix_padding_ms, silence_duration_ms=20),
|
||||
)
|
||||
samples = np.zeros(detector.frame_samples * 5, dtype=np.float32)
|
||||
|
||||
batches = [
|
||||
await pipeline.push(samples[:73]),
|
||||
await pipeline.push(samples[73:431]),
|
||||
await pipeline.push(samples[431:]),
|
||||
]
|
||||
frames = [frame for batch in batches for frame in batch.frames]
|
||||
|
||||
assert len(frames) == 5
|
||||
assert [index for index, frame in enumerate(frames) if frame.decision.speech_started] == [1]
|
||||
assert [index for index, frame in enumerate(frames) if frame.decision.speech_stopped] == [4]
|
||||
assert frames[1].decision.audio_start_ms == max(0, 10 - prefix_padding_ms)
|
||||
assert frames[4].decision.audio_end_ms == 50
|
||||
|
||||
pipeline.reset()
|
||||
reset_batch = await pipeline.push(samples)
|
||||
assert [index for index, frame in enumerate(reset_batch.frames) if frame.decision.speech_started] == [1]
|
||||
assert [index for index, frame in enumerate(reset_batch.frames) if frame.decision.speech_stopped] == [4]
|
||||
# Keep session-relative timestamps, but never reference prefix audio
|
||||
# discarded by reset at 50 ms.
|
||||
assert reset_batch.frames[1].decision.audio_start_ms == max(50, 60 - prefix_padding_ms)
|
||||
assert reset_batch.frames[4].decision.audio_end_ms == 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_retained_audio_does_not_keep_input_chunk():
|
||||
pipeline = ServerVADPipeline(SequenceDetector([]), ServerVADConfig())
|
||||
session = DuplexSession("server-vad-memory", DuplexSessionConfig())
|
||||
samples = np.zeros(160 * 100 + 1, dtype=np.float32)
|
||||
source_ref = weakref.ref(samples)
|
||||
batch = await pipeline.push(samples)
|
||||
session.append_server_vad_frame(
|
||||
batch.frames[-1].samples,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=80,
|
||||
)
|
||||
session.append_server_vad_frame(
|
||||
batch.frames[-1].samples,
|
||||
speech_started=True,
|
||||
speech_stopped=False,
|
||||
prefix_samples=80,
|
||||
)
|
||||
del batch, samples
|
||||
# Allow the completed to_thread callback to release its result, too.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert source_ref() is None
|
||||
assert pipeline.scratch_bytes == 4
|
||||
assert session.server_vad_utterance_bytes == (80 + 160) * 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_pipeline_pcm16_resampling_is_split_invariant():
|
||||
sample_index = np.arange(2_400, dtype=np.int32)
|
||||
source = ((sample_index * 7919) % 65_536 - 32_768).astype("<i2")
|
||||
# Supply ample silence after the signal so the streaming resampler can
|
||||
# emit a complete set of VAD frames without depending on its FIR width.
|
||||
right_context = np.zeros(240, dtype="<i2")
|
||||
continuous_source = np.concatenate((source, right_context))
|
||||
|
||||
async def process(chunk_sizes: list[int]) -> tuple[np.ndarray, list[SpeechEndpointDecision]]:
|
||||
pipeline = ServerVADPipeline(
|
||||
SequenceDetector([0.0] * 10),
|
||||
ServerVADConfig(),
|
||||
)
|
||||
frames: list[ServerVADFrame] = []
|
||||
offset = 0
|
||||
for chunk_size in chunk_sizes:
|
||||
chunk = continuous_source[offset : offset + chunk_size]
|
||||
batch = await pipeline.push_pcm16(chunk.tobytes(), source_sample_rate_hz=24_000)
|
||||
frames.extend(batch.frames)
|
||||
offset += chunk_size
|
||||
assert offset == continuous_source.size
|
||||
return np.concatenate([frame.samples for frame in frames]), [frame.decision for frame in frames]
|
||||
|
||||
whole_samples, whole_decisions = await process([continuous_source.size])
|
||||
split_samples, split_decisions = await process([7, 13, 511, 512, 513, continuous_source.size - 1_556])
|
||||
|
||||
np.testing.assert_array_equal(split_samples, whole_samples)
|
||||
assert whole_samples.size == 1_600
|
||||
assert split_decisions == whole_decisions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_pipeline_reset_clears_resampler_and_source_rate_lock():
|
||||
pipeline = ServerVADPipeline(
|
||||
SequenceDetector([0.0]),
|
||||
ServerVADConfig(),
|
||||
)
|
||||
|
||||
batch = await pipeline.push_pcm16(
|
||||
np.asarray([30_000, -30_000], dtype="<i2").tobytes(),
|
||||
source_sample_rate_hz=24_000,
|
||||
)
|
||||
assert not batch.frames
|
||||
|
||||
pipeline.reset()
|
||||
|
||||
source = np.zeros(160, dtype="<i2")
|
||||
source[:3] = [-32768, 0, 32767]
|
||||
batch = await pipeline.push_pcm16(source.tobytes(), source_sample_rate_hz=16_000)
|
||||
|
||||
assert len(batch.frames) == 1
|
||||
np.testing.assert_array_equal(
|
||||
batch.frames[0].samples[:3],
|
||||
np.asarray([-1.0, 0.0, 32767 / 32768], dtype=np.float32),
|
||||
)
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_uses_silero_v62_exit_hysteresis():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(threshold=0.5, silence_duration_ms=20),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
started = policy.update(0.8, frame_start_sample=0, frame_samples=frame_samples)
|
||||
assert started.speech_started is True
|
||||
|
||||
# Values below the activation threshold but above threshold - 0.15 must
|
||||
# not begin a silence candidate or close an active speech segment.
|
||||
for frame_index in range(1, 5):
|
||||
decision = policy.update(
|
||||
0.4,
|
||||
frame_start_sample=frame_index * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
assert decision.speech_stopped is False
|
||||
|
||||
first_silence = policy.update(
|
||||
0.2,
|
||||
frame_start_sample=5 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
stopped = policy.update(
|
||||
0.2,
|
||||
frame_start_sample=6 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
|
||||
assert first_silence.speech_stopped is False
|
||||
assert stopped.speech_stopped is True
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_honors_minimum_speech_duration():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(prefix_padding_ms=0, min_speech_duration_ms=25),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
assert not policy.update(0.9, frame_start_sample=0, frame_samples=frame_samples).speech_started
|
||||
assert not policy.update(0.9, frame_start_sample=160, frame_samples=frame_samples).speech_started
|
||||
started = policy.update(0.9, frame_start_sample=320, frame_samples=frame_samples)
|
||||
|
||||
assert started.speech_started is True
|
||||
assert started.audio_start_ms == 0
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_clamps_silero_exit_threshold():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(threshold=0.1, silence_duration_ms=20),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
started = policy.update(0.9, frame_start_sample=0, frame_samples=frame_samples)
|
||||
assert started.speech_started is True
|
||||
|
||||
first_silence = policy.update(
|
||||
0.0,
|
||||
frame_start_sample=frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
stopped = policy.update(
|
||||
0.0,
|
||||
frame_start_sample=2 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
|
||||
assert first_silence.speech_stopped is False
|
||||
assert stopped.speech_stopped is True
|
||||
# The committed audio ends at 30 ms: 10 ms of speech followed by the
|
||||
# 20 ms of silence required to detect the endpoint.
|
||||
assert stopped.audio_end_ms == 30
|
||||
assert stopped.endpoint_delay_ms == 20
|
||||
|
||||
|
||||
def test_silero_backend_matches_upstream_v62_streaming_contract(monkeypatch, tmp_path):
|
||||
class FakeSessionOptions:
|
||||
inter_op_num_threads = 0
|
||||
intra_op_num_threads = 0
|
||||
|
||||
class FakeInferenceSession:
|
||||
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
|
||||
self.providers = providers
|
||||
self.sess_options = sess_options
|
||||
self.calls: list[dict[str, np.ndarray]] = []
|
||||
self.run_barrier: threading.Barrier | None = None
|
||||
sessions.append(self)
|
||||
|
||||
def get_inputs(self) -> list[SimpleNamespace]:
|
||||
return [SimpleNamespace(name=name) for name in ("input", "state", "sr")]
|
||||
|
||||
def run(self, _outputs: object, inputs: dict[str, np.ndarray]) -> list[np.ndarray]:
|
||||
self.calls.append({name: np.array(value, copy=True) for name, value in inputs.items()})
|
||||
if self.run_barrier is not None:
|
||||
self.run_barrier.wait(timeout=5)
|
||||
return [
|
||||
np.asarray([[0.75]], dtype=np.float32),
|
||||
np.full((2, 1, 128), len(self.calls), dtype=np.float32),
|
||||
]
|
||||
|
||||
sessions: list[FakeInferenceSession] = []
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"onnxruntime",
|
||||
SimpleNamespace(
|
||||
SessionOptions=FakeSessionOptions,
|
||||
InferenceSession=FakeInferenceSession,
|
||||
),
|
||||
)
|
||||
|
||||
model_path = tmp_path / "silero_vad.onnx"
|
||||
model_path.write_bytes(b"fake-model")
|
||||
backend = SileroVADBackend(model_path)
|
||||
session = sessions[0]
|
||||
|
||||
assert session.providers == ["CPUExecutionProvider"]
|
||||
assert session.sess_options.inter_op_num_threads == 1
|
||||
assert session.sess_options.intra_op_num_threads == 1
|
||||
assert session.calls[0]["input"].shape == (1, 576)
|
||||
assert session.calls[0]["state"].shape == (2, 1, 128)
|
||||
np.testing.assert_array_equal(session.calls[0]["state"], np.zeros((2, 1, 128), dtype=np.float32))
|
||||
assert session.calls[0]["sr"].shape == ()
|
||||
assert session.calls[0]["sr"].item() == 16_000
|
||||
|
||||
state = backend.new_state()
|
||||
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
|
||||
probability, state = backend.infer(first_frame, state)
|
||||
first_call = session.calls[1]
|
||||
|
||||
assert probability == pytest.approx(0.75)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, : backend.context_samples],
|
||||
np.zeros((1, backend.context_samples), dtype=np.float32),
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, backend.context_samples :],
|
||||
first_frame[None, :],
|
||||
)
|
||||
np.testing.assert_array_equal(first_call["state"], np.zeros((2, 1, 128), dtype=np.float32))
|
||||
|
||||
second_frame = -first_frame
|
||||
backend.infer(second_frame, state)
|
||||
second_call = session.calls[2]
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, : backend.context_samples],
|
||||
first_frame[None, -backend.context_samples :],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, backend.context_samples :],
|
||||
second_frame[None, :],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
second_call["state"],
|
||||
np.full((2, 1, 128), 2, dtype=np.float32),
|
||||
)
|
||||
|
||||
# Independent pipeline states may enter the shared ORT session concurrently.
|
||||
session.run_barrier = threading.Barrier(2)
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [
|
||||
executor.submit(backend.infer, first_frame, backend.new_state()),
|
||||
executor.submit(backend.infer, second_frame, backend.new_state()),
|
||||
]
|
||||
assert [future.result()[0] for future in futures] == pytest.approx([0.75, 0.75])
|
||||
|
||||
|
||||
def test_duplex_session_owns_server_vad_prefix_and_utterance_audio():
|
||||
session = DuplexSession("server-vad-session", DuplexSessionConfig())
|
||||
frame = np.zeros(160, dtype=np.float32)
|
||||
|
||||
session.reserve_input_bytes(frame.nbytes * 3, limit=frame.nbytes * 4)
|
||||
released = session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
assert released == 0
|
||||
released = session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
assert released == frame.nbytes
|
||||
session.release_input_bytes(released)
|
||||
session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=True,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
|
||||
assert session.server_vad_utterance_bytes == frame.nbytes * 2
|
||||
assert session.stage_server_vad_audio_for_commit() is True
|
||||
committed = session.commit_user_input()
|
||||
assert committed is not None
|
||||
assert committed.message["role"] == "user"
|
||||
audio_url = committed.message["content"][0]["audio_url"]["url"]
|
||||
assert audio_url.startswith("data:audio/wav;base64,")
|
||||
wav_bytes = base64.b64decode(audio_url.partition(",")[2])
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file:
|
||||
assert wav_file.getnchannels() == 1
|
||||
assert wav_file.getsampwidth() == 2
|
||||
assert wav_file.getframerate() == 16_000
|
||||
assert wav_file.getnframes() == frame.size * 2
|
||||
|
||||
|
||||
def test_silero_provider_rejects_missing_or_invalid_local_artifact(tmp_path):
|
||||
missing = SileroVADBackendProvider(model_path=str(tmp_path / "missing.onnx"))
|
||||
with pytest.raises(RuntimeError, match="does not exist"):
|
||||
missing.get()
|
||||
|
||||
invalid_path = tmp_path / "silero_vad.onnx"
|
||||
invalid_path.write_bytes(b"not-the-pinned-model")
|
||||
invalid = SileroVADBackendProvider(model_path=str(invalid_path))
|
||||
with pytest.raises(RuntimeError, match="checksum mismatch"):
|
||||
invalid.get()
|
||||
@@ -27,7 +27,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@dataclass
|
||||
class _EngineClientStub:
|
||||
stage_configs: list[dict[str, object]]
|
||||
stage_configs: list[object]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -149,6 +149,37 @@ async def test_chat_reference_audio_reaches_code2wav_transport(
|
||||
assert code2wav_payload.meta.ref_audio_sr == 16000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_typed_stage_enables_reference_audio(
|
||||
serving_chat: OmniOpenAIServingChat,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
reference_audio_url = "data:audio/wav;base64,AAAA"
|
||||
reference_waveform = np.array([0.25, -0.5], dtype=np.float32)
|
||||
serving_chat.engine_client = _EngineClientStub(
|
||||
[
|
||||
SimpleNamespace(
|
||||
model_config=SimpleNamespace(model_arch="MiniCPMO45OmniForConditionalGeneration"),
|
||||
model_stage="llm",
|
||||
)
|
||||
]
|
||||
)
|
||||
serving_chat.model_config = _ModelConfigStub()
|
||||
engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
|
||||
media_connector = mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
|
||||
).return_value
|
||||
media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000))
|
||||
|
||||
await serving_chat._attach_minicpmo45_reference_audio(
|
||||
engine_prompt,
|
||||
_chat_request(reference_audio_url),
|
||||
)
|
||||
|
||||
media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url)
|
||||
assert MINICPMO45_REFERENCE_AUDIO_KEY in cast(dict[str, object], engine_prompt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("request_shape", ["extra_body", "model_extra"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_reference_audio_accepts_request_extra_mappings(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for MiniCPM-o 4.5 audio-streaming output-kind coercion.
|
||||
|
||||
Chat ``stream=true`` coerces every AR stage to DELTA. For MiniCPM-o 4.5 that
|
||||
@@ -33,6 +34,13 @@ def _stage(model_arch: str, model_stage: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(engine_args=SimpleNamespace(model_arch=model_arch, model_stage=model_stage))
|
||||
|
||||
|
||||
def _typed_stage(model_arch: str, model_stage: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(model_arch=model_arch),
|
||||
model_stage=model_stage,
|
||||
)
|
||||
|
||||
|
||||
def _minicpmo45_stages():
|
||||
return [_stage(MINICPMO45_ARCH, "llm"), _stage(MINICPMO45_ARCH, "tts")]
|
||||
|
||||
@@ -50,6 +58,20 @@ def test_thinker_forced_final_only_talker_stays_delta(serving_chat):
|
||||
assert out[1].output_kind is RequestOutputKind.DELTA
|
||||
|
||||
|
||||
def test_typed_stages_force_thinker_final_only(serving_chat):
|
||||
serving_chat.engine_client = SimpleNamespace(
|
||||
stage_configs=[
|
||||
_typed_stage(MINICPMO45_ARCH, "llm"),
|
||||
_typed_stage(MINICPMO45_ARCH, "tts"),
|
||||
]
|
||||
)
|
||||
|
||||
out = serving_chat._fix_minicpmo45_audio_stream_output_kinds(_delta_params(), ["text", "audio"])
|
||||
|
||||
assert out[0].output_kind is RequestOutputKind.FINAL_ONLY
|
||||
assert out[1].output_kind is RequestOutputKind.DELTA
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modalities", [None, [], ["text"]])
|
||||
def test_non_audio_request_is_untouched(serving_chat, modalities):
|
||||
serving_chat.engine_client = SimpleNamespace(stage_configs=_minicpmo45_stages())
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for multistage generation input construction."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -94,6 +95,34 @@ def test_build_multistage_generation_inputs_applies_stage_specific_overrides(ser
|
||||
assert engine.default_sampling_params_list[2].lora_request is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_body", [{}, {"bot_task": "think"}])
|
||||
def test_build_multistage_generation_inputs_leaves_mm_uuids_to_content_hash(serving_chat, extra_body):
|
||||
"""Fixed image UUIDs collide across requests with the same processor kwargs.
|
||||
|
||||
Leave UUID generation to vLLM so cache keys depend on image content.
|
||||
"""
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
engine = SimpleNamespace(
|
||||
stage_configs=[
|
||||
SimpleNamespace(stage_type="llm", is_comprehension=True),
|
||||
SimpleNamespace(stage_type="diffusion", is_comprehension=False),
|
||||
],
|
||||
default_sampling_params_list=[SamplingParams(temperature=0.0), OmniDiffusionSamplingParams()],
|
||||
)
|
||||
for images in ([Image.new("RGB", (32, 32), color="red")], [Image.new("RGB", (32, 32), c) for c in ("red", "blue")]):
|
||||
engine_prompt, _ = OmniOpenAIServingChat._build_multistage_generation_inputs(
|
||||
serving_chat,
|
||||
engine=engine,
|
||||
prompt="edit me",
|
||||
extra_body=dict(extra_body),
|
||||
reference_images=images,
|
||||
gen_params=OmniDiffusionSamplingParams(seed=0),
|
||||
)
|
||||
assert engine_prompt["multi_modal_data"]
|
||||
assert "multi_modal_uuids" not in engine_prompt
|
||||
|
||||
|
||||
def test_prepare_multistage_multimodal_inputs_defers_downstream_modalities(serving_chat):
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
@@ -538,3 +567,84 @@ def test_build_multistage_generation_inputs_no_stop_token_ids_without_size(servi
|
||||
assert sampling_params_list[0].stop_token_ids == [], (
|
||||
"Without bot_task, AR stage stop_token_ids must be the default empty list"
|
||||
)
|
||||
|
||||
|
||||
def test_build_multistage_generation_inputs_omitted_bot_task_matches_prompt_default(serving_chat):
|
||||
"""Regression: an omitted bot_task must resolve identically for the AR
|
||||
prompt and for its stop_token_ids.
|
||||
|
||||
``build_prompt_tokens``/``build_prompt`` default an omitted bot_task
|
||||
per-task via the ``_DEFAULT_BOT_TASK`` sentinel (``"think"`` for the base
|
||||
``it2i`` task, which isn't itself a key in ``_TASK_PRESETS``).
|
||||
``resolve_stop_token_ids`` must normalize from that same omitted-or-not
|
||||
starting point. Passing the raw -- still ``None`` -- outer ``bot_task``
|
||||
variable instead of mirroring ``build_kwargs``'s own omitted-or-not
|
||||
``"bot_task"`` entry made it normalize ``bot_task=None`` (plain mode), so
|
||||
it fell through to the full ``<img_ratio_*>`` range instead of the
|
||||
think/recaption terminator pair.
|
||||
|
||||
An explicit (non-"auto") size is required to observe this: with
|
||||
``need_ratio=True`` the ratio range is returned regardless of bot_task, so
|
||||
the two paths only visibly disagree once a concrete size selects the
|
||||
narrower stop set.
|
||||
"""
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
||||
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
class FakeTokenizer:
|
||||
SPECIAL = {
|
||||
"<|startoftext|>": 1,
|
||||
"<img>": 2,
|
||||
"<recaption>": 4,
|
||||
}
|
||||
|
||||
def convert_tokens_to_ids(self, tok: str) -> int:
|
||||
return self.SPECIAL.get(tok, 0)
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
return list(range(100, 100 + len(text)))
|
||||
|
||||
def _build(extra_body):
|
||||
engine = SimpleNamespace(
|
||||
stage_configs=[
|
||||
SimpleNamespace(stage_type="llm", is_comprehension=True),
|
||||
SimpleNamespace(stage_type="diffusion", is_comprehension=False),
|
||||
],
|
||||
default_sampling_params_list=[
|
||||
SamplingParams(temperature=0.0),
|
||||
OmniDiffusionSamplingParams(),
|
||||
],
|
||||
)
|
||||
_, sampling_params_list = OmniOpenAIServingChat._build_multistage_generation_inputs(
|
||||
serving_chat,
|
||||
engine=engine,
|
||||
prompt="draw a cat",
|
||||
extra_body=extra_body,
|
||||
reference_images=[Image.new("RGB", (32, 32), color="red")],
|
||||
gen_params=OmniDiffusionSamplingParams(height=768, width=1024),
|
||||
tokenizer=FakeTokenizer(),
|
||||
)
|
||||
return sampling_params_list[0].stop_token_ids
|
||||
|
||||
# bot_task omitted entirely; use_system_prompt is what carries the request
|
||||
# into the AR-prompt path.
|
||||
omitted = _build({"use_system_prompt": "en_recaption"})
|
||||
# Same request with the default spelled out explicitly.
|
||||
explicit = _build({"bot_task": "think", "use_system_prompt": "en_recaption"})
|
||||
|
||||
expected = [
|
||||
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["</think>"],
|
||||
HUNYUAN_IMAGE3_SPECIAL_TOKEN_IDS["</recaption>"],
|
||||
]
|
||||
assert omitted == expected, (
|
||||
f"omitted bot_task with an explicit size must resolve stop_token_ids for the "
|
||||
f"default 'think' bot_task ({expected}); got {omitted}. A long list here is the "
|
||||
f"full <img_ratio_*> range, meaning resolve_stop_token_ids normalized bot_task=None "
|
||||
f"(plain mode) instead of agreeing with build_prompt_tokens's default."
|
||||
)
|
||||
assert omitted == explicit, (
|
||||
"omitting bot_task must resolve the same stop_token_ids as passing the default "
|
||||
f"explicitly; omitted={omitted} explicit={explicit}"
|
||||
)
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""
|
||||
Unit tests for OmniOpenAIServingChat sampling params handling.
|
||||
|
||||
@@ -8,6 +9,7 @@ are correctly applied to the comprehension stage while preserving YAML defaults.
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -326,6 +328,64 @@ def test_mixed_consumer_keeps_root_common_args_with_nested_extras(mock_engine_cl
|
||||
assert captured["prompt"]["negative_prompt"] == "avoid blur"
|
||||
|
||||
|
||||
def test_text_only_request_reaches_engine_with_comprehension_task_mode(mock_engine_client, mocker: MockerFixture):
|
||||
"""Exercise the production chat path, not only the tagging helper."""
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(stage_type="llm", is_comprehension=True),
|
||||
SimpleNamespace(stage_type="diffusion", is_comprehension=False),
|
||||
]
|
||||
mock_engine_client.default_sampling_params_list = [
|
||||
SamplingParams(),
|
||||
OmniDiffusionSamplingParams(),
|
||||
]
|
||||
mock_engine_client.output_modalities = ["text", "image"]
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.renderer = SimpleNamespace(get_tokenizer=lambda: object())
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def results():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
def generate(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return results()
|
||||
|
||||
mock_engine_client.generate = generate
|
||||
serving_chat = build_serving_chat(
|
||||
engine_client=mock_engine_client,
|
||||
models=SimpleNamespace(model_name=lambda _: "test"),
|
||||
online_renderer=SimpleNamespace(validate_chat_template=lambda **_: None),
|
||||
trust_request_chat_template=True,
|
||||
)
|
||||
serving_chat._diffusion_mode = False
|
||||
serving_chat._diffusion_extra_body_params = frozenset()
|
||||
mocker.patch.multiple(
|
||||
serving_chat,
|
||||
_check_model=mocker.AsyncMock(return_value=None),
|
||||
_maybe_get_adapters=mocker.Mock(return_value=None),
|
||||
_effective_chat_template_kwargs=mocker.Mock(return_value={}),
|
||||
_preprocess_chat=mocker.AsyncMock(return_value=([], [{"prompt": "raw"}])),
|
||||
_base_request_id=mocker.Mock(return_value="test"),
|
||||
_log_inputs=mocker.Mock(),
|
||||
chat_completion_full_generator=mocker.AsyncMock(return_value="done"),
|
||||
)
|
||||
request = ChatCompletionRequest(
|
||||
model="test",
|
||||
messages=[{"role": "user", "content": "describe this image"}],
|
||||
modalities=["text"],
|
||||
)
|
||||
|
||||
assert asyncio.run(serving_chat._create_chat_completion(request)) == "done"
|
||||
|
||||
sampling_params_list = cast(list[Any], captured["sampling_params_list"])
|
||||
assert sampling_params_list[0].extra_args == {"ar_task_mode": "comprehension"}
|
||||
assert sampling_params_list[1].extra_args == {}
|
||||
assert captured["output_modalities"] == ["text"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request(mocker: MockerFixture):
|
||||
"""Create a mock request with all OpenAI sampling params set to None."""
|
||||
@@ -865,3 +925,54 @@ class TestResolveHeightWidth:
|
||||
h, w = OmniOpenAIServingChat._resolve_height_width_from_extra_body({"size": "invalid"})
|
||||
assert h is None
|
||||
assert w is None
|
||||
|
||||
|
||||
# Tests for _apply_text_chat_ar_task_mode (#6088)
|
||||
|
||||
|
||||
def _tag_request(modalities):
|
||||
return SimpleNamespace(modalities=modalities)
|
||||
|
||||
|
||||
def _apply_tag(params, modalities):
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
OmniOpenAIServingChat._apply_text_chat_ar_task_mode(params, _tag_request(modalities))
|
||||
return params
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modalities", [[], ["text"]])
|
||||
def test_text_only_chat_tags_ar_stage_as_comprehension(modalities):
|
||||
ar = SamplingParams()
|
||||
params = _apply_tag([ar], modalities)
|
||||
assert params[0].extra_args == {"ar_task_mode": "comprehension"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modalities", [["image"], ["text", "audio"], ["video"]])
|
||||
def test_non_text_output_request_is_untouched(modalities):
|
||||
ar = SamplingParams()
|
||||
_apply_tag([ar], modalities)
|
||||
assert ar.extra_args is None
|
||||
|
||||
|
||||
def test_diffusion_stage_params_are_untouched():
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
ar = SamplingParams()
|
||||
dit = OmniDiffusionSamplingParams()
|
||||
dit_extra_before = getattr(dit, "extra_args", None)
|
||||
_apply_tag([ar, dit], ["text"])
|
||||
assert ar.extra_args == {"ar_task_mode": "comprehension"}
|
||||
assert getattr(dit, "extra_args", None) == dit_extra_before
|
||||
|
||||
|
||||
def test_explicit_caller_ar_task_mode_is_preserved():
|
||||
ar = SamplingParams(extra_args={"ar_task_mode": "generation"})
|
||||
_apply_tag([ar], ["text"])
|
||||
assert ar.extra_args["ar_task_mode"] == "generation"
|
||||
|
||||
|
||||
def test_existing_extra_args_are_merged_not_replaced():
|
||||
ar = SamplingParams(extra_args={"custom": 1})
|
||||
_apply_tag([ar], None)
|
||||
assert ar.extra_args == {"custom": 1, "ar_task_mode": "comprehension"}
|
||||
@@ -27,6 +27,7 @@ from pydantic import ValidationError
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler
|
||||
from vllm_omni.diffusion.sched.step_scheduler import StepScheduler
|
||||
@@ -1039,6 +1040,38 @@ class TestTTSMethods:
|
||||
asyncio.run(server._prepare_speech_generation(request))
|
||||
server.shutdown()
|
||||
|
||||
def test_prepare_speech_rejects_typed_non_tts_omni_model(self, mocker: MockerFixture):
|
||||
"""Typed Qwen3-Omni stages must hit the same talker guard as legacy stages."""
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.tts_max_instructions_length = None
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
stage_pipeline_config=StagePipelineConfig(stage_id=index, model_stage=model_stage),
|
||||
model_config=SimpleNamespace(model_arch=None),
|
||||
worker_type=worker_type,
|
||||
)
|
||||
for index, (model_stage, worker_type) in enumerate(
|
||||
(
|
||||
("thinker", "ar"),
|
||||
("talker", "ar"),
|
||||
("code2wav", "generation"),
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
request = OpenAICreateSpeechRequest(input="Hello world")
|
||||
with pytest.raises(ValueError, match="only supported for dedicated TTS models"):
|
||||
asyncio.run(server._prepare_speech_generation(request))
|
||||
server.shutdown()
|
||||
|
||||
def test_estimate_prompt_len_fallback(self, speech_server):
|
||||
"""Test prompt length estimation falls back to 2048 when model is unavailable."""
|
||||
tts_params = {"text": ["Hello"], "task_type": ["CustomVoice"]}
|
||||
@@ -1559,7 +1592,8 @@ class TestTTSMethods:
|
||||
|
||||
assert first[1] == 24000
|
||||
assert second[1] == 24000
|
||||
assert first[0] is second[0]
|
||||
assert first[0] == second[0]
|
||||
assert first[0] is not second[0]
|
||||
assert first[0][0] == pytest.approx(float(wav[0]), abs=1e-4)
|
||||
cache_key = first[2]
|
||||
assert speech_server._get_resolved_ref_audio_artifact_key(
|
||||
@@ -1826,6 +1860,53 @@ class TestTTSMethods:
|
||||
assert prompt_a["additional_information"]["ref_audio_cache_key"] == "key_aaa"
|
||||
assert prompt_b["additional_information"]["ref_audio_cache_key"] == "key_bbb"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_higgs_v2_cache_salt_changes_for_same_shaped_reference_audio(self, speech_server, mocker):
|
||||
"""Higgs v2 must salt identical placeholder prompts by resolved audio content."""
|
||||
build_voice_clone_prompt = mocker.patch(
|
||||
"vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer.build_voice_clone_prompt"
|
||||
)
|
||||
build_voice_clone_prompt.side_effect = lambda _processor, _input, wav, _sr, _ref_text: {
|
||||
"prompt_token_ids": [1, 151700, 151700, 2],
|
||||
"audio_input_ids": torch.tensor([[100 if wav[0] < 0.5 else 900, 2], [3, 4]], dtype=torch.long),
|
||||
"audio_input_ids_mask": torch.ones(2, dtype=torch.bool),
|
||||
}
|
||||
|
||||
speech_server._tts_model_type = "higgs_audio_v2"
|
||||
speech_server._adapter = speech_server._get_tts_adapter()
|
||||
speech_server._adapter._resolve_higgs_audio_v2_processor = mocker.AsyncMock(return_value=object())
|
||||
|
||||
req_a = OpenAICreateSpeechRequest(
|
||||
input="hello",
|
||||
ref_audio="file:///data/spk.wav",
|
||||
ref_text="transcript",
|
||||
)
|
||||
req_b = OpenAICreateSpeechRequest(
|
||||
input="hello",
|
||||
ref_audio="file:///data/spk.wav",
|
||||
ref_text="transcript",
|
||||
)
|
||||
speech_server._adapter._resolve_ref_audio = mocker.AsyncMock(
|
||||
side_effect=[([0.1] * 48000, 24000, "key_aaa"), ([0.9] * 48000, 24000, "key_bbb")]
|
||||
)
|
||||
|
||||
prompt_a = await speech_server._adapter._build_higgs_audio_v2_params(req_a)
|
||||
prompt_b = await speech_server._adapter._build_higgs_audio_v2_params(req_b)
|
||||
|
||||
assert prompt_a["prompt_token_ids"] == [1, 151700, 151700, 2]
|
||||
assert prompt_a["prompt_token_ids"] == prompt_b["prompt_token_ids"]
|
||||
assert (
|
||||
prompt_a["additional_information"]["audio_input_ids"].shape
|
||||
== prompt_b["additional_information"]["audio_input_ids"].shape
|
||||
)
|
||||
assert not torch.equal(
|
||||
prompt_a["additional_information"]["audio_input_ids"],
|
||||
prompt_b["additional_information"]["audio_input_ids"],
|
||||
)
|
||||
assert prompt_a["cache_salt"] != prompt_b["cache_salt"]
|
||||
assert prompt_a["additional_information"]["ref_audio_cache_key"] == "key_aaa"
|
||||
assert prompt_b["additional_information"]["ref_audio_cache_key"] == "key_bbb"
|
||||
|
||||
# ── MossReferenceEncoder: speaker cache invalidation ──
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -2241,6 +2322,7 @@ class TestTTSMethods:
|
||||
|
||||
assert params["task_type"] == ["Base"]
|
||||
assert params["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in params
|
||||
|
||||
def test_build_tts_params_base_omits_non_streaming_mode_by_default(self, speech_server):
|
||||
"""Base task should keep using the model default when no override is sent."""
|
||||
@@ -2255,6 +2337,7 @@ class TestTTSMethods:
|
||||
|
||||
assert params["task_type"] == ["Base"]
|
||||
assert "non_streaming_mode" not in params
|
||||
assert "full_utterance_decode" not in params
|
||||
|
||||
def test_build_tts_params_explicit_non_streaming_mode_overrides_voicedesign_default(self, speech_server):
|
||||
"""Explicit false should not be replaced by the VoiceDesign fallback."""
|
||||
@@ -2269,6 +2352,38 @@ class TestTTSMethods:
|
||||
|
||||
assert params["task_type"] == ["VoiceDesign"]
|
||||
assert params["non_streaming_mode"] == [False]
|
||||
assert "full_utterance_decode" not in params
|
||||
|
||||
def test_build_tts_params_streaming_voicedesign_keeps_prompt_mode_not_full_decode(self, speech_server):
|
||||
"""Streaming VoiceDesign defaults: prompt-mode True, no full_utterance_decode."""
|
||||
req = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
task_type="VoiceDesign",
|
||||
instructions="warm and calm",
|
||||
stream=True,
|
||||
response_format="pcm",
|
||||
)
|
||||
|
||||
params = speech_server._build_tts_params(req)
|
||||
|
||||
assert params["task_type"] == ["VoiceDesign"]
|
||||
assert params["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in params
|
||||
|
||||
def test_build_tts_params_streaming_customvoice_explicit_non_streaming_mode_still_windowed(self, speech_server):
|
||||
"""Explicit non_streaming_mode=True must not inject full_utterance_decode."""
|
||||
req = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
voice="Vivian",
|
||||
non_streaming_mode=True,
|
||||
stream=True,
|
||||
response_format="pcm",
|
||||
)
|
||||
|
||||
params = speech_server._build_tts_params(req)
|
||||
|
||||
assert params["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in params
|
||||
|
||||
def test_load_supported_speakers(self, mocker: MockerFixture):
|
||||
"""Test _load_supported_speakers."""
|
||||
@@ -3287,7 +3402,7 @@ class TestStreamingResponse:
|
||||
finalized_tts_params = {"_qwen3_tts_effective_max_tokens": [192]}
|
||||
captured: dict = {}
|
||||
|
||||
async def prepare(_request, request_id=None):
|
||||
async def prepare(_request, request_id=None, arrival_time=None):
|
||||
return request_id, object(), finalized_tts_params
|
||||
|
||||
async def generate_chunks(
|
||||
@@ -3296,6 +3411,7 @@ class TestStreamingResponse:
|
||||
_response_format="pcm",
|
||||
raw_request=None,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
usage_acc=None,
|
||||
tts_params=None,
|
||||
@@ -3758,6 +3874,7 @@ def test_api_server_create_speech_wraps_error_response_status(mocker: MockerFixt
|
||||
def _make_api_server_request(handler, *, method: str = "POST", path: str = "/v1/audio/voices") -> Request:
|
||||
app = FastAPI()
|
||||
app.state.openai_serving_speech = handler
|
||||
app.state.api_server_count = 1
|
||||
scope = {
|
||||
"type": "http",
|
||||
"app": app,
|
||||
@@ -4919,89 +5036,6 @@ class TestMingFlashOmniTTSServing:
|
||||
assert prompt["additional_information"]["voice_name"] == "test"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dots_tts_server(mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_supported_speakers",
|
||||
return_value=set(),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.tts_adapters.base.load_codec_frame_rate",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(
|
||||
model="dots-studio/dots.tts-soar",
|
||||
)
|
||||
mock_engine_client.default_sampling_params_list = [
|
||||
SimpleNamespace(max_tokens=2048, min_tokens=None, extra_args=None)
|
||||
]
|
||||
mock_engine_client.tts_batch_max_items = 32
|
||||
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(model_stage="latent_generator", model_arch="DotsTTSForConditionalGeneration"),
|
||||
tts_args={},
|
||||
)
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
return OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
class TestDotsTTSServing:
|
||||
def test_dots_tts_prompt_validation(self, dots_tts_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_text="Reference transcript")
|
||||
error = dots_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "ref_text" in error
|
||||
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
||||
error = dots_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "ref_audio" in error
|
||||
|
||||
request = OpenAICreateSpeechRequest(input="Hello", voice="test")
|
||||
error = dots_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "voice" in error
|
||||
|
||||
request = OpenAICreateSpeechRequest(input="Hello", speaker_embedding=[1, 2, 3])
|
||||
error = dots_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "speaker_embedding" in error
|
||||
|
||||
request = OpenAICreateSpeechRequest(input="Hello", x_vector_only_mode=True)
|
||||
error = dots_tts_server._validate_tts_request(request)
|
||||
assert error is not None
|
||||
assert "x_vector_only_mode" in error
|
||||
|
||||
def test_dots_tts_adapter_awaits_async_prompt_builder(self, dots_tts_server, mocker: MockerFixture):
|
||||
build_prompt_async = mocker.patch.object(
|
||||
dots_tts_server._adapter,
|
||||
"_build_prompt_async",
|
||||
new=mocker.AsyncMock(return_value={"prompt_token_ids": [1, 2, 3]}),
|
||||
)
|
||||
request = OpenAICreateSpeechRequest(input="Hello")
|
||||
asyncio.run(dots_tts_server._prepare_speech_generation(request))
|
||||
build_prompt_async.assert_awaited_once_with("Hello")
|
||||
|
||||
def test_dots_tts_adapter_apply_sampling_overrides(self, dots_tts_server, mocker: MockerFixture):
|
||||
mocker.patch.object(dots_tts_server._adapter, "build", return_value=PreparedRequest(prompt="Hello"))
|
||||
request = OpenAICreateSpeechRequest(input="Hello", max_new_tokens=10)
|
||||
asyncio.run(dots_tts_server._prepare_speech_generation(request))
|
||||
sampling_params_list = dots_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"]
|
||||
assert sampling_params_list[0].max_tokens == 10
|
||||
|
||||
|
||||
class TestTTSAsyncOffloading:
|
||||
"""Tests for event-loop-safe offloading of blocking TTS operations."""
|
||||
|
||||
@@ -5098,6 +5132,7 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
def test_prepare_speech_generation_awaits_voxtral_async(self, voxtral_server, mocker: MockerFixture):
|
||||
"""Voxtral path in _prepare_speech_generation should call the async wrapper."""
|
||||
voxtral_server._adapter._encoder_loaded = mocker.AsyncMock(return_value=False)
|
||||
voxtral_server._adapter._build_prompt_async = mocker.AsyncMock(
|
||||
return_value={
|
||||
"prompt_token_ids": [1, 2, 3],
|
||||
@@ -5176,6 +5211,52 @@ class TestTTSAsyncOffloading:
|
||||
for call in log_info.call_args_list
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("word_timestamps", [False, True])
|
||||
def test_streaming_timestamps_retain_complete_audio(self, qwen3_tts_server, mocker, word_timestamps):
|
||||
from vllm import SamplingParams
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
|
||||
defaults = [SamplingParams(), SamplingParams()]
|
||||
qwen3_tts_server.engine_client.default_sampling_params_list = defaults
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._adapter._build_tts_params = mocker.MagicMock(
|
||||
return_value={"text": ["hello"], "task_type": ["CustomVoice"], "speaker": ["Vivian"]}
|
||||
)
|
||||
qwen3_tts_server._adapter._estimate_prompt_len_async = mocker.AsyncMock(return_value=512)
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="hello", stream=True, response_format="pcm", word_timestamps=word_timestamps
|
||||
)
|
||||
asyncio.run(qwen3_tts_server._prepare_speech_generation(request))
|
||||
|
||||
params = qwen3_tts_server.engine_client.generate.call_args.kwargs["sampling_params_list"]
|
||||
expected = RequestOutputKind.CUMULATIVE if word_timestamps else RequestOutputKind.DELTA
|
||||
assert all(sp.output_kind == expected for sp in params)
|
||||
assert all(sp.output_kind == RequestOutputKind.CUMULATIVE for sp in defaults)
|
||||
assert all(sp is not original for sp, original in zip(params, defaults))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cumulative_timestamp_audio_emits_each_sample_once(self, qwen3_tts_server):
|
||||
audio = torch.linspace(-0.5, 0.5, 2400)
|
||||
|
||||
async def cumulative_generator():
|
||||
for end in (800, 1600, 2400, 2400):
|
||||
yield OmniRequestOutput(
|
||||
request_id="req-cumulative",
|
||||
final_output_type="audio",
|
||||
_multimodal_output={"audio": audio[:end], "sr": 24000},
|
||||
)
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in qwen3_tts_server._generate_pcm_chunks(
|
||||
cumulative_generator(), "req-cumulative", cumulative_audio=True
|
||||
)
|
||||
]
|
||||
assert len(chunks) == 3
|
||||
assert len(b"".join(chunks)) == 2400 * 2
|
||||
decoded = np.frombuffer(b"".join(chunks), dtype=np.int16)
|
||||
assert np.all(np.diff(decoded.astype(np.int32)) >= 0)
|
||||
|
||||
def test_prepare_speech_generation_treats_sse_as_streaming(self, qwen3_tts_server, mocker: MockerFixture):
|
||||
"""stream_format=sse should request delta-style multimodal outputs."""
|
||||
qwen3_tts_server._adapter.validate = mocker.MagicMock(return_value=None)
|
||||
@@ -5253,6 +5334,7 @@ class TestTTSAsyncOffloading:
|
||||
"""FINAL_ONLY streaming for async_chunk=False is scoped to qwen3_tts only."""
|
||||
voxtral_server.engine_client.model_config.async_chunk = False
|
||||
mocker.patch.object(voxtral_server._get_tts_adapter(), "validate", return_value=None)
|
||||
voxtral_server._adapter._encoder_loaded = mocker.AsyncMock(return_value=False)
|
||||
voxtral_server._adapter._build_prompt_async = mocker.AsyncMock(
|
||||
return_value={
|
||||
"prompt_token_ids": [1, 2, 3],
|
||||
@@ -5286,9 +5368,34 @@ class TestTTSAsyncOffloading:
|
||||
|
||||
assert tts_params["task_type"] == ["VoiceDesign"]
|
||||
assert tts_params["non_streaming_mode"] == [False]
|
||||
assert "full_utterance_decode" not in tts_params
|
||||
prompt = qwen3_tts_server.engine_client.generate.call_args.kwargs["prompt"]
|
||||
assert prompt["additional_information"] is tts_params
|
||||
assert prompt["additional_information"]["non_streaming_mode"] == [False]
|
||||
assert "full_utterance_decode" not in prompt["additional_information"]
|
||||
|
||||
def test_prepare_speech_generation_qwen3_streaming_voicedesign_default_params(
|
||||
self, qwen3_tts_server, mocker: MockerFixture
|
||||
):
|
||||
"""Streaming VoiceDesign defaults keep prompt-mode True without full_utterance_decode."""
|
||||
qwen3_tts_server._validate_tts_request = mocker.MagicMock(return_value=None)
|
||||
qwen3_tts_server._estimate_prompt_len_async = mocker.AsyncMock(return_value=512)
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="hello",
|
||||
task_type="VoiceDesign",
|
||||
instructions="warm and calm",
|
||||
stream=True,
|
||||
response_format="pcm",
|
||||
)
|
||||
_request_id, _generator, tts_params = asyncio.run(qwen3_tts_server._prepare_speech_generation(request))
|
||||
|
||||
assert tts_params["task_type"] == ["VoiceDesign"]
|
||||
assert tts_params["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in tts_params
|
||||
prompt = qwen3_tts_server.engine_client.generate.call_args.kwargs["prompt"]
|
||||
assert prompt["additional_information"]["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in prompt["additional_information"]
|
||||
|
||||
def test_prepare_speech_generation_qwen3_base_non_streaming_mode_true(
|
||||
self, qwen3_tts_server, mocker: MockerFixture
|
||||
@@ -5311,9 +5418,11 @@ class TestTTSAsyncOffloading:
|
||||
assert tts_params["task_type"] == ["Base"]
|
||||
assert tts_params["ref_text"] == ["reference transcript"]
|
||||
assert tts_params["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in tts_params
|
||||
prompt = qwen3_tts_server.engine_client.generate.call_args.kwargs["prompt"]
|
||||
assert prompt["additional_information"] is tts_params
|
||||
assert prompt["additional_information"]["non_streaming_mode"] == [True]
|
||||
assert "full_utterance_decode" not in prompt["additional_information"]
|
||||
|
||||
def test_qwen3_repeated_ref_audio_hot_path_sends_cache_key_without_waveform(self, qwen3_tts_server):
|
||||
"""After a ref artifact is marked ready, repeated requests avoid ref_audio payload IPC."""
|
||||
@@ -5322,7 +5431,7 @@ class TestTTSAsyncOffloading:
|
||||
ref_audio = "data:audio/wav;base64,same"
|
||||
qwen3_tts_server._put_resolved_ref_audio(
|
||||
hashlib.sha1(ref_audio.encode("utf-8")).hexdigest(),
|
||||
wav_list,
|
||||
np.asarray(wav_list, dtype=np.float32),
|
||||
24000,
|
||||
artifact_key,
|
||||
)
|
||||
@@ -5358,9 +5467,9 @@ class TestTTSAsyncOffloading:
|
||||
qwen3_tts_server._ref_audio_resolve_cache_max_entries = 1
|
||||
qwen3_tts_server._ref_audio_resolve_cache_max_bytes = 1_000_000
|
||||
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a")
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", np.zeros(8, dtype=np.float32), 24000, "artifact-a")
|
||||
qwen3_tts_server._ref_audio_model_artifact_ready.add(("artifact-a", False))
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-b", [0.0] * 8, 24000, "artifact-b")
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-b", np.zeros(8, dtype=np.float32), 24000, "artifact-b")
|
||||
|
||||
assert ("artifact-a", False) not in qwen3_tts_server._ref_audio_model_artifact_ready
|
||||
assert "artifact-b" in {entry[3] for entry in qwen3_tts_server._ref_audio_resolve_cache.values()}
|
||||
@@ -5607,7 +5716,7 @@ class TestTTSAsyncOffloading:
|
||||
def test_qwen3_xvector_ready_artifact_does_not_enable_icl_artifact_only(self, qwen3_tts_server):
|
||||
# An x-vector-only artifact (speaker embedding, no ref_code) must not enable
|
||||
# the artifact-only path for a later ICL request with the same ref_audio (#5049).
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a")
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", np.zeros(8, dtype=np.float32), 24000, "artifact-a")
|
||||
qwen3_tts_server._track_ref_audio_artifact_warmup("req-xvec", "artifact-a", x_vector_only=True)
|
||||
qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-xvec")
|
||||
|
||||
@@ -5615,7 +5724,7 @@ class TestTTSAsyncOffloading:
|
||||
assert qwen3_tts_server._adapter._qwen3_tts_can_use_ref_audio_artifact_only(icl_params, "artifact-a") is False
|
||||
|
||||
def test_qwen3_xvector_ready_artifact_still_reusable_by_xvector_request(self, qwen3_tts_server):
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a")
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", np.zeros(8, dtype=np.float32), 24000, "artifact-a")
|
||||
qwen3_tts_server._track_ref_audio_artifact_warmup("req-xvec", "artifact-a", x_vector_only=True)
|
||||
qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-xvec")
|
||||
|
||||
@@ -5623,7 +5732,7 @@ class TestTTSAsyncOffloading:
|
||||
assert qwen3_tts_server._adapter._qwen3_tts_can_use_ref_audio_artifact_only(xvec_params, "artifact-a") is True
|
||||
|
||||
def test_qwen3_icl_ready_artifact_enables_icl_artifact_only(self, qwen3_tts_server):
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", [0.0] * 8, 24000, "artifact-a")
|
||||
qwen3_tts_server._put_resolved_ref_audio("ref-a", np.zeros(8, dtype=np.float32), 24000, "artifact-a")
|
||||
qwen3_tts_server._track_ref_audio_artifact_warmup("req-icl", "artifact-a", x_vector_only=False)
|
||||
qwen3_tts_server._mark_ref_audio_artifact_ready_for_request("req-icl")
|
||||
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Prometheus metric coverage for the Speech API audio path."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.openai import serving_speech as speech_module
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _MetricsStub:
|
||||
def __init__(self) -> None:
|
||||
self.ttfp_calls: list[tuple[str, str, float]] = []
|
||||
self.underrun_calls: list[tuple[str, str, float]] = []
|
||||
self.continuity_calls: list[tuple[str, str, int]] = []
|
||||
self.abort_calls: list[str] = []
|
||||
self.completed = 0
|
||||
|
||||
def inc_speech_stream_completed(self) -> None:
|
||||
self.completed += 1
|
||||
|
||||
def inc_speech_stream_aborted(self, reason: str) -> None:
|
||||
self.abort_calls.append(reason)
|
||||
|
||||
def observe_audio_ttfp(self, stage: str, replica: str, seconds: float) -> None:
|
||||
self.ttfp_calls.append((stage, replica, seconds))
|
||||
|
||||
def observe_audio_underrun(self, stage: str, replica: str, seconds: float) -> None:
|
||||
self.underrun_calls.append((stage, replica, seconds))
|
||||
|
||||
def inc_audio_continuity_ok(self, stage: str, replica: str, threshold_ms: int) -> None:
|
||||
self.continuity_calls.append((stage, replica, threshold_ms))
|
||||
|
||||
|
||||
def _serving(metrics: _MetricsStub, *, adapter=None) -> OmniOpenAIServingSpeech:
|
||||
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||
serving._tts_model_type = "qwen3_tts"
|
||||
serving.engine_client = SimpleNamespace(mod_metrics=metrics, request_states={})
|
||||
serving._get_tts_adapter = lambda: adapter
|
||||
serving._speech_output_policies = {}
|
||||
serving.create_audio = lambda audio_obj: SimpleNamespace(
|
||||
audio_data=b"\0\0" * int(audio_obj.audio_tensor.size),
|
||||
media_type="audio/pcm",
|
||||
)
|
||||
serving._mark_ref_audio_artifact_ready_for_request = lambda request_id: None
|
||||
serving._discard_ref_audio_artifact_warmup = lambda request_id: None
|
||||
return serving
|
||||
|
||||
|
||||
def _result(samples: int = 320, *, stage_id: int = 1, replica_id: int | None = 2) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
multimodal_output={"audio": torch.zeros(samples), "sr": 16000},
|
||||
stage_id=stage_id,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
|
||||
|
||||
async def _generate(*results):
|
||||
for result in results:
|
||||
yield result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_speech_observes_ttfp_once_on_first_pcm_payload(monkeypatch):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.openai.serving_speech.time.time", lambda: 100.25)
|
||||
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_generate(_result(), _result()),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
)
|
||||
assert len([chunk async for chunk in chunks]) == 2
|
||||
assert metrics.ttfp_calls == [("1", "2", pytest.approx(0.25))]
|
||||
assert metrics.underrun_calls == [("1", "2", pytest.approx(0.0))]
|
||||
assert metrics.continuity_calls == [("1", "2", 100)]
|
||||
assert metrics.abort_calls == []
|
||||
assert metrics.completed == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("before_pcm", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
("error_type", "reason"),
|
||||
[(asyncio.CancelledError, "cancelled"), (speech_module.EngineDeadError, "engine_dead"), (RuntimeError, "error")],
|
||||
)
|
||||
async def test_speech_stream_abort_counted_once_without_continuity(before_pcm, error_type, reason):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
|
||||
async def results():
|
||||
if not before_pcm:
|
||||
yield _result()
|
||||
raise error_type()
|
||||
|
||||
chunks = serving._generate_audio_chunks(results(), request_id="speech-test", request_arrival_ts=100.0)
|
||||
with pytest.raises(error_type):
|
||||
_ = [chunk async for chunk in chunks]
|
||||
|
||||
assert metrics.abort_calls == [reason]
|
||||
assert metrics.completed == 0
|
||||
assert len(metrics.ttfp_calls) == (0 if before_pcm else 1)
|
||||
assert metrics.underrun_calls == []
|
||||
assert metrics.continuity_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_speech_stream_close_counted_once_without_continuity():
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_generate(_result(), _result()), request_id="speech-test", request_arrival_ts=100.0
|
||||
)
|
||||
assert await anext(chunks)
|
||||
await chunks.aclose()
|
||||
await chunks.aclose()
|
||||
assert metrics.abort_calls == ["closed"]
|
||||
assert metrics.completed == 0
|
||||
assert metrics.underrun_calls == []
|
||||
assert metrics.continuity_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_speech_retries_ttfp_labels_without_moving_first_packet_time(monkeypatch):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
clock = {"now": 100.25, "perf": 0.25}
|
||||
monkeypatch.setattr(speech_module.time, "time", lambda: clock["now"])
|
||||
monkeypatch.setattr(speech_module.time, "perf_counter", lambda: clock["perf"])
|
||||
|
||||
async def results():
|
||||
yield _result(replica_id=None)
|
||||
clock["now"] = 101.0
|
||||
clock["perf"] = 0.26
|
||||
yield _result(replica_id=2)
|
||||
clock["now"] = 101.25
|
||||
clock["perf"] = 0.27
|
||||
yield _result(replica_id=2)
|
||||
|
||||
chunks = serving._generate_audio_chunks(
|
||||
results(),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
)
|
||||
assert len([chunk async for chunk in chunks]) == 3
|
||||
assert metrics.ttfp_calls == [("1", "2", pytest.approx(0.25))]
|
||||
assert metrics.underrun_calls == [("1", "2", pytest.approx(0.0))]
|
||||
assert metrics.continuity_calls == [("1", "2", 100)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mode", ["raw", "sse", "pcm"])
|
||||
async def test_speech_disconnect_during_send_closes_entire_generator_chain(mode):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
closed = []
|
||||
|
||||
async def source():
|
||||
try:
|
||||
yield _result()
|
||||
yield _result()
|
||||
finally:
|
||||
closed.append(True)
|
||||
|
||||
# Retain the source to prove cleanup does not depend on garbage collection.
|
||||
generator = source()
|
||||
method = {
|
||||
"raw": serving._generate_audio_chunks,
|
||||
"sse": serving._generate_audio_sse_events,
|
||||
"pcm": serving._generate_pcm_chunks,
|
||||
}[mode]
|
||||
stream = method(generator, request_id="speech-test", request_arrival_ts=100.0)
|
||||
|
||||
async def send(message):
|
||||
if message["type"] == "http.response.body":
|
||||
raise OSError("client disconnected")
|
||||
|
||||
if mode == "pcm":
|
||||
# WebSocket handlers already explicitly close their PCM iterator.
|
||||
assert await anext(stream)
|
||||
await stream.aclose()
|
||||
else:
|
||||
response = speech_module._SpeechStreamingResponse(stream)
|
||||
with pytest.raises(OSError, match="client disconnected"):
|
||||
await response.stream_response(send)
|
||||
assert closed == [True]
|
||||
assert metrics.abort_calls == ["closed"]
|
||||
assert metrics.completed == 0
|
||||
assert metrics.underrun_calls == []
|
||||
assert metrics.continuity_calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arrival_ts", [0.0, -1.0])
|
||||
def test_speech_ttfp_invalid_arrival_does_not_consume_guard(arrival_ts):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
state = SimpleNamespace(
|
||||
external_request_id="speech-test",
|
||||
request_arrival_ts=arrival_ts,
|
||||
first_audio_ts=None,
|
||||
audio_emit_stage_id=None,
|
||||
audio_emit_replica_id=None,
|
||||
)
|
||||
serving.engine_client.request_states = {"internal-test": state}
|
||||
|
||||
assert serving._observe_speech_audio_ttfp(request_id="speech-test", result=_result(), first_packet_ts=100.25) == (
|
||||
1,
|
||||
2,
|
||||
False,
|
||||
)
|
||||
assert metrics.ttfp_calls == []
|
||||
assert state.first_audio_ts is None
|
||||
assert state.audio_emit_stage_id is None
|
||||
assert state.audio_emit_replica_id is None
|
||||
|
||||
state.request_arrival_ts = 100.0
|
||||
assert serving._observe_speech_audio_ttfp(request_id="speech-test", result=_result(), first_packet_ts=100.25) == (
|
||||
1,
|
||||
2,
|
||||
True,
|
||||
)
|
||||
assert metrics.ttfp_calls == [("1", "2", pytest.approx(0.25))]
|
||||
assert state.first_audio_ts == 100.25
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("during_send", [False, True])
|
||||
async def test_speech_response_task_cancellation_closes_source(during_send):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
waiting = asyncio.Event()
|
||||
blocker = asyncio.Event()
|
||||
closed = []
|
||||
|
||||
async def source():
|
||||
try:
|
||||
if not during_send:
|
||||
waiting.set()
|
||||
await blocker.wait()
|
||||
yield _result()
|
||||
finally:
|
||||
closed.append(True)
|
||||
|
||||
async def send(message):
|
||||
if during_send and message["type"] == "http.response.body":
|
||||
waiting.set()
|
||||
await blocker.wait()
|
||||
|
||||
generator = source()
|
||||
stream = serving._generate_audio_chunks(generator, request_id="speech-test", request_arrival_ts=100.0)
|
||||
response = speech_module._SpeechStreamingResponse(stream)
|
||||
task = asyncio.create_task(response.stream_response(send))
|
||||
try:
|
||||
await asyncio.wait_for(waiting.wait(), timeout=5)
|
||||
finally:
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert closed == [True]
|
||||
assert metrics.abort_calls == ["closed" if during_send else "cancelled"]
|
||||
assert metrics.completed == 0
|
||||
assert metrics.underrun_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_speech_does_not_count_empty_payload_as_first_packet(monkeypatch):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.openai.serving_speech.time.time", lambda: 100.25)
|
||||
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_generate(_result(samples=0), _result()),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
)
|
||||
assert len([chunk async for chunk in chunks]) == 2
|
||||
assert metrics.ttfp_calls == [("1", "2", pytest.approx(0.25))]
|
||||
assert metrics.underrun_calls == [("1", "2", pytest.approx(0.0))]
|
||||
assert metrics.continuity_calls == [("1", "2", 100)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_speech_does_not_finalize_continuity_on_error(monkeypatch):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.openai.serving_speech.time.time", lambda: 100.25)
|
||||
|
||||
async def failing_stream():
|
||||
yield _result()
|
||||
raise RuntimeError("stream failed")
|
||||
|
||||
chunks = serving._generate_audio_chunks(
|
||||
failing_stream(),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="stream failed"):
|
||||
_ = [chunk async for chunk in chunks]
|
||||
|
||||
assert len(metrics.ttfp_calls) == 1
|
||||
assert metrics.underrun_calls == []
|
||||
assert metrics.continuity_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_speech_does_not_finalize_continuity_on_validation_error(monkeypatch):
|
||||
metrics = _MetricsStub()
|
||||
|
||||
def reject_generation(_tts_params, **_kwargs):
|
||||
raise RuntimeError("generation validation failed")
|
||||
|
||||
adapter = SimpleNamespace(validates_generation=True, validate_generation=reject_generation)
|
||||
serving = _serving(metrics, adapter=adapter)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.openai.serving_speech.time.time", lambda: 100.25)
|
||||
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_generate(_result()),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
tts_params={"task_type": ["Base"]},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="generation validation failed"):
|
||||
_ = [chunk async for chunk in chunks]
|
||||
|
||||
assert len(metrics.ttfp_calls) == 1
|
||||
assert metrics.underrun_calls == []
|
||||
assert metrics.continuity_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_speech_reports_late_chunk_as_underrun(monkeypatch):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.openai.serving_speech.time.time", lambda: 100.25)
|
||||
perf_times = iter((0.0, 0.1, 0.1, 2.0, 2.0))
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.openai.serving_speech.time.perf_counter", lambda: next(perf_times))
|
||||
|
||||
chunks = serving._generate_audio_chunks(
|
||||
_generate(_result(), _result()),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
)
|
||||
assert len([chunk async for chunk in chunks]) == 2
|
||||
assert metrics.underrun_calls[0][:2] == ("1", "2")
|
||||
assert metrics.underrun_calls[0][2] > 0.1
|
||||
assert metrics.continuity_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streaming_speech_does_not_observe_ttfp():
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
serving._audio_encode_speed = lambda _request: 1.0
|
||||
|
||||
async def prepare(_request, **_kwargs):
|
||||
result = _result()
|
||||
result.metrics = {}
|
||||
return "speech-test", _generate(result), {}
|
||||
|
||||
serving._prepare_speech_generation = prepare
|
||||
request = OpenAICreateSpeechRequest(input="hello", response_format="pcm")
|
||||
|
||||
audio_data, media_type = await serving._generate_audio_bytes(request, request_arrival_ts=100.0)
|
||||
|
||||
assert audio_data
|
||||
assert media_type == "audio/pcm"
|
||||
assert metrics.ttfp_calls == []
|
||||
assert metrics.completed == 0
|
||||
assert metrics.abort_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("flush_only", [False, True])
|
||||
@pytest.mark.parametrize("response_format", ["pcm", "wav"])
|
||||
async def test_resampled_speech_records_flush_with_audio_producer(monkeypatch, flush_only, response_format):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
clock = {"now": 0.0}
|
||||
finalized = []
|
||||
original_finalize = speech_module.observe_audio_streaming_finalize
|
||||
|
||||
def capture_finalize(*args, **kwargs):
|
||||
finalized.append(kwargs)
|
||||
original_finalize(*args, **kwargs)
|
||||
|
||||
class BufferedResampler:
|
||||
def __init__(self, source_rate, target_rate):
|
||||
assert (source_rate, target_rate) == (16000, 24000)
|
||||
|
||||
def process(self, chunk, *, final=False):
|
||||
clock["now"] = 0.5 if final else 0.25
|
||||
if not final and flush_only:
|
||||
return np.empty(0, dtype=np.float32)
|
||||
return np.zeros(2400, dtype=np.float32)
|
||||
|
||||
monkeypatch.setattr(speech_module, "StreamingAudioResampler", BufferedResampler)
|
||||
monkeypatch.setattr(speech_module, "observe_audio_streaming_finalize", capture_finalize)
|
||||
monkeypatch.setattr(speech_module.time, "time", lambda: 100.0 + clock["now"])
|
||||
monkeypatch.setattr(speech_module.time, "perf_counter", lambda: clock["now"])
|
||||
# The last result is not audio and must not supply the flush metric labels.
|
||||
non_audio = SimpleNamespace(multimodal_output={"timestamps": []}, stage_id=9, replica_id=8)
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in serving._generate_audio_chunks(
|
||||
_generate(_result(), non_audio),
|
||||
request_id="speech-test",
|
||||
response_format=response_format,
|
||||
request_start_s=0.0,
|
||||
request_arrival_ts=100.0,
|
||||
target_sample_rate=24000,
|
||||
)
|
||||
]
|
||||
|
||||
if response_format == "wav":
|
||||
assert chunks.pop(0).startswith(b"RIFF")
|
||||
expected_arrivals = [0.5] if flush_only else [0.25, 0.5]
|
||||
assert [len(chunk) for chunk in chunks] == [4800] * len(expected_arrivals)
|
||||
assert metrics.ttfp_calls == [("1", "2", pytest.approx(expected_arrivals[0]))]
|
||||
assert len(finalized) == 1
|
||||
assert finalized[0]["sample_rate"] == 24000
|
||||
assert finalized[0]["channels"] == 1
|
||||
assert finalized[0]["chunk_bytes"] == [4800] * len(expected_arrivals)
|
||||
assert finalized[0]["chunk_arrival_times_s"] == expected_arrivals
|
||||
assert metrics.underrun_calls == [("1", "2", pytest.approx(0.0 if flush_only else 0.15))]
|
||||
assert metrics.continuity_calls == ([("1", "2", 100)] if flush_only else [])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("empty_pcm", [False, True])
|
||||
async def test_resampled_speech_without_pcm_does_not_emit_metrics(monkeypatch, empty_pcm):
|
||||
metrics = _MetricsStub()
|
||||
serving = _serving(metrics)
|
||||
|
||||
class EmptyResampler:
|
||||
def __init__(self, source_rate, target_rate):
|
||||
pass
|
||||
|
||||
def process(self, chunk, *, final=False):
|
||||
# Also cover a nonempty flush waveform whose encoder emits no PCM.
|
||||
return np.zeros(10 if final and empty_pcm else 0, dtype=np.float32)
|
||||
|
||||
monkeypatch.setattr(speech_module, "StreamingAudioResampler", EmptyResampler)
|
||||
serving.create_audio = lambda audio_obj: SimpleNamespace(audio_data=b"", media_type="audio/pcm")
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in serving._generate_audio_chunks(
|
||||
_generate(_result()),
|
||||
request_id="speech-test",
|
||||
request_arrival_ts=100.0,
|
||||
target_sample_rate=24000,
|
||||
)
|
||||
]
|
||||
assert not any(chunks)
|
||||
assert metrics.ttfp_calls == []
|
||||
assert metrics.underrun_calls == []
|
||||
assert metrics.continuity_calls == []
|
||||
@@ -48,7 +48,15 @@ def _build_test_app(
|
||||
speech_service.forced_aligner_enabled = False
|
||||
|
||||
async def mock_generate_pcm_chunks(
|
||||
_generator, _request_id, *, include_sample_rate=False, tts_params=None, collect=None
|
||||
_generator,
|
||||
_request_id,
|
||||
*,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
tts_params=None,
|
||||
collect=None,
|
||||
cumulative_audio=False,
|
||||
):
|
||||
for chunk in (b"\x01\x02", b"\x03\x04\x05"):
|
||||
yield (chunk, 24000) if include_sample_rate else chunk
|
||||
@@ -249,6 +257,7 @@ class TestStreamingSpeechWebSocket:
|
||||
|
||||
def test_streaming_multiple_binary_frames(self, mocker: MockerFixture):
|
||||
captured_requests = []
|
||||
captured_timing = {}
|
||||
captured_tts_params = []
|
||||
|
||||
speech_service = mocker.MagicMock(spec=OmniOpenAIServingSpeech)
|
||||
@@ -257,13 +266,27 @@ class TestStreamingSpeechWebSocket:
|
||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||
speech_service.forced_aligner_enabled = False
|
||||
|
||||
async def mock_prepare_speech_generation(request):
|
||||
async def mock_prepare_speech_generation(request, *, arrival_time=None):
|
||||
captured_requests.append(request)
|
||||
assert arrival_time is not None
|
||||
captured_timing["prepare_arrival"] = arrival_time
|
||||
return "req-stream", object(), {"_qwen3_tts_effective_max_tokens": [192]}
|
||||
|
||||
speech_service._prepare_speech_generation = mock_prepare_speech_generation
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False, tts_params=None):
|
||||
async def mock_generate_pcm_chunks(
|
||||
_generator,
|
||||
_request_id,
|
||||
*,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
tts_params=None,
|
||||
):
|
||||
assert request_start_s is not None
|
||||
assert request_arrival_ts is not None
|
||||
captured_timing["chunk_start"] = request_start_s
|
||||
captured_timing["chunk_arrival"] = request_arrival_ts
|
||||
captured_tts_params.append(tts_params)
|
||||
for chunk in (b"\x01\x02", b"\x03\x04\x05", b"\x06"):
|
||||
yield (chunk, 24000) if include_sample_rate else chunk
|
||||
@@ -309,6 +332,8 @@ class TestStreamingSpeechWebSocket:
|
||||
assert captured_requests[0].stream is True
|
||||
assert captured_requests[0].response_format == "pcm"
|
||||
assert captured_requests[0].initial_codec_chunk_frames == 12
|
||||
assert captured_timing["prepare_arrival"] == captured_timing["chunk_arrival"]
|
||||
assert captured_timing["chunk_start"] > 0
|
||||
assert captured_tts_params == [{"_qwen3_tts_effective_max_tokens": [192]}]
|
||||
assert speech_service._generate_audio_bytes.await_count == 0
|
||||
|
||||
@@ -341,8 +366,9 @@ class TestStreamingSpeechWebSocket:
|
||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||
speech_service.forced_aligner_enabled = True
|
||||
|
||||
async def mock_prepare_speech_generation(request):
|
||||
async def mock_prepare_speech_generation(request, *, arrival_time=None):
|
||||
captured_requests.append(request)
|
||||
assert arrival_time is not None
|
||||
return "req-stream", object(), {}
|
||||
|
||||
speech_service._prepare_speech_generation = mock_prepare_speech_generation
|
||||
@@ -353,8 +379,19 @@ class TestStreamingSpeechWebSocket:
|
||||
# The forced-aligner stage rides the same generator: its pooling output
|
||||
# is surfaced via the ``collect`` channel once the audio has streamed.
|
||||
async def mock_generate_pcm_chunks(
|
||||
_generator, _request_id, *, include_sample_rate=False, tts_params=None, collect=None
|
||||
_generator,
|
||||
_request_id,
|
||||
*,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
tts_params=None,
|
||||
collect=None,
|
||||
cumulative_audio=False,
|
||||
):
|
||||
assert request_start_s is not None
|
||||
assert request_arrival_ts is not None
|
||||
assert cumulative_audio is True
|
||||
for chunk in (first_chunk, second_chunk):
|
||||
yield (chunk, 1000) if include_sample_rate else chunk
|
||||
if collect is not None:
|
||||
@@ -438,7 +475,15 @@ class TestStreamingSpeechWebSocket:
|
||||
speech_service._prepare_speech_generation = mocker.AsyncMock(return_value=("req", object(), {}))
|
||||
|
||||
async def mock_generate_pcm_chunks(
|
||||
_generator, _request_id, *, include_sample_rate=False, tts_params=None, collect=None
|
||||
_generator,
|
||||
_request_id,
|
||||
*,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
tts_params=None,
|
||||
collect=None,
|
||||
cumulative_audio=False,
|
||||
):
|
||||
chunk = b"\x01" * 1000
|
||||
yield (chunk, 1000) if include_sample_rate else chunk
|
||||
@@ -620,7 +665,15 @@ class TestStreamingSpeechWebSocket:
|
||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||
speech_service.forced_aligner_enabled = False
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False, tts_params=None):
|
||||
async def mock_generate_pcm_chunks(
|
||||
_generator,
|
||||
_request_id,
|
||||
*,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
tts_params=None,
|
||||
):
|
||||
yield b"\x01\x02"
|
||||
raise RuntimeError("stream boom")
|
||||
|
||||
@@ -716,7 +769,15 @@ class TestStreamingSpeechWebSocket:
|
||||
speech_service.engine_client.abort = mocker.AsyncMock()
|
||||
speech_service.forced_aligner_enabled = False
|
||||
|
||||
async def mock_generate_pcm_chunks(_generator, _request_id, *, include_sample_rate=False, tts_params=None):
|
||||
async def mock_generate_pcm_chunks(
|
||||
_generator,
|
||||
_request_id,
|
||||
*,
|
||||
request_start_s=None,
|
||||
request_arrival_ts=None,
|
||||
include_sample_rate=False,
|
||||
tts_params=None,
|
||||
):
|
||||
yield b"\x01\x02"
|
||||
|
||||
speech_service._generate_pcm_chunks = mock_generate_pcm_chunks
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for `/v1/realtime/video` WebSocket video output streaming."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -7,12 +7,13 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi import FastAPI, WebSocket
|
||||
from fastapi import FastAPI, HTTPException, WebSocket
|
||||
from pytest_mock import MockerFixture
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
@@ -957,3 +958,100 @@ class TestStreamingVideoOutputPromptUpdate:
|
||||
"transition_chunks": 2,
|
||||
},
|
||||
)
|
||||
|
||||
def test_composite_prompt_and_camera_interaction_forwarded(self, mocker: MockerFixture):
|
||||
"""Prompt + camera multi_modal_data is admitted and forwarded to the engine."""
|
||||
|
||||
async def mock_generate(*_args, **_kwargs):
|
||||
yield OmniRequestOutput.from_diffusion(
|
||||
request_id="req-ws",
|
||||
images=[_fake_video_frames(2)],
|
||||
final_output_type="image",
|
||||
finished=False,
|
||||
)
|
||||
await asyncio.sleep(0.3)
|
||||
yield OmniRequestOutput.from_diffusion(
|
||||
request_id="req-ws",
|
||||
images=[_fake_video_frames(2)],
|
||||
final_output_type="image",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
app, _handler, engine_client = _build_test_app(
|
||||
mocker=mocker,
|
||||
streaming_chunks=[(b"mp4-chunk-0", False), (b"mp4-chunk-1", True)],
|
||||
mock_generate=mock_generate,
|
||||
)
|
||||
|
||||
camera_payload = {
|
||||
"mode": "target",
|
||||
"data": {"translation": [0.0, 0.0, 1.0]},
|
||||
}
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect("/v1/realtime/video") as ws:
|
||||
ws.send_json({"type": "session.start", "prompt": "initial scene"})
|
||||
start = ws.receive_json()
|
||||
assert start["type"] == "video.start"
|
||||
request_id = start["request_id"]
|
||||
|
||||
assert _receive_video_chunk(ws)[1] == b"mp4-chunk-0"
|
||||
|
||||
ws.send_json(
|
||||
{
|
||||
"type": "session.interaction",
|
||||
"interaction": {
|
||||
"event_id": "combo-1",
|
||||
"event": {
|
||||
"prompt": "new scene",
|
||||
"multi_modal_data": {"camera": camera_payload},
|
||||
},
|
||||
"transition_chunks": 2,
|
||||
},
|
||||
}
|
||||
)
|
||||
queued = ws.receive_json()
|
||||
assert queued["type"] == "session.interaction.queued"
|
||||
assert queued["event_id"] == "combo-1"
|
||||
|
||||
assert _receive_video_chunk(ws)[1] == b"mp4-chunk-1"
|
||||
assert ws.receive_json()["type"] == "session.done"
|
||||
|
||||
engine_client.submit_interaction_async.assert_awaited_once_with(
|
||||
request_id,
|
||||
interaction={
|
||||
"event_id": "combo-1",
|
||||
"event": {
|
||||
"prompt": "new scene",
|
||||
"multi_modal_data": {"camera": camera_payload},
|
||||
},
|
||||
"transition_chunks": 2,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestStreamingVideoOutputExtraParams:
|
||||
"""extra_params handling when building streaming generation inputs."""
|
||||
|
||||
@staticmethod
|
||||
def _handler() -> OmniStreamingVideoOutputHandler:
|
||||
handler = object.__new__(OmniStreamingVideoOutputHandler)
|
||||
handler._engine_client = SimpleNamespace(default_sampling_params_list=[])
|
||||
return handler
|
||||
|
||||
async def test_preencode_mp4_is_rejected_for_streaming_sessions(self):
|
||||
"""Worker-side pre-encoding emits one progressive MP4 the fMP4 encoder cannot consume."""
|
||||
request = VideoGenerationRequest(prompt="p", extra_params={"preencode_mp4": True})
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await self._handler()._build_prompt_and_sampling_params(request)
|
||||
|
||||
assert excinfo.value.status_code == HTTPStatus.BAD_REQUEST.value
|
||||
assert "preencode_mp4" in excinfo.value.detail
|
||||
|
||||
async def test_other_extra_params_still_reach_the_engine(self):
|
||||
request = VideoGenerationRequest(prompt="p", extra_params={"flow_shift": 3.0})
|
||||
|
||||
_, gen_params, _ = await self._handler()._build_prompt_and_sampling_params(request)
|
||||
|
||||
assert gen_params.extra_args["flow_shift"] == 3.0
|
||||
@@ -1,19 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Tests for the serving-layer streaming video WebSocket handler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import gc
|
||||
import io
|
||||
import json
|
||||
import threading
|
||||
import weakref
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
|
||||
from tests.helpers.serving_chat import build_serving_chat
|
||||
from vllm_omni.config.stage_config import StageConfig
|
||||
from vllm_omni.entrypoints.openai import video_stream_base, video_stream_envs
|
||||
from vllm_omni.entrypoints.openai.serving_video_stream import (
|
||||
QwenOmniStreamingVideoHandler,
|
||||
@@ -38,10 +45,10 @@ def _b64(data: bytes) -> str:
|
||||
|
||||
def _text_result(text: str) -> OmniRequestOutput:
|
||||
class Output:
|
||||
pass
|
||||
text: str
|
||||
|
||||
class RequestOutput:
|
||||
pass
|
||||
outputs: list[Output]
|
||||
|
||||
output = Output()
|
||||
output.text = text
|
||||
@@ -52,10 +59,10 @@ def _text_result(text: str) -> OmniRequestOutput:
|
||||
|
||||
def _audio_result(audio_data: Any) -> OmniRequestOutput:
|
||||
class Output:
|
||||
pass
|
||||
multimodal_output: dict[str, Any]
|
||||
|
||||
class RequestOutput:
|
||||
pass
|
||||
outputs: list[Output]
|
||||
|
||||
output = Output()
|
||||
output.multimodal_output = {"audio": audio_data}
|
||||
@@ -138,6 +145,130 @@ async def test_receive_config_accepts_client_legacy_aliases():
|
||||
assert config.frame_filter_threshold == 0.87
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def run_sampling_session(monkeypatch):
|
||||
async def run(config_overrides):
|
||||
engine = MagicMock()
|
||||
engine.stage_configs = [
|
||||
StageConfig(stage_id=index, model_stage=name)
|
||||
for index, name in enumerate(("thinker", "talker", "code2wav"))
|
||||
]
|
||||
engine.default_sampling_params_list = [
|
||||
SamplingParams(temperature=0.4, max_tokens=64, top_p=0.85),
|
||||
SamplingParams(temperature=0.7, max_tokens=96),
|
||||
SamplingParams(temperature=0.8, max_tokens=128),
|
||||
]
|
||||
|
||||
async def generate(**kwargs):
|
||||
yield _text_result("answer")
|
||||
|
||||
engine.generate.side_effect = generate
|
||||
handler = QwenOmniStreamingVideoHandler(
|
||||
chat_service=build_serving_chat(engine_client=engine),
|
||||
engine_client=engine,
|
||||
)
|
||||
monkeypatch.setattr(handler, "_preprocess_to_engine_prompt", AsyncMock(return_value={"prompt": "video"}))
|
||||
ws = MockWebSocket(
|
||||
[
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": "test",
|
||||
"modalities": ["text"],
|
||||
"enable_frame_filter": False,
|
||||
**config_overrides,
|
||||
}
|
||||
),
|
||||
json.dumps({"type": "video.frame", "data": _b64(_make_jpeg())}),
|
||||
json.dumps({"type": "video.query", "text": "describe"}),
|
||||
json.dumps({"type": "video.done"}),
|
||||
]
|
||||
)
|
||||
await handler.handle_session(ws)
|
||||
return ws, engine
|
||||
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_chunk_mode", ["on", "off"])
|
||||
@pytest.mark.parametrize("num_stage_overrides", [1, 3])
|
||||
async def test_video_sampling_params_reach_engine(
|
||||
run_sampling_session, monkeypatch, async_chunk_mode, num_stage_overrides
|
||||
):
|
||||
monkeypatch.setenv("VLLM_VIDEO_ASYNC_CHUNK", async_chunk_mode)
|
||||
overrides = [
|
||||
{"temperature": 0.2, "max_tokens": 1, "seed": 42},
|
||||
{"temperature": 0.6, "max_tokens": 12},
|
||||
{"temperature": 0.9, "max_tokens": 24},
|
||||
][:num_stage_overrides]
|
||||
|
||||
ws, engine = await run_sampling_session({"sampling_params_list": overrides})
|
||||
|
||||
assert not [msg for msg in ws.sent if msg["type"] == "error"]
|
||||
engine.generate.assert_called_once()
|
||||
params = engine.generate.call_args.kwargs["sampling_params_list"]
|
||||
assert len(params) == 3
|
||||
for index, param in enumerate(params):
|
||||
assert isinstance(param, SamplingParams)
|
||||
default = engine.default_sampling_params_list[index]
|
||||
expected = (
|
||||
overrides[index]
|
||||
if index < len(overrides)
|
||||
else {
|
||||
"temperature": default.temperature,
|
||||
"max_tokens": default.max_tokens,
|
||||
}
|
||||
)
|
||||
assert param.temperature == expected["temperature"]
|
||||
assert param.max_tokens == expected["max_tokens"]
|
||||
assert param.output_kind == RequestOutputKind.DELTA
|
||||
assert param is not default
|
||||
assert params[0].seed == 42
|
||||
assert all(param.output_kind == RequestOutputKind.CUMULATIVE for param in engine.default_sampling_params_list)
|
||||
assert next(msg for msg in ws.sent if msg["type"] == "response.text.done")["text"] == "answer"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_sampling_params_provided_stage_uses_constructor_defaults(run_sampling_session):
|
||||
ws, engine = await run_sampling_session({"sampling_params_list": [{"temperature": 0.2}]})
|
||||
|
||||
assert not [msg for msg in ws.sent if msg["type"] == "error"]
|
||||
engine.generate.assert_called_once()
|
||||
params = engine.generate.call_args.kwargs["sampling_params_list"]
|
||||
constructor_defaults = SamplingParams()
|
||||
assert params[0].temperature == 0.2
|
||||
for field in ("max_tokens", "top_p"):
|
||||
assert getattr(params[0], field) == getattr(constructor_defaults, field)
|
||||
assert getattr(params[0], field) != getattr(engine.default_sampling_params_list[0], field)
|
||||
assert len(params) == 3
|
||||
for param, default in zip(params[1:], engine.default_sampling_params_list[1:]):
|
||||
assert param.temperature == default.temperature
|
||||
assert param.max_tokens == default.max_tokens
|
||||
assert param.top_p == default.top_p
|
||||
assert param is not default
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("config_overrides", [{}, {"sampling_params_list": None}, {"sampling_params_list": []}])
|
||||
async def test_video_sampling_params_omitted_preserve_engine_defaults(run_sampling_session, config_overrides):
|
||||
ws, engine = await run_sampling_session(config_overrides)
|
||||
|
||||
assert not [msg for msg in ws.sent if msg["type"] == "error"]
|
||||
engine.generate.assert_called_once()
|
||||
assert engine.generate.call_args.kwargs.get("sampling_params_list") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("invalid_params", [{"temperature": -1}, {"max_tokens": 0}, {"unknown_sampling_option": 1}])
|
||||
async def test_video_sampling_params_invalid_do_not_reach_engine(run_sampling_session, invalid_params):
|
||||
ws, engine = await run_sampling_session({"sampling_params_list": [invalid_params]})
|
||||
|
||||
engine.generate.assert_not_called()
|
||||
assert any(msg["type"] == "error" for msg in ws.sent)
|
||||
assert ws.sent[-1]["type"] == "session.done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_frame_ack_reports_receiver_buffer_state():
|
||||
ws = MockWebSocket(
|
||||
@@ -244,6 +375,200 @@ async def test_video_frames_consumed_is_emitted_after_engine_uses_frame_prompt()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames,bad_index,expected_indices",
|
||||
[
|
||||
pytest.param(2, 4, [0], id="bad-last-no-refill"),
|
||||
pytest.param(1, 4, [], id="all-selected-frames-bad"),
|
||||
pytest.param(8, 4, [0, 1, 2, 3], id="below-sampling-limit"),
|
||||
pytest.param(2, 1, [0, 4], id="bad-frame-not-selected"),
|
||||
pytest.param(2, 0, [4], id="bad-first-frame"),
|
||||
],
|
||||
)
|
||||
async def test_video_frames_consumed_excludes_bad_frame_in_query_snapshot(
|
||||
monkeypatch, num_frames, bad_index, expected_indices
|
||||
):
|
||||
"""A query queued before bad-frame cleanup must report only its prompt images."""
|
||||
query_queued = asyncio.Event()
|
||||
decode_failed = asyncio.Event()
|
||||
release_decode = asyncio.Event()
|
||||
pong_blocked = asyncio.Event()
|
||||
release_pong = asyncio.Event()
|
||||
turn_done = asyncio.Event()
|
||||
original_to_thread = asyncio.to_thread
|
||||
bad_image = b"not-a-jpeg"
|
||||
frames = [_make_jpeg(r=index * 40) for index in range(5)]
|
||||
frames[bad_index] = bad_image
|
||||
engine_prompts = []
|
||||
|
||||
class ObservedQueue(asyncio.Queue):
|
||||
def put_nowait(self, item):
|
||||
super().put_nowait(item)
|
||||
if isinstance(item, dict):
|
||||
if item.get("type") == "video.query":
|
||||
query_queued.set()
|
||||
elif item.get("type") == "_internal.frame_decode_failed":
|
||||
decode_failed.set()
|
||||
|
||||
async def gated_to_thread(function, *args, **kwargs):
|
||||
if function is video_stream_base._decode_frame_bytes and args[0] == bad_image:
|
||||
await release_decode.wait()
|
||||
return await original_to_thread(function, *args, **kwargs)
|
||||
|
||||
class BackpressuredWebSocket(TimedWebSocket):
|
||||
async def send_json(self, data):
|
||||
await super().send_json(data)
|
||||
if data["type"] == "pong":
|
||||
pong_blocked.set()
|
||||
await release_pong.wait()
|
||||
elif data["type"] == "response.text.done":
|
||||
turn_done.set()
|
||||
|
||||
class OneOutputEngine:
|
||||
async def generate(self, **kwargs):
|
||||
engine_prompts.append(kwargs["prompt"])
|
||||
yield _text_result("visible")
|
||||
|
||||
class CapturingHandler(QwenOmniStreamingVideoHandler):
|
||||
async def _preprocess_to_engine_prompt(self, request):
|
||||
return {"messages": request.messages}
|
||||
|
||||
# Construct the client queue before observing the server's message queue.
|
||||
ws = BackpressuredWebSocket()
|
||||
monkeypatch.setattr(asyncio, "Queue", ObservedQueue)
|
||||
monkeypatch.setattr(asyncio, "to_thread", gated_to_thread)
|
||||
handler = CapturingHandler(chat_service=object(), engine_client=OneOutputEngine(), idle_timeout=5.0)
|
||||
ws.put(
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": "test",
|
||||
"modalities": ["text"],
|
||||
"num_frames": num_frames,
|
||||
"enable_frame_filter": False,
|
||||
}
|
||||
)
|
||||
for index, frame in enumerate(frames):
|
||||
ws.put({"type": "video.frame", "data": _b64(frame), "frame_id": f"frame-{index}", "pts_ms": index * 100})
|
||||
ws.put({"type": "ping"})
|
||||
ws.put({"type": "video.query", "text": "describe"})
|
||||
task = asyncio.create_task(handler.handle_session(ws))
|
||||
try:
|
||||
# Hold the processor at a WebSocket send. Queue the query first, then
|
||||
# finish the real failing decode so its cleanup lands behind the query.
|
||||
await asyncio.wait_for(pong_blocked.wait(), timeout=5.0)
|
||||
await asyncio.wait_for(query_queued.wait(), timeout=5.0)
|
||||
release_decode.set()
|
||||
await asyncio.wait_for(decode_failed.wait(), timeout=5.0)
|
||||
release_pong.set()
|
||||
await asyncio.wait_for(turn_done.wait(), timeout=5.0)
|
||||
ws.put({"type": "video.done"})
|
||||
await asyncio.wait_for(task, timeout=5.0)
|
||||
finally:
|
||||
release_decode.set()
|
||||
release_pong.set()
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
assert len(engine_prompts) == 1
|
||||
images = [part for part in engine_prompts[0]["messages"][-1]["content"] if part["type"].startswith("image_")]
|
||||
# Expected positions preserve sample-then-filter: do not refill a dropped
|
||||
# position or shift the remaining metadata when a frame fails to decode.
|
||||
assert len(images) == len(expected_indices)
|
||||
for image, index in zip(images, expected_indices):
|
||||
if image["type"] == "image_pil":
|
||||
assert image["image_pil"].tobytes() == Image.open(io.BytesIO(frames[index])).convert("RGB").tobytes()
|
||||
else:
|
||||
assert image["image_url"]["url"] == f"data:image/jpeg;base64,{_b64(frames[index])}"
|
||||
consumed_events = [message for message in ws.sent if message.get("type") == "video.frames.consumed"]
|
||||
assert len(consumed_events) == 1
|
||||
consumed = consumed_events[0]
|
||||
expected_ids = [f"frame-{index}" for index in expected_indices]
|
||||
assert consumed["frame_ids"] == expected_ids
|
||||
assert [frame["frame_id"] for frame in consumed["frames"]] == expected_ids
|
||||
assert [frame["pts_ms"] for frame in consumed["frames"]] == [index * 100 for index in expected_indices]
|
||||
assert consumed["latest_pts_ms"] == (expected_indices[-1] * 100 if expected_indices else None)
|
||||
assert {"type": "error", "message": "Frame decode failed"} in ws.sent
|
||||
assert ws.sent[-1]["type"] == "session.done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames,frame_ids,repeat_image,expected_indices",
|
||||
[
|
||||
pytest.param(2, ["f0", "f1", "f2", "f3", "f4"], False, [0, 4], id="stride-and-last"),
|
||||
pytest.param(1, ["f0", "f1", "f2"], False, [2], id="last-only"),
|
||||
pytest.param(8, ["f0", "f1", "f2"], False, [0, 1, 2], id="below-sampling-limit"),
|
||||
pytest.param(2, [None, "f1", None, "f3", None], False, [0, 4], id="unidentified-selected-frames"),
|
||||
pytest.param(2, [None, None, None], False, [0, 2], id="no-frame-ids"),
|
||||
pytest.param(2, ["same", "same", "same"], False, [0, 2], id="repeated-frame-id"),
|
||||
pytest.param(2, ["f0", "f1", "f2"], True, [0, 2], id="repeated-image"),
|
||||
],
|
||||
)
|
||||
async def test_video_frame_selection_preserves_uncached_frames(
|
||||
monkeypatch, num_frames, frame_ids, repeat_image, expected_indices
|
||||
):
|
||||
"""Prewarm misses still use image URLs, with metadata paired by buffer position."""
|
||||
engine_prompts = []
|
||||
pending_decode = asyncio.Event()
|
||||
|
||||
async def blocked_to_thread(*args, **kwargs):
|
||||
await pending_decode.wait()
|
||||
|
||||
monkeypatch.setattr(asyncio, "to_thread", blocked_to_thread)
|
||||
|
||||
class OneOutputEngine:
|
||||
async def generate(self, **kwargs):
|
||||
engine_prompts.append(kwargs["prompt"])
|
||||
yield _text_result("visible")
|
||||
|
||||
class CapturingHandler(QwenOmniStreamingVideoHandler):
|
||||
async def _preprocess_to_engine_prompt(self, request):
|
||||
return {"messages": request.messages}
|
||||
|
||||
frames = [_b64(_make_jpeg(r=0 if repeat_image else index * 40)) for index in range(len(frame_ids))]
|
||||
messages = [
|
||||
{
|
||||
"type": "session.config",
|
||||
"model": "test",
|
||||
"modalities": ["text"],
|
||||
"num_frames": num_frames,
|
||||
"enable_frame_filter": False,
|
||||
},
|
||||
*[
|
||||
{"type": "video.frame", "data": frame, "frame_id": frame_ids[index], "pts_ms": index * 100}
|
||||
for index, frame in enumerate(frames)
|
||||
],
|
||||
{"type": "video.query", "text": "describe"},
|
||||
{"type": "video.done"},
|
||||
]
|
||||
ws = MockWebSocket([json.dumps(message) for message in messages])
|
||||
handler = CapturingHandler(chat_service=object(), engine_client=OneOutputEngine())
|
||||
await asyncio.wait_for(handler.handle_session(ws), timeout=5.0)
|
||||
|
||||
assert not [message for message in ws.sent if message["type"] == "error"]
|
||||
assert len(engine_prompts) == 1
|
||||
content = engine_prompts[0]["messages"][-1]["content"]
|
||||
assert content == [
|
||||
*[
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{frames[index]}"}}
|
||||
for index in expected_indices
|
||||
],
|
||||
{"type": "text", "text": "describe"},
|
||||
]
|
||||
consumed_events = [message for message in ws.sent if message["type"] == "video.frames.consumed"]
|
||||
if not any(frame_ids):
|
||||
assert consumed_events == []
|
||||
else:
|
||||
assert len(consumed_events) == 1
|
||||
consumed = consumed_events[0]
|
||||
assert consumed["frame_ids"] == [frame_ids[index] for index in expected_indices if frame_ids[index] is not None]
|
||||
assert [frame["frame_id"] for frame in consumed["frames"]] == [frame_ids[index] for index in expected_indices]
|
||||
assert [frame["pts_ms"] for frame in consumed["frames"]] == [index * 100 for index in expected_indices]
|
||||
assert consumed["latest_pts_ms"] == expected_indices[-1] * 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_in_video_sets_mm_processor_kwargs():
|
||||
captured_requests = []
|
||||
@@ -455,6 +780,54 @@ async def test_async_chunk_mode_is_read_by_engine_path_at_runtime(monkeypatch):
|
||||
assert not any(m.get("type") == "response.text.delta" for m in ws_off.sent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_chunk_mode", ["on", "off"])
|
||||
async def test_audio_events_use_output_audio_names(monkeypatch, async_chunk_mode):
|
||||
class AudioEngine:
|
||||
def generate(self, **_kwargs):
|
||||
async def _gen():
|
||||
yield _audio_result([torch.zeros(1)])
|
||||
|
||||
return _gen()
|
||||
|
||||
class AudioHandler(QwenOmniStreamingVideoHandler):
|
||||
async def _preprocess_to_engine_prompt(self, request):
|
||||
return {"prompt": "x"}
|
||||
|
||||
monkeypatch.setenv("VLLM_VIDEO_ASYNC_CHUNK", async_chunk_mode)
|
||||
monkeypatch.setattr(
|
||||
OmniStreamingVideoHandler,
|
||||
"_encode_audio_wav_b64",
|
||||
staticmethod(lambda _audio: "audio-b64"),
|
||||
)
|
||||
|
||||
ws = MockWebSocket()
|
||||
handler = AudioHandler(chat_service=object(), engine_client=AudioEngine())
|
||||
config = StreamingVideoSessionConfig(model="test", modalities=["text", "audio"])
|
||||
|
||||
await handler._process_query_engine(
|
||||
ws,
|
||||
config,
|
||||
[_b64(_make_jpeg())],
|
||||
bytearray(),
|
||||
[],
|
||||
"describe",
|
||||
"req-audio-events",
|
||||
asyncio.Event(),
|
||||
{},
|
||||
)
|
||||
|
||||
audio_events = [message for message in ws.sent if "audio" in message.get("type", "")]
|
||||
assert audio_events == [
|
||||
{
|
||||
"type": "response.output_audio.delta",
|
||||
"data": "audio-b64",
|
||||
"format": "wav",
|
||||
},
|
||||
{"type": "response.output_audio.done"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_without_engine_client_sends_error():
|
||||
ws = MockWebSocket()
|
||||
@@ -513,6 +886,88 @@ async def test_new_query_cancels_in_flight_query():
|
||||
assert "session.done" in ws.sent_types()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("delay_abort", [False, True], ids=["immediate-ack", "delayed-ack"])
|
||||
async def test_interrupted_queries_wait_for_abort_without_fixed_delay(monkeypatch, delay_abort):
|
||||
started: asyncio.Queue[str] = asyncio.Queue()
|
||||
abort_started: asyncio.Queue[str] = asyncio.Queue()
|
||||
allow_abort = asyncio.Event()
|
||||
if not delay_abort:
|
||||
allow_abort.set()
|
||||
request_ids: list[str] = []
|
||||
closed: list[str] = []
|
||||
aborted: list[str] = []
|
||||
delays: list[float] = []
|
||||
original_sleep = asyncio.sleep
|
||||
|
||||
async def record_sleep(delay, result=None):
|
||||
# Observe requested delays without a machine-dependent latency limit.
|
||||
delays.append(delay)
|
||||
return await original_sleep(0, result)
|
||||
|
||||
monkeypatch.setattr(video_stream_base.asyncio, "sleep", record_sleep)
|
||||
|
||||
class BlockingEngine:
|
||||
async def generate(self, *, request_id, **kwargs):
|
||||
if request_ids:
|
||||
assert aborted[-1] == request_ids[-1]
|
||||
request_ids.append(request_id)
|
||||
started.put_nowait(request_id)
|
||||
try:
|
||||
if len(request_ids) <= 3:
|
||||
yield _text_result("partial")
|
||||
await asyncio.Event().wait()
|
||||
else:
|
||||
yield _text_result("final")
|
||||
finally:
|
||||
closed.append(request_id)
|
||||
|
||||
async def abort(self, request_id):
|
||||
assert request_id in closed
|
||||
abort_started.put_nowait(request_id)
|
||||
await allow_abort.wait()
|
||||
aborted.append(request_id)
|
||||
|
||||
class PreprocessedHandler(QwenOmniStreamingVideoHandler):
|
||||
async def _preprocess_to_engine_prompt(self, request):
|
||||
return {"prompt_token_ids": [1]}
|
||||
|
||||
ws = TimedWebSocket()
|
||||
handler = PreprocessedHandler(chat_service=object(), engine_client=BlockingEngine(), idle_timeout=5.0)
|
||||
task = asyncio.create_task(handler.handle_session(ws))
|
||||
try:
|
||||
ws.put({"type": "session.config", "modalities": ["text"], "enable_frame_filter": False})
|
||||
ws.put({"type": "video.frame", "data": _b64(_make_jpeg())})
|
||||
ws.put({"type": "video.query", "text": "first"})
|
||||
previous_id = await asyncio.wait_for(started.get(), timeout=2.0)
|
||||
|
||||
for _ in range(3):
|
||||
ws.put({"type": "video.query", "text": "interrupt"})
|
||||
assert await asyncio.wait_for(abort_started.get(), timeout=2.0) == previous_id
|
||||
if delay_abort:
|
||||
assert previous_id not in aborted
|
||||
assert started.empty()
|
||||
allow_abort.set()
|
||||
previous_id = await asyncio.wait_for(started.get(), timeout=2.0)
|
||||
if delay_abort:
|
||||
allow_abort.clear()
|
||||
|
||||
ws.put({"type": "video.done"})
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
assert aborted == request_ids[:-1]
|
||||
assert len(set(request_ids)) == 4
|
||||
assert [msg["text"] for msg in ws.sent if msg["type"] == "response.text.done"] == ["final"]
|
||||
assert "error" not in ws.sent_types()
|
||||
assert "session.done" in ws.sent_types()
|
||||
assert not [delay for delay in delays if delay > 0], "Restart must not add a timed grace period after abort"
|
||||
finally:
|
||||
allow_abort.set()
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_done_waits_for_in_flight_query():
|
||||
query_started = asyncio.Event()
|
||||
@@ -588,6 +1043,68 @@ async def test_frame_prewarm_does_not_block_following_query(monkeypatch):
|
||||
assert "session.done" in ws.sent_types()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("max_frames", [1, 2], ids=["evicted", "retained"])
|
||||
async def test_frame_prewarm_only_keeps_retained_images(monkeypatch, max_frames):
|
||||
frame_a = _make_jpeg(255, 0, 0)
|
||||
frame_b = _make_jpeg(0, 255, 0)
|
||||
decode_started = asyncio.Event()
|
||||
release_decode = asyncio.Event()
|
||||
frame_b_accepted = asyncio.Event()
|
||||
decoded_images: dict[bytes, weakref.ReferenceType[Image.Image]] = {}
|
||||
prewarm_task: asyncio.Task | None = None
|
||||
original_to_thread = asyncio.to_thread
|
||||
|
||||
async def controlled_to_thread(function, *args, **kwargs):
|
||||
nonlocal prewarm_task
|
||||
if function is video_stream_base._decode_frame_bytes:
|
||||
if args[0] == frame_a:
|
||||
prewarm_task = asyncio.current_task()
|
||||
decode_started.set()
|
||||
await release_decode.wait()
|
||||
image = await original_to_thread(function, *args, **kwargs)
|
||||
decoded_images[args[0]] = weakref.ref(image)
|
||||
return image
|
||||
return await original_to_thread(function, *args, **kwargs)
|
||||
|
||||
class AckWebSocket(TimedWebSocket):
|
||||
async def send_json(self, data):
|
||||
await super().send_json(data)
|
||||
if data.get("type") == "video.frame.ack" and data.get("frame_id") == "B":
|
||||
frame_b_accepted.set()
|
||||
|
||||
monkeypatch.setattr(video_stream_base.asyncio, "to_thread", controlled_to_thread)
|
||||
ws = AckWebSocket()
|
||||
handler = QwenOmniStreamingVideoHandler(chat_service=object(), idle_timeout=5.0)
|
||||
session_task = asyncio.create_task(handler.handle_session(ws))
|
||||
ws.put({"type": "session.config", "max_frames": max_frames, "enable_frame_filter": False})
|
||||
try:
|
||||
ws.put({"type": "video.frame", "frame_id": "A", "data": _b64(frame_a)})
|
||||
await asyncio.wait_for(decode_started.wait(), timeout=5.0)
|
||||
ws.put({"type": "video.frame", "frame_id": "B", "data": _b64(frame_b)})
|
||||
await asyncio.wait_for(frame_b_accepted.wait(), timeout=5.0)
|
||||
ack = next(message for message in ws.sent if message.get("frame_id") == "B")
|
||||
assert ack["accepted"] is True
|
||||
assert ack.get("dropped_frame_id") == ("A" if max_frames == 1 else None)
|
||||
|
||||
# Finish A's real decode only after B has either evicted A or joined it.
|
||||
release_decode.set()
|
||||
assert prewarm_task is not None
|
||||
await asyncio.wait_for(asyncio.shield(prewarm_task), timeout=5.0)
|
||||
gc.collect()
|
||||
assert not session_task.done()
|
||||
# A finished task must not keep an evicted PIL image alive for the session.
|
||||
assert (decoded_images[frame_a]() is not None) == (max_frames == 2)
|
||||
finally:
|
||||
release_decode.set()
|
||||
ws.put({"type": "video.done"})
|
||||
await asyncio.wait_for(session_task, timeout=5.0)
|
||||
|
||||
gc.collect()
|
||||
assert decoded_images[frame_a]() is None
|
||||
assert not any(message.get("type") == "error" for message in ws.sent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_cannot_send_internal_frame_decode_failed_message():
|
||||
captured_frames: list[list[str]] = []
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import (
|
||||
detect_tts_model_type,
|
||||
resolve_adapter,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import resolve_stage_model_path
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.indextts2 import (
|
||||
@@ -65,10 +66,11 @@ EXPECTED_MODEL_TYPES = {
|
||||
"higgs_audio_v2",
|
||||
"higgs_audio_v3",
|
||||
"glm_tts",
|
||||
"breeze_tts_2",
|
||||
"step_audio2",
|
||||
"indextts2",
|
||||
"indextts2_5",
|
||||
"dots_tts",
|
||||
"gepard",
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +99,31 @@ def test_resolve_unknown_returns_none():
|
||||
assert resolve_adapter(None) is None
|
||||
|
||||
|
||||
def test_stage_model_path_prefers_typed_override():
|
||||
engine_client = SimpleNamespace(
|
||||
stage_configs=[
|
||||
SimpleNamespace(engine_args=SimpleNamespace(model="legacy-stage-model")),
|
||||
SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="typed-stage-model"),
|
||||
),
|
||||
],
|
||||
model="served-model",
|
||||
)
|
||||
|
||||
assert resolve_stage_model_path(engine_client) == "typed-stage-model"
|
||||
|
||||
|
||||
def test_stage_model_path_falls_back_to_legacy_then_served_model():
|
||||
legacy_client = SimpleNamespace(
|
||||
stage_configs=[SimpleNamespace(engine_args=SimpleNamespace(model="legacy-stage-model"))],
|
||||
model="served-model",
|
||||
)
|
||||
served_client = SimpleNamespace(stage_configs=[SimpleNamespace()], model="served-model")
|
||||
|
||||
assert resolve_stage_model_path(legacy_client) == "legacy-stage-model"
|
||||
assert resolve_stage_model_path(served_client) == "served-model"
|
||||
|
||||
|
||||
def test_voxcpm2_resolves():
|
||||
"""VoxCPM2 (the served ``latent_generator`` model) resolves cleanly.
|
||||
|
||||
@@ -142,6 +169,17 @@ def _build_moss_tts_request(adapter_cls, mocker, *, request_seed):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("adapter_cls", "expect_accumulate"),
|
||||
[(MossTTSAdapter, False), (MossTTSNanoAdapter, True)],
|
||||
)
|
||||
def test_moss_tts_accumulate_nonstreaming_follows_adapter_flag(adapter_cls, expect_accumulate, mocker):
|
||||
prepared = _build_moss_tts_request(adapter_cls, mocker, request_seed=7)
|
||||
|
||||
assert adapter_cls.accumulate_nonstreaming is expect_accumulate
|
||||
assert prepared.output_policy.accumulate_nonstreaming is expect_accumulate
|
||||
|
||||
|
||||
# Full-family coverage pins the adapter contract; only Nano consumes this seed end to end today.
|
||||
@pytest.mark.parametrize("adapter_cls", [MossTTSAdapter, MossTTSNanoAdapter])
|
||||
@pytest.mark.parametrize("request_seed", [0, 1234])
|
||||
@@ -216,6 +254,115 @@ def test_moss_reference_transcript_mode(variant, ref_text, mode, mocker):
|
||||
assert request.input == "Target."
|
||||
|
||||
|
||||
def _moss_adapter_with_stage_configs(stage_configs, mocker):
|
||||
engine_client = SimpleNamespace(
|
||||
model_config=SimpleNamespace(model="OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5"),
|
||||
stage_configs=stage_configs,
|
||||
)
|
||||
return MossTTSAdapter(SpeechServingContext(server=mocker.Mock(), engine_client=engine_client))
|
||||
|
||||
|
||||
def _moss_cuda_available(mocker, device_count=8):
|
||||
mocker.patch("torch.cuda.is_available", return_value=True)
|
||||
mocker.patch("torch.accelerator.device_count", return_value=device_count)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stage_configs,expected",
|
||||
[
|
||||
# Follows the code2wav stage's first device.
|
||||
(
|
||||
[
|
||||
SimpleNamespace(model_stage="moss_tts_local", runtime_config=SimpleNamespace(devices="0")),
|
||||
SimpleNamespace(model_stage="moss_tts_local_codec", runtime_config=SimpleNamespace(devices="3")),
|
||||
],
|
||||
"cuda:3",
|
||||
),
|
||||
# No codec-named stage: anchors on the last stage of the pipeline.
|
||||
(
|
||||
[
|
||||
SimpleNamespace(model_stage="talker", runtime_config=SimpleNamespace(devices="1")),
|
||||
SimpleNamespace(model_stage="decoder", runtime_config=SimpleNamespace(devices="2")),
|
||||
],
|
||||
"cuda:2",
|
||||
),
|
||||
# Multi-device codec stage pins replica 0 on the first device.
|
||||
(
|
||||
[
|
||||
SimpleNamespace(model_stage="moss_tts_local_codec", runtime_config=SimpleNamespace(devices="5,6")),
|
||||
],
|
||||
"cuda:5",
|
||||
),
|
||||
# No explicit pinning anywhere: stage worker defaults to cuda:0.
|
||||
(
|
||||
[SimpleNamespace(model_stage="moss_tts_local_codec", runtime_config=SimpleNamespace(devices=None))],
|
||||
"cuda:0",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_moss_ref_encoder_device_follows_codec_stage(stage_configs, expected, mocker):
|
||||
import torch
|
||||
|
||||
_moss_cuda_available(mocker)
|
||||
adapter = _moss_adapter_with_stage_configs(stage_configs, mocker)
|
||||
assert adapter._resolve_ref_encoder_device() == torch.device(expected)
|
||||
|
||||
|
||||
def test_moss_ref_encoder_device_accepts_dict_runtime(mocker):
|
||||
import torch
|
||||
|
||||
_moss_cuda_available(mocker)
|
||||
adapter = _moss_adapter_with_stage_configs(
|
||||
[SimpleNamespace(model_stage="moss_tts_local_codec", runtime={"devices": "2"})],
|
||||
mocker,
|
||||
)
|
||||
assert adapter._resolve_ref_encoder_device() == torch.device("cuda:2")
|
||||
|
||||
|
||||
def test_moss_ref_encoder_device_cpu_when_cuda_unavailable(mocker):
|
||||
import torch
|
||||
|
||||
mocker.patch("torch.cuda.is_available", return_value=False)
|
||||
adapter = _moss_adapter_with_stage_configs(
|
||||
[SimpleNamespace(model_stage="moss_tts_local_codec", runtime_config=SimpleNamespace(devices="3"))],
|
||||
mocker,
|
||||
)
|
||||
assert adapter._resolve_ref_encoder_device() == torch.device("cpu")
|
||||
|
||||
|
||||
def test_moss_ref_encoder_device_cpu_when_index_out_of_range(mocker):
|
||||
import torch
|
||||
|
||||
_moss_cuda_available(mocker, device_count=2)
|
||||
adapter = _moss_adapter_with_stage_configs(
|
||||
[SimpleNamespace(model_stage="moss_tts_local_codec", runtime_config=SimpleNamespace(devices="5"))],
|
||||
mocker,
|
||||
)
|
||||
assert adapter._resolve_ref_encoder_device() == torch.device("cpu")
|
||||
|
||||
|
||||
def test_moss_get_processor_places_audio_tokenizer_on_codec_stage_device(mocker):
|
||||
import torch
|
||||
|
||||
_moss_cuda_available(mocker)
|
||||
adapter = _moss_adapter_with_stage_configs(
|
||||
[SimpleNamespace(model_stage="moss_tts_local_codec", runtime_config=SimpleNamespace(devices="3"))],
|
||||
mocker,
|
||||
)
|
||||
audio_tokenizer = mocker.Mock()
|
||||
audio_tokenizer.to.return_value = audio_tokenizer
|
||||
processor = SimpleNamespace(audio_tokenizer=audio_tokenizer)
|
||||
from_pretrained = mocker.patch("transformers.AutoProcessor.from_pretrained", return_value=processor)
|
||||
|
||||
assert adapter._get_moss_processor() is processor
|
||||
audio_tokenizer.to.assert_called_once_with(torch.device("cuda", 3))
|
||||
audio_tokenizer.eval.assert_called_once_with()
|
||||
# Lazy cache: no re-load / re-placement on the second call.
|
||||
assert adapter._get_moss_processor() is processor
|
||||
from_pretrained.assert_called_once()
|
||||
audio_tokenizer.to.assert_called_once()
|
||||
|
||||
|
||||
def test_qwen3_tts_metadata():
|
||||
assert Qwen3TTSAdapter.backend == "ar"
|
||||
assert issubclass(Qwen3TTSAdapter, ARTTSAdapter)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""TTS model detection: registry-driven stage -> model-type resolution.
|
||||
|
||||
``serving_speech.py`` used to carry a hand-written 20-branch ladder mapping
|
||||
@@ -138,6 +139,7 @@ _PIPELINE_STAGES = [
|
||||
"audio_tokenizer",
|
||||
"audio_vae",
|
||||
"aura",
|
||||
"breeze_tts_2",
|
||||
"code2wav",
|
||||
"cosyvoice3_code2wav",
|
||||
"cosyvoice3_talker",
|
||||
@@ -146,6 +148,7 @@ _PIPELINE_STAGES = [
|
||||
"dit",
|
||||
"fish_speech_slow_ar",
|
||||
"fused_thinker_talker",
|
||||
"gepard",
|
||||
"glm_tts",
|
||||
"glm_tts_dit",
|
||||
"higgs_audio_v2",
|
||||
@@ -184,6 +187,7 @@ _STAGES = [*_PIPELINE_STAGES, None, "vae", "not_a_real_stage"]
|
||||
_ARCHS = [
|
||||
None,
|
||||
"VoxCPM2TalkerForConditionalGeneration",
|
||||
"BreezeForConditionalGeneration",
|
||||
"MingTTSForConditionalGeneration",
|
||||
"CovoAudioForConditionalGeneration",
|
||||
"MyCovoAudioThing",
|
||||
@@ -259,7 +263,6 @@ def test_arch_matching_is_a_fallback_not_an_override():
|
||||
|
||||
def test_shared_latent_generator_resolves_by_architecture_priority():
|
||||
assert detect_tts_model_type("latent_generator", "VoxCPM2TalkerForConditionalGeneration") == "voxcpm2"
|
||||
assert detect_tts_model_type("latent_generator", "DotsTTSForConditionalGeneration") == "dots_tts"
|
||||
|
||||
|
||||
def test_stage_keys_cover_legacy_stage_set():
|
||||
|
||||
@@ -11,6 +11,7 @@ import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import CancelledError as FutureCancelledError
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -25,6 +26,7 @@ from PIL import Image
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm import envs
|
||||
|
||||
from vllm_omni.diffusion.data import DIFFUSION_REQUEST_LIFECYCLE_KEY, DIFFUSION_REQUEST_STARTED
|
||||
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
|
||||
from vllm_omni.entrypoints.openai import api_server, video_api_utils
|
||||
from vllm_omni.entrypoints.openai.api_server import router
|
||||
@@ -58,6 +60,7 @@ class MockVideoResult:
|
||||
multimodal_output=None,
|
||||
stage_durations=None,
|
||||
peak_memory_mb=0.0,
|
||||
custom_output=None,
|
||||
):
|
||||
self.multimodal_output = dict(multimodal_output or {"video": videos})
|
||||
if audios is not None:
|
||||
@@ -66,6 +69,7 @@ class MockVideoResult:
|
||||
self.multimodal_output["audio_sample_rate"] = sample_rate
|
||||
self.stage_durations = stage_durations or {}
|
||||
self.peak_memory_mb = peak_memory_mb
|
||||
self.custom_output = custom_output or {}
|
||||
|
||||
|
||||
class FakeAsyncOmni:
|
||||
@@ -81,6 +85,7 @@ class FakeAsyncOmni:
|
||||
self.model_class_name = "WanPipeline"
|
||||
self.captured_prompt = None
|
||||
self.captured_reference_video_bytes = None
|
||||
self.captured_latent_edit_source_bytes = {}
|
||||
self.captured_control_reference_bytes = {}
|
||||
self.captured_sampling_params_list = None
|
||||
|
||||
@@ -101,10 +106,22 @@ class FakeAsyncOmni:
|
||||
and all(isinstance(item, str) for item in reference_videos)
|
||||
):
|
||||
self.captured_reference_video_bytes = [Path(item).read_bytes() for item in reference_videos]
|
||||
for field_name in ("source_video", "source_audio"):
|
||||
source_path = prompt.get("multi_modal_data", {}).get(field_name)
|
||||
if isinstance(source_path, str):
|
||||
self.captured_latent_edit_source_bytes[field_name] = Path(source_path).read_bytes()
|
||||
num_outputs = sampling_params_list[0].num_outputs_per_prompt
|
||||
if sampling_params_list[0].emit_request_lifecycle:
|
||||
yield MockVideoResult(
|
||||
[],
|
||||
custom_output={DIFFUSION_REQUEST_LIFECYCLE_KEY: DIFFUSION_REQUEST_STARTED},
|
||||
)
|
||||
videos = [object() for _ in range(num_outputs)]
|
||||
yield MockVideoResult(videos)
|
||||
|
||||
async def abort(self, request_id, *, timeout=None):
|
||||
del request_id, timeout
|
||||
|
||||
|
||||
def test_raw_and_base64_encoders_receive_persistent_converter(mocker: MockerFixture):
|
||||
engine = FakeAsyncOmni()
|
||||
@@ -134,6 +151,60 @@ def test_raw_and_base64_encoders_receive_persistent_converter(mocker: MockerFixt
|
||||
handler.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_frames", [0, -1, True, 1.5, "17", None])
|
||||
def test_preencode_rejects_invalid_batch_frames_before_generation(batch_frames):
|
||||
engine = FakeAsyncOmni()
|
||||
handler = OmniOpenAIServingVideo.for_diffusion(engine, model_name="test-model")
|
||||
request = VideoGenerationRequest(
|
||||
prompt="test", extra_params={"preencode_mp4": True, "preencode_batch_frames": batch_frames}
|
||||
)
|
||||
try:
|
||||
with pytest.raises(HTTPException, match="preencode_batch_frames") as exc:
|
||||
asyncio.run(handler.generate_video_bytes(request, "invalid-batch"))
|
||||
assert exc.value.status_code == 400
|
||||
assert engine.captured_prompt is None
|
||||
finally:
|
||||
handler.shutdown()
|
||||
|
||||
|
||||
def test_preencoded_video_bytes_preserve_metadata(mocker: MockerFixture):
|
||||
from vllm_omni.entrypoints.openai.serving_video import VideoGenerationArtifacts
|
||||
|
||||
handler = OmniOpenAIServingVideo.for_diffusion(FakeAsyncOmni(), model_name="test-model")
|
||||
# Resolved frame count differs from anything the request asked for, so the
|
||||
# metadata has to come from the encoded stream rather than request defaults.
|
||||
preencoded = _make_test_video_bytes((32, 24), num_frames=7)
|
||||
artifacts = VideoGenerationArtifacts(
|
||||
videos=[preencoded],
|
||||
audios=[None],
|
||||
actions=[None],
|
||||
audio_sample_rate=24000,
|
||||
output_fps=24.0,
|
||||
stage_durations={"decode": 0.5},
|
||||
peak_memory_mb=123.0,
|
||||
metrics={"generation_time": 1.25},
|
||||
)
|
||||
mocker.patch.object(handler, "_run_and_extract", return_value=artifacts)
|
||||
encoder = mocker.patch("vllm_omni.entrypoints.openai.serving_video._encode_video_bytes")
|
||||
try:
|
||||
result = asyncio.run(handler.generate_video_bytes(VideoGenerationRequest(prompt="test"), "preencoded"))
|
||||
assert result == (
|
||||
preencoded,
|
||||
{"decode": 0.5},
|
||||
123.0,
|
||||
None,
|
||||
{
|
||||
"fps": 24.0,
|
||||
"num_frames": 7,
|
||||
"duration_s": 7 / 24.0,
|
||||
"metrics": {"generation_time": 1.25},
|
||||
},
|
||||
)
|
||||
encoder.assert_not_called()
|
||||
finally:
|
||||
handler.shutdown()
|
||||
|
||||
|
||||
def test_resolve_diffusion_od_config_falls_back_to_attribute():
|
||||
od_config = SimpleNamespace(model_class_name="WanPipeline")
|
||||
handler = OmniOpenAIServingVideo.for_diffusion(
|
||||
@@ -159,6 +230,10 @@ def test_resolve_diffusion_od_config_prefers_getter_over_attribute():
|
||||
|
||||
|
||||
class BlockingVideoHandler:
|
||||
supports_mixed_reference_inputs = False
|
||||
supports_latent_mask_editing = False
|
||||
supported_control_upload_types = frozenset()
|
||||
|
||||
def __init__(self):
|
||||
self.model_name = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||
self.stage_configs = None
|
||||
@@ -170,9 +245,19 @@ class BlockingVideoHandler:
|
||||
self.stage_configs = stage_configs
|
||||
|
||||
async def generate_video_bytes(
|
||||
self, request, reference_id, *, reference_image=None, reference_video=None, reference_audio=None
|
||||
self,
|
||||
request,
|
||||
reference_id,
|
||||
*,
|
||||
reference_image=None,
|
||||
reference_video=None,
|
||||
reference_audio=None,
|
||||
on_started=None,
|
||||
latent_edit_input=None,
|
||||
):
|
||||
del request, reference_id, reference_image, reference_video, reference_audio
|
||||
del request, reference_id, reference_image, reference_video, reference_audio, latent_edit_input
|
||||
if on_started is not None:
|
||||
await on_started()
|
||||
self.started.set()
|
||||
try:
|
||||
await asyncio.Future()
|
||||
@@ -180,6 +265,106 @@ class BlockingVideoHandler:
|
||||
self.cancelled.set()
|
||||
raise
|
||||
|
||||
async def abort_request(self, request_id):
|
||||
del request_id
|
||||
|
||||
|
||||
class HangingAbortHandler(BlockingVideoHandler):
|
||||
"""Engine abort never returns; DELETE must time out and still cancel."""
|
||||
|
||||
async def abort_request(self, request_id):
|
||||
del request_id
|
||||
await asyncio.sleep(30)
|
||||
|
||||
|
||||
class CompletingDuringAbortHandler(BlockingVideoHandler):
|
||||
"""Finishes and persists output while DELETE is still awaiting abort."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._finish = asyncio.Event()
|
||||
|
||||
async def generate_video_bytes(
|
||||
self,
|
||||
request,
|
||||
reference_id,
|
||||
*,
|
||||
reference_image=None,
|
||||
reference_video=None,
|
||||
reference_audio=None,
|
||||
on_started=None,
|
||||
latent_edit_input=None,
|
||||
):
|
||||
del request, reference_image, reference_video, reference_audio, latent_edit_input
|
||||
if on_started is not None:
|
||||
await on_started()
|
||||
self.started.set()
|
||||
await self._finish.wait()
|
||||
return b"late-complete", {}, 0.0, None
|
||||
|
||||
async def abort_request(self, request_id):
|
||||
self._finish.set()
|
||||
deadline = time.monotonic() + 2.0
|
||||
while time.monotonic() < deadline:
|
||||
job = await api_server.VIDEO_STORE.get(request_id)
|
||||
if job is not None and job.status is VideoGenerationStatus.COMPLETED and job.file_name is not None:
|
||||
return
|
||||
await asyncio.sleep(0.01)
|
||||
raise RuntimeError(f"video job {request_id} did not complete during abort")
|
||||
|
||||
|
||||
class SchedulerQueuedVideoHandler(BlockingVideoHandler):
|
||||
"""Parks after engine submission but before scheduler admission."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.admit = threading.Event()
|
||||
self.in_progress = threading.Event()
|
||||
|
||||
async def generate_video_bytes(
|
||||
self,
|
||||
request,
|
||||
reference_id,
|
||||
*,
|
||||
reference_image=None,
|
||||
reference_video=None,
|
||||
reference_audio=None,
|
||||
on_started=None,
|
||||
latent_edit_input=None,
|
||||
):
|
||||
del request, reference_id, reference_image, reference_video, reference_audio, latent_edit_input
|
||||
self.started.set()
|
||||
try:
|
||||
while not self.admit.is_set():
|
||||
await asyncio.sleep(0.01)
|
||||
if on_started is not None:
|
||||
await on_started()
|
||||
self.in_progress.set()
|
||||
await asyncio.Future()
|
||||
except asyncio.CancelledError:
|
||||
self.cancelled.set()
|
||||
raise
|
||||
|
||||
|
||||
class AbortTrackingOmni(FakeAsyncOmni):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.aborted: list[str] = []
|
||||
self.entered = threading.Event()
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params_list):
|
||||
del prompt, request_id, sampling_params_list
|
||||
self.entered.set()
|
||||
yield MockVideoResult(
|
||||
[],
|
||||
custom_output={DIFFUSION_REQUEST_LIFECYCLE_KEY: DIFFUSION_REQUEST_STARTED},
|
||||
)
|
||||
await asyncio.Future()
|
||||
|
||||
async def abort(self, request_id, *, timeout=None):
|
||||
assert timeout is not None
|
||||
self.aborted.append(request_id)
|
||||
|
||||
|
||||
class FakeServerSocket:
|
||||
def __init__(self):
|
||||
@@ -291,6 +476,7 @@ async def test_server_worker_keeps_engine_alive_until_http_shutdown(monkeypatch)
|
||||
@pytest.fixture
|
||||
def test_client():
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 1
|
||||
app.include_router(router)
|
||||
app.state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion(
|
||||
diffusion_engine=FakeAsyncOmni(),
|
||||
@@ -323,6 +509,11 @@ def _make_test_video_bytes(size=(32, 24), num_frames=3) -> bytes:
|
||||
return mux_video_audio_bytes(frames, fps=8, video_codec_options={"preset": "ultrafast", "threads": "0"})
|
||||
|
||||
|
||||
def _mask_file(value) -> tuple[str, bytes, str]:
|
||||
payload = value if isinstance(value, str) else json.dumps(value)
|
||||
return "mask.json", payload.encode(), "application/json"
|
||||
|
||||
|
||||
def test_mux_video_audio_marks_aac_priming_timestamp():
|
||||
frames = np.zeros((2, 16, 16, 3), dtype=np.uint8)
|
||||
audio = np.zeros((2, 2048), dtype=np.float32)
|
||||
@@ -538,12 +729,12 @@ def test_i2v_video_generation_resizes_input_to_requested_dimensions(test_client,
|
||||
|
||||
def test_i2v_resize_policy_can_defer_to_pipeline(monkeypatch):
|
||||
engine = FakeAsyncOmni()
|
||||
engine.get_diffusion_od_config = lambda: SimpleNamespace(
|
||||
engine.get_diffusion_od_config = lambda: SimpleNamespace( # type: ignore[method-assign]
|
||||
model="org/model",
|
||||
model_class_name="ExamplePipeline",
|
||||
revision="pinned-revision",
|
||||
)
|
||||
captured = {}
|
||||
captured: dict[str, str | None] = {}
|
||||
|
||||
def fake_policy(model_class_name, *, model, revision=None):
|
||||
captured.update(
|
||||
@@ -571,6 +762,7 @@ def test_i2v_resize_policy_can_defer_to_pipeline(monkeypatch):
|
||||
)
|
||||
)
|
||||
|
||||
assert engine.captured_prompt is not None
|
||||
input_image = engine.captured_prompt["multi_modal_data"]["image"]
|
||||
assert isinstance(input_image, Image.Image)
|
||||
assert input_image.size == (48, 32)
|
||||
@@ -581,6 +773,32 @@ def test_i2v_resize_policy_can_defer_to_pipeline(monkeypatch):
|
||||
}
|
||||
|
||||
|
||||
def test_i2v_minimax_h3_preserves_reference_geometry():
|
||||
engine = FakeAsyncOmni()
|
||||
engine.model_class_name = "MiniMaxH3Pipeline"
|
||||
handler = OmniOpenAIServingVideo.for_diffusion(
|
||||
diffusion_engine=engine,
|
||||
model_name="MiniMaxAI/MiniMax-H3",
|
||||
)
|
||||
image = Image.new("RGB", (48, 32))
|
||||
|
||||
asyncio.run(
|
||||
handler._run_and_extract(
|
||||
VideoGenerationRequest(prompt="A bear playing with yarn.", width=96, height=64),
|
||||
"minimax-h3-reference-geometry",
|
||||
reference_image=ReferenceImage(image),
|
||||
)
|
||||
)
|
||||
|
||||
assert engine.captured_prompt is not None
|
||||
assert engine.captured_sampling_params_list is not None
|
||||
input_image = engine.captured_prompt["multi_modal_data"]["image"]
|
||||
assert isinstance(input_image, Image.Image)
|
||||
assert input_image.size == (48, 32)
|
||||
sampling_params = engine.captured_sampling_params_list[0]
|
||||
assert (sampling_params.width, sampling_params.height) == (96, 64)
|
||||
|
||||
|
||||
def test_i2v_extra_params_dimensions_preserve_input_image_geometry(test_client, mocker: MockerFixture):
|
||||
image_bytes = _make_test_image_bytes((48, 48))
|
||||
mocker.patch(
|
||||
@@ -638,6 +856,7 @@ def test_video_generation_bridges_request_fields(generation_request, expected_nu
|
||||
|
||||
asyncio.run(handler._run_and_extract(generation_request, "field-bridge"))
|
||||
|
||||
assert engine.captured_sampling_params_list is not None
|
||||
sampling = engine.captured_sampling_params_list[0]
|
||||
# Top-level ``seconds`` bridges into extra_args["duration"]; num_frames is
|
||||
# passed through (or derived as seconds x fps when omitted). No private
|
||||
@@ -934,6 +1153,251 @@ def test_mixed_reference_capability_uses_model_metadata_when_config_defaults_fal
|
||||
assert handler.supports_mixed_reference_inputs
|
||||
|
||||
|
||||
def test_typed_stage_drives_video_capability_checks():
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata
|
||||
|
||||
minimax_stage = StageConfigFactory.create_typed_default_diffusion(
|
||||
"minimax-h3",
|
||||
{"model_class_name": "MiniMaxH3Pipeline"},
|
||||
).stage_configs[0]
|
||||
minimax_handler = OmniOpenAIServingVideo.for_diffusion(
|
||||
SimpleNamespace(od_config=SimpleNamespace(model_class_name=None)),
|
||||
model_name="minimax-h3",
|
||||
stage_configs=[minimax_stage],
|
||||
)
|
||||
assert minimax_handler.supports_mixed_reference_inputs
|
||||
assert minimax_handler.supports_latent_mask_editing
|
||||
assert get_diffusion_model_metadata("MiniMaxH3ModularPipeline").supports_latent_mask_editing
|
||||
|
||||
cosmos_stage = StageConfigFactory.create_typed_default_diffusion(
|
||||
"cosmos3",
|
||||
{"model_class_name": "Cosmos3OmniDiffusersPipeline"},
|
||||
).stage_configs[0]
|
||||
cosmos_handler = OmniOpenAIServingVideo.for_diffusion(
|
||||
SimpleNamespace(od_config=SimpleNamespace(model_class_name=None)),
|
||||
model_name="cosmos3",
|
||||
stage_configs=[cosmos_stage],
|
||||
)
|
||||
assert cosmos_handler.supported_control_upload_types == frozenset({"edge", "blur", "depth", "seg", "wsm"})
|
||||
assert not cosmos_handler.supports_latent_mask_editing
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["/v1/videos", "/v1/videos/sync"])
|
||||
def test_h3_latent_edit_uploads_reach_generation_and_are_cleaned(endpoint, test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
engine.model_class_name = "MiniMaxH3Pipeline"
|
||||
video_mask = [[[0.0, 1.0], [0.5, 1.0]]]
|
||||
audio_mask = [0.0, 0.25, 1.0]
|
||||
|
||||
response = test_client.post(
|
||||
endpoint,
|
||||
data={"prompt": "Replace only the masked regions."},
|
||||
files=[
|
||||
("source_video", ("source.mov", b"source-video", "video/quicktime")),
|
||||
("source_audio", ("source.mp3", b"source-audio", "audio/mpeg")),
|
||||
("video_noise_mask", _mask_file(video_mask)),
|
||||
("audio_noise_mask", _mask_file(audio_mask)),
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
if not endpoint.endswith("/sync"):
|
||||
_wait_for_status(test_client, response.json()["id"], VideoGenerationStatus.COMPLETED.value)
|
||||
|
||||
multi_modal_data = engine.captured_prompt["multi_modal_data"]
|
||||
assert engine.captured_latent_edit_source_bytes == {
|
||||
"source_video": b"source-video",
|
||||
"source_audio": b"source-audio",
|
||||
}
|
||||
assert multi_modal_data["video_noise_mask"] == video_mask
|
||||
assert multi_modal_data["audio_noise_mask"] == audio_mask
|
||||
_wait_until(
|
||||
lambda: (
|
||||
not Path(multi_modal_data["source_video"]).exists() and not Path(multi_modal_data["source_audio"]).exists()
|
||||
)
|
||||
)
|
||||
assert not Path(multi_modal_data["source_video"]).exists()
|
||||
assert not Path(multi_modal_data["source_audio"]).exists()
|
||||
|
||||
|
||||
def test_latent_edit_fields_are_rejected_for_models_without_capability(test_client):
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "edit"},
|
||||
files=[
|
||||
("source_video", ("source.mp4", b"video", "video/mp4")),
|
||||
("video_noise_mask", _mask_file("1")),
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "not supported" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("files", "message"),
|
||||
[
|
||||
(
|
||||
[("source_video", ("source.mp4", b"video", "video/mp4"))],
|
||||
"at least one",
|
||||
),
|
||||
(
|
||||
[
|
||||
("source_audio", ("source.wav", b"audio", "audio/wav")),
|
||||
("video_noise_mask", _mask_file("0")),
|
||||
],
|
||||
"source_audio requires an audio_noise_mask",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_h3_latent_edit_cross_field_validation(test_client, files, message):
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
|
||||
response = test_client.post("/v1/videos/sync", data={"prompt": "edit"}, files=files)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert message in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_name", "filename", "content_type", "message"),
|
||||
[
|
||||
("source_video", "source.webm", "video/webm", "MP4 or MOV"),
|
||||
("source_audio", "source.flac", "audio/flac", "WAV or MP3"),
|
||||
],
|
||||
)
|
||||
def test_h3_latent_edit_rejects_unsupported_source_formats(
|
||||
test_client,
|
||||
field_name,
|
||||
filename,
|
||||
content_type,
|
||||
message,
|
||||
):
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
mask_name = "video_noise_mask" if field_name == "source_video" else "audio_noise_mask"
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "edit"},
|
||||
files=[
|
||||
(field_name, (filename, b"source", content_type)),
|
||||
(mask_name, _mask_file("0")),
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert message in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mask", "message"),
|
||||
[
|
||||
("[not-json", "Invalid JSON"),
|
||||
("[" * 2000 + "0" + "]" * 2000, "nested too deeply"),
|
||||
("[" * 65 + "0" + "]" * 65, "nested too deeply"),
|
||||
],
|
||||
)
|
||||
def test_h3_latent_edit_rejects_malformed_mask_json(test_client, mask, message):
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "edit"},
|
||||
files=[
|
||||
("source_video", ("source.mp4", b"video", "video/mp4")),
|
||||
("video_noise_mask", _mask_file(mask)),
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert message in response.json()["detail"]
|
||||
|
||||
|
||||
def test_h3_latent_edit_rejects_mask_text_form_fields(test_client):
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "edit", "video_noise_mask": "0"},
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_h3_latent_edit_masks_without_sources_are_forwarded_to_stage0(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
engine.model_class_name = "MiniMaxH3Pipeline"
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "ordinary generation"},
|
||||
files=[
|
||||
("video_noise_mask", _mask_file("1")),
|
||||
("audio_noise_mask", _mask_file("[1, 1]")),
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert engine.captured_prompt["multi_modal_data"]["video_noise_mask"] == 1
|
||||
assert engine.captured_prompt["multi_modal_data"]["audio_noise_mask"] == [1, 1]
|
||||
|
||||
|
||||
def test_h3_latent_edit_rejects_oversized_mask_json(test_client, monkeypatch):
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
monkeypatch.setattr(video_generation_helpers, "LATENT_EDIT_MASK_FILE_MAX_BYTES", 4)
|
||||
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "edit"},
|
||||
files=[
|
||||
("source_video", ("source.mp4", b"video", "video/mp4")),
|
||||
("video_noise_mask", _mask_file("[0.0]")),
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "JSON exceeds" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_cancelled_latent_edit_persistence_cleans_prior_source(test_client, monkeypatch):
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
persist = video_generation_helpers._persist_latent_edit_source
|
||||
persisted_paths: list[str] = []
|
||||
|
||||
async def persist_then_cancel(upload, *, field_name):
|
||||
if field_name == "source_audio":
|
||||
raise asyncio.CancelledError
|
||||
path = await persist(upload, field_name=field_name)
|
||||
persisted_paths.append(path)
|
||||
return path
|
||||
|
||||
monkeypatch.setattr(video_generation_helpers, "_persist_latent_edit_source", persist_then_cancel)
|
||||
|
||||
with pytest.raises(FutureCancelledError):
|
||||
test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "edit"},
|
||||
files=[
|
||||
("source_video", ("source.mp4", b"source-video", "video/mp4")),
|
||||
("source_audio", ("source.wav", b"source-audio", "audio/wav")),
|
||||
("video_noise_mask", _mask_file("0")),
|
||||
("audio_noise_mask", _mask_file("0")),
|
||||
],
|
||||
)
|
||||
|
||||
assert len(persisted_paths) == 1
|
||||
assert not Path(persisted_paths[0]).exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_class_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
|
||||
def test_control_upload_capability_is_declared_only_by_cosmos3(test_client, model_class_name):
|
||||
handler = test_client.app.state.openai_serving_video
|
||||
@@ -991,6 +1455,35 @@ def test_cosmos3_reference_video_limit_uses_v2v_condition_frames():
|
||||
assert spec.keep == "first"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||
@pytest.mark.parametrize(
|
||||
("num_frames", "extra_params", "expected"),
|
||||
[
|
||||
(189, {"condition_frame_indexes_vision": [0, 2], "condition_video_keep": "last"}, (9, "last")),
|
||||
(189, {"condition_frame_indexes_vision": [0, 2]}, (9, "first")),
|
||||
(5, {"condition_frame_indexes_vision": [0, 20]}, (5, "first")),
|
||||
(None, {"action_mode": "inverse_dynamics", "action_chunk_size": 16}, (17, "first")),
|
||||
],
|
||||
)
|
||||
def test_cosmos3_reference_video_decode_policy_with_runtime_configs(typed, num_frames, extra_params, expected):
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
|
||||
if typed:
|
||||
stages = list(
|
||||
StageConfigFactory.create_typed_default_diffusion(
|
||||
"cosmos3",
|
||||
{"model_class_name": "Cosmos3OmniDiffusersPipeline"},
|
||||
).stage_configs
|
||||
)
|
||||
else:
|
||||
stages = _cosmos3_stage_configs()
|
||||
request = VideoGenerationRequest(prompt="Continue this motion.", num_frames=num_frames, extra_params=extra_params)
|
||||
|
||||
spec = _reference_video_decode_spec(request, stages)
|
||||
|
||||
assert (spec.max_frames, spec.keep) == expected
|
||||
|
||||
|
||||
def test_cosmos3_reference_video_limit_preserves_action_frames():
|
||||
request = VideoGenerationRequest(
|
||||
prompt="Predict the action.",
|
||||
@@ -1423,7 +1916,7 @@ def test_video_generation_response_exposes_action_payload(mocker: MockerFixture)
|
||||
},
|
||||
)
|
||||
|
||||
engine.generate = _generate
|
||||
engine.generate = _generate # type: ignore[method-assign]
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video.encode_video_base64",
|
||||
return_value="encoded-video",
|
||||
@@ -1549,6 +2042,7 @@ def test_action_extraction_accepts_multimodal_actions_payload():
|
||||
|
||||
def test_missing_handler_returns_503():
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 1
|
||||
app.include_router(router)
|
||||
app.state.openai_serving_video = None
|
||||
client = TestClient(app)
|
||||
@@ -1773,6 +2267,33 @@ def test_failed_generation_awaits_storage_cleanup(test_client, isolated_video_ba
|
||||
delete_spy.assert_called_once_with(video_id)
|
||||
|
||||
|
||||
def test_failed_async_latent_edit_cleans_source_upload(test_client, mocker: MockerFixture):
|
||||
handler = test_client.app.state.openai_serving_video
|
||||
handler._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
captured_path: list[str] = []
|
||||
|
||||
async def fail_generation(*_args, latent_edit_input=None, **_kwargs):
|
||||
assert latent_edit_input is not None
|
||||
captured_path.append(latent_edit_input.source_video)
|
||||
assert Path(latent_edit_input.source_video).read_bytes() == b"source-video"
|
||||
raise RuntimeError("edit failed")
|
||||
|
||||
mocker.patch.object(handler, "generate_video_bytes", side_effect=fail_generation)
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
data={"prompt": "will fail"},
|
||||
files=[
|
||||
("source_video", ("source.mp4", b"source-video", "video/mp4")),
|
||||
("video_noise_mask", _mask_file("0")),
|
||||
],
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
_wait_for_status(test_client, response.json()["id"], VideoGenerationStatus.FAILED.value)
|
||||
assert len(captured_path) == 1
|
||||
_wait_until(lambda: not Path(captured_path[0]).exists())
|
||||
|
||||
|
||||
def test_async_guardrail_error_returns_400_on_retrieve(test_client, mocker: MockerFixture):
|
||||
mocker.patch.object(
|
||||
OmniOpenAIServingVideo,
|
||||
@@ -2016,6 +2537,80 @@ def test_delete_in_progress_job_cancels_task_and_removes_metadata(test_client):
|
||||
assert retrieve_resp.status_code == 404
|
||||
|
||||
|
||||
def test_async_video_stays_queued_until_scheduler_admission(test_client):
|
||||
handler = SchedulerQueuedVideoHandler()
|
||||
test_client.app.state.openai_serving_video = handler
|
||||
|
||||
create_resp = test_client.post("/v1/videos", data={"prompt": "Queue this video"})
|
||||
assert create_resp.status_code == 200
|
||||
video_id = create_resp.json()["id"]
|
||||
assert handler.started.wait(timeout=2.0)
|
||||
|
||||
queued = test_client.get(f"/v1/videos/{video_id}")
|
||||
assert queued.status_code == 200
|
||||
assert queued.json()["status"] == VideoGenerationStatus.QUEUED.value
|
||||
|
||||
handler.admit.set()
|
||||
assert handler.in_progress.wait(timeout=2.0)
|
||||
in_progress = test_client.get(f"/v1/videos/{video_id}")
|
||||
assert in_progress.status_code == 200
|
||||
assert in_progress.json()["status"] == VideoGenerationStatus.IN_PROGRESS.value
|
||||
|
||||
assert test_client.delete(f"/v1/videos/{video_id}").status_code == 200
|
||||
|
||||
|
||||
def test_delete_times_out_engine_abort_and_still_cancels(test_client, monkeypatch):
|
||||
monkeypatch.setattr(api_server, "VIDEO_ABORT_TIMEOUT_S", 0.05)
|
||||
handler = HangingAbortHandler()
|
||||
test_client.app.state.openai_serving_video = handler
|
||||
|
||||
create_resp = test_client.post("/v1/videos", data={"prompt": "Hang abort"})
|
||||
assert create_resp.status_code == 200
|
||||
video_id = create_resp.json()["id"]
|
||||
assert handler.started.wait(timeout=2.0)
|
||||
|
||||
started = time.monotonic()
|
||||
delete_resp = test_client.delete(f"/v1/videos/{video_id}")
|
||||
assert time.monotonic() - started < 2.0
|
||||
assert delete_resp.status_code == 200
|
||||
assert handler.cancelled.wait(timeout=2.0)
|
||||
|
||||
|
||||
def test_delete_removes_artifact_if_job_completes_during_abort(test_client):
|
||||
handler = CompletingDuringAbortHandler()
|
||||
test_client.app.state.openai_serving_video = handler
|
||||
|
||||
create_resp = test_client.post("/v1/videos", data={"prompt": "Complete during delete"})
|
||||
assert create_resp.status_code == 200
|
||||
video_id = create_resp.json()["id"]
|
||||
assert handler.started.wait(timeout=2.0)
|
||||
|
||||
assert test_client.delete(f"/v1/videos/{video_id}").status_code == 200
|
||||
file_path = os.path.join(api_server.STORAGE_MANAGER.storage_path, video_id)
|
||||
assert not os.path.exists(file_path)
|
||||
assert asyncio.run(api_server.VIDEO_STORE.get(video_id)) is None
|
||||
|
||||
|
||||
def test_delete_aborts_engine_request_before_cancelling_task(test_client):
|
||||
engine = AbortTrackingOmni()
|
||||
test_client.app.state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion(
|
||||
engine,
|
||||
model_name="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||
)
|
||||
|
||||
create_resp = test_client.post("/v1/videos", data={"prompt": "Abort this video"})
|
||||
assert create_resp.status_code == 200
|
||||
video_id = create_resp.json()["id"]
|
||||
assert engine.entered.wait(timeout=2.0)
|
||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.IN_PROGRESS.value)
|
||||
|
||||
delete_resp = test_client.delete(f"/v1/videos/{video_id}")
|
||||
assert delete_resp.status_code == 200
|
||||
assert delete_resp.json()["deleted"] is True
|
||||
_wait_until(lambda: engine.aborted == [video_id])
|
||||
assert asyncio.run(api_server.VIDEO_STORE.get(video_id)) is None
|
||||
|
||||
|
||||
def test_video_response_file_extension_is_robust():
|
||||
response = VideoResponse(model="test-model", prompt="Make something beautiful")
|
||||
assert response.file_extension == "mp4"
|
||||
@@ -2461,6 +3056,7 @@ def test_cosmos3_control_upload_rejects_invalid_size(control_bytes, message, tes
|
||||
|
||||
def test_sync_missing_handler_returns_503():
|
||||
app = FastAPI()
|
||||
app.state.api_server_count = 1
|
||||
app.include_router(router)
|
||||
app.state.openai_serving_video = None
|
||||
client = TestClient(app)
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.entrypoints import async_omni as async_omni_mod
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -38,7 +39,8 @@ def get_fake_add_request(submitted_request_ids, submitted_lora_requests=None):
|
||||
|
||||
|
||||
def get_fake_abort(aborted_request_batches):
|
||||
async def fake_abort_async(request_ids):
|
||||
async def fake_abort_async(request_ids, timeout=None):
|
||||
del timeout
|
||||
aborted_request_batches.append(list(request_ids))
|
||||
|
||||
return fake_abort_async
|
||||
@@ -196,8 +198,8 @@ def test_abort_keeps_request_states_until_generate_cleanup():
|
||||
release = asyncio.Event()
|
||||
seen_during_wait: list[int] = []
|
||||
|
||||
async def slow_abort_async(request_ids):
|
||||
del request_ids
|
||||
async def slow_abort_async(request_ids, timeout=None):
|
||||
del request_ids, timeout
|
||||
seen_during_wait.append(len(omni.request_states))
|
||||
await release.wait()
|
||||
|
||||
@@ -224,8 +226,8 @@ def test_abort_keeps_request_states_until_generate_cleanup():
|
||||
@pytest.mark.cpu
|
||||
def test_abort_propagates_engine_errors_without_popping_state():
|
||||
async def run():
|
||||
async def failing_abort_async(request_ids):
|
||||
del request_ids
|
||||
async def failing_abort_async(request_ids, timeout=None):
|
||||
del request_ids, timeout
|
||||
raise RuntimeError("orchestrator abort failed")
|
||||
|
||||
omni = get_async_omni_instance(fake_abort_request=failing_abort_async)
|
||||
@@ -253,7 +255,8 @@ def test_abort_enqueues_prefix_tokens_from_engine():
|
||||
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
async def abort_with_prefix(request_ids):
|
||||
async def abort_with_prefix(request_ids, timeout=None):
|
||||
del timeout
|
||||
rid = request_ids[0]
|
||||
engine_output = OmniRequestOutput(
|
||||
request_id=rid,
|
||||
@@ -308,8 +311,8 @@ def test_abort_enqueues_synthetic_finished_when_engine_returns_empty():
|
||||
async def run():
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
async def empty_abort_async(request_ids):
|
||||
del request_ids
|
||||
async def empty_abort_async(request_ids, timeout=None):
|
||||
del request_ids, timeout
|
||||
return []
|
||||
|
||||
omni = get_async_omni_instance(fake_abort_request=empty_abort_async)
|
||||
@@ -329,6 +332,95 @@ def test_abort_enqueues_synthetic_finished_when_engine_returns_empty():
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_abort_drops_consumed_metric_message_ids_with_state():
|
||||
"""Exercise the production leak path from #6462: a metric-bearing
|
||||
``OutputMessage`` goes through ``_handle_output_message`` (which populates
|
||||
the per-request de-dup set), the request is aborted (which, since #6367,
|
||||
keeps ``request_states`` registered and enqueues a terminal abort output),
|
||||
and the set must be released with the state by ``generate()``'s normal
|
||||
cleanup — with no independent request-keyed metric state surviving on the
|
||||
instance at any point. A reintroduced
|
||||
class-level ``_consumed_metric_messages``-style map populated by the
|
||||
production handler fails the sweep below.
|
||||
"""
|
||||
import time as _time
|
||||
from collections.abc import Mapping
|
||||
|
||||
from vllm_omni.engine.messages import OutputMessage
|
||||
from vllm_omni.entrypoints.client_request_state import ClientRequestState
|
||||
from vllm_omni.metrics.stats import OrchestratorAggregator
|
||||
|
||||
async def run():
|
||||
omni = get_async_omni_instance()
|
||||
omni.engine.get_stage_metadata = lambda stage_id: SimpleNamespace(
|
||||
stage_type="llm",
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
)
|
||||
|
||||
rid = "req-1-cccc"
|
||||
state = ClientRequestState(request_id=rid, external_request_id="req-1")
|
||||
state.metrics = OrchestratorAggregator(
|
||||
num_stages=1,
|
||||
log_stats=False,
|
||||
wall_start_ts=_time.time(),
|
||||
final_stage_id_for_e2e=0,
|
||||
)
|
||||
state.input_stream_task = None
|
||||
omni.request_states[rid] = state
|
||||
|
||||
msg = OutputMessage(
|
||||
request_id=rid,
|
||||
stage_id=0,
|
||||
engine_outputs=SimpleNamespace(final_output_type="text"),
|
||||
metrics=SimpleNamespace(
|
||||
num_tokens_in=3,
|
||||
num_tokens_out=5,
|
||||
stage_gen_time_ms=1.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
),
|
||||
finished=False,
|
||||
)
|
||||
handled, out_rid, out_stage, out_state = omni._handle_output_message(msg)
|
||||
assert handled is False and out_state is state
|
||||
# The production handler recorded the de-dup entry on the state...
|
||||
assert id(msg) in state.consumed_metric_message_ids
|
||||
# ...and de-duplicates a replay of the same message object.
|
||||
omni._handle_output_message(msg)
|
||||
assert len(state.consumed_metric_message_ids) == 1
|
||||
|
||||
assert not hasattr(omni, "_consumed_metric_messages")
|
||||
|
||||
# #6367 contract: abort() keeps the state registered and enqueues a
|
||||
# terminal abort output; generate()'s ``finally`` owns the cleanup.
|
||||
await omni.abort("req-1")
|
||||
assert rid in omni.request_states
|
||||
terminal = state.queue.get_nowait()
|
||||
assert terminal.finished is True
|
||||
assert terminal.engine_outputs.outputs[0].finish_reason == "abort"
|
||||
# The de-dup set still lives only on the (still-registered) state.
|
||||
for name, value in vars(omni).items():
|
||||
if name == "request_states":
|
||||
continue
|
||||
assert not (isinstance(value, Mapping) and rid in value), (
|
||||
f"request-keyed metric state kept outside request_states in {name!r}"
|
||||
)
|
||||
|
||||
# generate()'s normal cleanup releases the state and the set with it.
|
||||
omni._log_summary_and_cleanup(rid)
|
||||
assert rid not in omni.request_states
|
||||
for name, value in vars(omni).items():
|
||||
assert not (isinstance(value, Mapping) and rid in value), (
|
||||
f"request-keyed state survived cleanup in {name!r}"
|
||||
)
|
||||
assert not hasattr(omni, "_consumed_metric_messages")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_generate_accepts_request_after_repeated_cancellations():
|
||||
async def run_test():
|
||||
@@ -378,6 +470,42 @@ def test_generate_accepts_request_after_repeated_cancellations():
|
||||
asyncio.run(run_test())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_generate_cancel_bounds_cleanup_abort(monkeypatch):
|
||||
"""Cancelled generate() must not wait forever on a wedged orchestrator abort."""
|
||||
monkeypatch.setattr(async_omni_mod, "ABORT_TIMEOUT_S", 0.05)
|
||||
|
||||
async def run():
|
||||
seen_timeouts: list[float | None] = []
|
||||
|
||||
async def hanging_abort_async(request_ids, timeout=None):
|
||||
del request_ids
|
||||
seen_timeouts.append(timeout)
|
||||
await asyncio.Event().wait()
|
||||
|
||||
omni = get_async_omni_instance(fake_abort_request=hanging_abort_async)
|
||||
|
||||
async def collect():
|
||||
async for _ in omni.generate(
|
||||
prompt={"prompt": "prompt"},
|
||||
request_id="cancel-hang-abort",
|
||||
sampling_params_list=[SimpleNamespace()],
|
||||
output_modalities=["image"],
|
||||
):
|
||||
pass
|
||||
|
||||
task = asyncio.create_task(collect())
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(task, timeout=1.0)
|
||||
assert seen_timeouts
|
||||
assert seen_timeouts[-1] == 0.05
|
||||
assert omni.request_states == {}
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_generate_cancellation_converges_after_engine_shutdown_starts(mocker):
|
||||
async def run_test():
|
||||
|
||||
@@ -2,13 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.resolver import OmniConfigResolution
|
||||
from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig, extract_diffusion_stage_config_kwargs
|
||||
from vllm_omni.config.resolver import OmniConfigResolution, resolve_omni_config
|
||||
from vllm_omni.diffusion.data import AttentionConfig, OmniDiffusionConfig
|
||||
from vllm_omni.engine import stage_init_utils
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
@@ -19,7 +21,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _terminal_config(stage_cfg: dict) -> OmniDiffusionConfig:
|
||||
return OmniDiffusionConfig.from_kwargs(**stage_cfg["engine_args"])
|
||||
kwargs = extract_diffusion_stage_config_kwargs(stage_cfg["engine_args"], stage_id=stage_cfg["stage_id"])
|
||||
return OmniDiffusionConfig.from_kwargs(**kwargs)
|
||||
|
||||
|
||||
def test_default_stage_config_includes_cache_backend():
|
||||
@@ -30,6 +33,9 @@ def test_default_stage_config_includes_cache_backend():
|
||||
"cache_config": '{"Fn_compute_blocks": 2}',
|
||||
"vae_use_slicing": True,
|
||||
"ulysses_degree": 2,
|
||||
"seed": 7,
|
||||
"kv_cache_dtype": "fp8",
|
||||
"diffusion_kv_cache_dtype": "fp8_e4m3",
|
||||
}
|
||||
)[0]
|
||||
|
||||
@@ -40,6 +46,9 @@ def test_default_stage_config_includes_cache_backend():
|
||||
assert engine_args["vae_use_slicing"] is True
|
||||
assert engine_args["parallel_config"]["ulysses_degree"] == 2
|
||||
assert engine_args["model_stage"] == "diffusion"
|
||||
assert "seed" not in engine_args
|
||||
assert "kv_cache_dtype" not in engine_args
|
||||
assert engine_args["diffusion_kv_cache_dtype"] == "fp8_e4m3"
|
||||
|
||||
|
||||
def test_default_stage_config_preserves_ulysses_a2a_permute() -> None:
|
||||
@@ -117,7 +126,7 @@ def test_stage_override_preserves_model_extras_for_default_diffusion_stage(mocke
|
||||
trust_remote_code=False,
|
||||
)
|
||||
|
||||
assert stage_configs[0]["engine_args"]["extras"]["ltx2_use_conv_vae"] is True
|
||||
assert stage_configs[0].diffusion_config.extras["ltx2_use_conv_vae"] is True
|
||||
|
||||
|
||||
def test_default_stage_rejects_unknown_nested_parallel_config_key():
|
||||
@@ -128,6 +137,67 @@ def test_default_stage_rejects_unknown_nested_parallel_config_key():
|
||||
)
|
||||
|
||||
|
||||
def test_default_stage_routes_ar_profiler_away_before_diffusion_build(mocker):
|
||||
stage_dict = StageConfigFactory.create_default_diffusion({"enable_ar_profiler": True})[0]
|
||||
assert "enable_ar_profiler" not in stage_dict["engine_args"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_name", "value"),
|
||||
[
|
||||
("enable_sleep_mod", None),
|
||||
("enable_lora", True),
|
||||
("kv_cache_dtype", "fp8"),
|
||||
("seed", 7),
|
||||
],
|
||||
)
|
||||
def test_legacy_diffusion_stage_rejects_unowned_field(field_name, value):
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.yaml_util import create_config
|
||||
from vllm_omni.engine.stage_init_utils import build_diffusion_config
|
||||
|
||||
stage_dict = StageConfigFactory.create_default_diffusion({"model": "unused"})[0]
|
||||
stage_dict["engine_args"][field_name] = value
|
||||
stage_cfg = create_config(stage_dict)
|
||||
metadata = SimpleNamespace(stage_id=0, cfg_kv_collect_func=None)
|
||||
|
||||
with pytest.raises(ValueError, match=rf"stage 0.*{field_name}"):
|
||||
build_diffusion_config("unused", stage_cfg, metadata)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field_name", "value"),
|
||||
[
|
||||
("enable_sleep_mod", None),
|
||||
("enable_lora", True),
|
||||
],
|
||||
)
|
||||
def test_default_diffusion_factory_rejects_unowned_field(field_name, value):
|
||||
with pytest.raises(ValueError, match=rf"stage 0.*{field_name}"):
|
||||
StageConfigFactory.create_default_diffusion({field_name: value})
|
||||
|
||||
|
||||
def test_legacy_default_stage_build_accepts_engine_adapter_metadata(monkeypatch):
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.yaml_util import create_config
|
||||
from vllm_omni.engine import stage_init_utils
|
||||
|
||||
stage_dict = StageConfigFactory.create_default_diffusion(
|
||||
{
|
||||
"model": "unused",
|
||||
"api_key": "frontend-owned",
|
||||
}
|
||||
)[0]
|
||||
assert "api_key" not in stage_dict["engine_args"]
|
||||
stage_cfg = create_config(stage_dict)
|
||||
metadata = SimpleNamespace(stage_id=0, cfg_kv_collect_func=None, default_sampling_params=None)
|
||||
monkeypatch.setattr(stage_init_utils.current_omni_platform, "get_device_count", lambda: 1)
|
||||
|
||||
config = stage_init_utils.build_diffusion_config("unused", stage_cfg, metadata)
|
||||
|
||||
assert config.model == "unused"
|
||||
|
||||
|
||||
def test_default_cache_config_used_when_missing():
|
||||
"""Ensure default cache_config is synthesized when only backend is given."""
|
||||
stage_cfg = StageConfigFactory.create_default_diffusion(
|
||||
@@ -239,6 +309,33 @@ def test_default_stage_config_includes_default_sampling_params():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||
@pytest.mark.parametrize("sampling_defaults", [{"0": {"guidance_scale": 7.5}}, '{"0":{"guidance_scale":7.5}}'])
|
||||
def test_generic_diffusion_sampling_defaults_remain_overridable(typed, sampling_defaults):
|
||||
from vllm_omni.config.yaml_util import create_config
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
kwargs = {"model_class_name": "QwenImagePipeline", "default_sampling_params": sampling_defaults}
|
||||
if typed:
|
||||
stage = StageConfigFactory.create_typed_default_diffusion("generic-diffusion", kwargs).stage_configs[0]
|
||||
metadata = stage_init_utils.extract_stage_metadata_from_omni_stage_config(stage)
|
||||
else:
|
||||
stage = create_config(StageConfigFactory.create_default_diffusion(kwargs))[0]
|
||||
metadata = stage_init_utils.extract_legacy_stage_metadata(stage)
|
||||
base = OmniBase.__new__(OmniBase)
|
||||
base.engine = SimpleNamespace(num_stages=1, stage_configs=[stage])
|
||||
base.default_sampling_params_list = [metadata.default_sampling_params]
|
||||
base.sampling_constraints_list = base._get_sampling_constraints_list([stage])
|
||||
|
||||
assert base.resolve_sampling_params_list(None)[0].guidance_scale == 7.5
|
||||
requested = OmniDiffusionSamplingParams(guidance_scale=2.0)
|
||||
assert base.resolve_sampling_params_list(requested)[0].guidance_scale == 2.0
|
||||
assert requested.guidance_scale == 2.0
|
||||
assert base.default_sampling_params_list[0].guidance_scale == 7.5
|
||||
assert base.sampling_constraints_list == [{}]
|
||||
|
||||
|
||||
def test_default_stage_config_includes_diffusion_attention_backend():
|
||||
"""Ensure diffusion attention shorthand lands in engine_args.diffusion_attention_config."""
|
||||
stage_cfg = StageConfigFactory.create_default_diffusion(
|
||||
@@ -642,10 +739,11 @@ def test_serve_cli_rejects_invalid_request_batch_max_wait_ms(bad_wait: str):
|
||||
)
|
||||
|
||||
|
||||
def test_serve_cli_accepts_additional_config():
|
||||
@pytest.mark.parametrize("subcommand_dest", ["command", "subparser"])
|
||||
def test_serve_cli_accepts_additional_config(subcommand_dest):
|
||||
"""Ensure diffusion serve CLI exposes additional_config and forwards it to stage config."""
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
subparsers = parser.add_subparsers(dest=subcommand_dest)
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
|
||||
args = parser.parse_args(
|
||||
@@ -658,7 +756,7 @@ def test_serve_cli_accepts_additional_config():
|
||||
]
|
||||
)
|
||||
|
||||
stage_cfg = StageConfigFactory.create_default_diffusion(vars(args))[0]
|
||||
stage_cfg = StageConfigFactory.create_default_diffusion(args.get_explicit_kwargs_dict())[0]
|
||||
|
||||
engine_args = stage_cfg["engine_args"]
|
||||
|
||||
@@ -674,7 +772,7 @@ def test_resolve_stage_configs_delegates_overrides_to_resolver(mocker):
|
||||
engine_args=SimpleNamespace(additional_config=additional_config),
|
||||
)
|
||||
resolve_config = mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.resolve_omni_config",
|
||||
"vllm_omni.engine.omni_engine_base.resolve_omni_config",
|
||||
return_value=OmniConfigResolution(
|
||||
config_path="dummy.yaml",
|
||||
stage_configs=(fake_diffusion_stage,),
|
||||
@@ -726,3 +824,124 @@ def test_default_stage_config_includes_quantization_config():
|
||||
stage_cfg = StageConfigFactory.create_default_diffusion({"quantization_config": quantization_config})[0]
|
||||
|
||||
assert stage_cfg["engine_args"]["quantization_config"] == quantization_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||
def test_default_diffusion_factory_preserves_engine_quantization(typed, monkeypatch):
|
||||
monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda _self: 29500)
|
||||
monkeypatch.setattr(OmniDiffusionConfig, "enrich_config", lambda _self: None)
|
||||
kwargs = {"quantization": "fp8"}
|
||||
|
||||
if typed:
|
||||
stage = StageConfigFactory.create_typed_default_diffusion("generic-diffusion", kwargs).stage_configs[0]
|
||||
config = stage.diffusion_config
|
||||
config.enrich_config()
|
||||
else:
|
||||
config = _terminal_config(StageConfigFactory.create_default_diffusion(kwargs)[0])
|
||||
|
||||
assert config.quantization_config is not None
|
||||
assert config.quantization_config.get_name() == "fp8"
|
||||
assert config.quantization_config_is_auto_detected is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_class_name", ["HeliosPipeline", "HunyuanVideo15Pipeline"])
|
||||
def test_generic_diffusion_uses_canonical_video_output_type(model_class_name):
|
||||
config = StageConfigFactory.create_typed_default_diffusion(
|
||||
"generic-video",
|
||||
{"model_class_name": model_class_name},
|
||||
)
|
||||
|
||||
assert config.stage_configs[0].final_output_type == "video"
|
||||
|
||||
|
||||
def test_generic_diffusion_resolves_structured_stage_without_legacy_conversion(mocker):
|
||||
"""Generic diffusion reaches runtime as the structured stage itself."""
|
||||
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||
return_value=(True, "FakeDiffusionPipeline"),
|
||||
)
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
"generic-diffusion",
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=None,
|
||||
cli_overrides={
|
||||
"num_gpus": 4,
|
||||
"tensor_parallel_size": 2,
|
||||
"default_sampling_params": '{"0": {"guidance_scale": 7.5}}',
|
||||
},
|
||||
stage_overrides=None,
|
||||
strategy_config_path=None,
|
||||
)
|
||||
|
||||
assert resolved.pipeline_config is not None
|
||||
assert resolved.pipeline_config.model_type == "generic_diffusion"
|
||||
assert len(resolved.stage_configs) == 1
|
||||
stage = resolved.stage_configs[0]
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.model_config.model == "generic-diffusion"
|
||||
assert stage.model_config.default_sampling_params == {"guidance_scale": 7.5}
|
||||
assert stage.parallel_config.tensor_parallel_size == 2
|
||||
assert stage.parallel_config.data_parallel_size == 2
|
||||
assert stage.parallel_config.world_size == 4
|
||||
assert stage.runtime_config.devices == "0,1,2,3"
|
||||
|
||||
|
||||
def test_generic_diffusion_structured_stage_reaches_standard_startup(mocker):
|
||||
"""Standard runtime resolves and starts the typed stage through the real launcher."""
|
||||
from vllm_omni.engine import stage_engine_startup as startup_module
|
||||
from vllm_omni.engine import stage_runtime as runtime_module
|
||||
from vllm_omni.engine.stage_runtime import StageRuntime
|
||||
|
||||
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||
return_value=(True, "FakeDiffusionPipeline"),
|
||||
)
|
||||
resolved = resolve_omni_config(
|
||||
"generic-diffusion",
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=None,
|
||||
cli_overrides={"num_gpus": 1},
|
||||
stage_overrides=None,
|
||||
strategy_config_path=None,
|
||||
)
|
||||
stage = resolved.stage_configs[0]
|
||||
launched: dict[str, Any] = {}
|
||||
client = SimpleNamespace(input_address=None, shutdown=mocker.Mock())
|
||||
|
||||
mocker.patch.object(runtime_module, "prepare_engine_environment")
|
||||
mocker.patch.object(runtime_module, "load_omni_transfer_config_for_model", return_value=None)
|
||||
mocker.patch.object(runtime_module, "get_stage_connector_spec", return_value={})
|
||||
mocker.patch.object(runtime_module, "resolve_omni_kv_config_for_stage", return_value=(None, None, None))
|
||||
|
||||
def _initialize_typed_stage(stage_id, model, stage_config, metadata, **kwargs):
|
||||
launched.update(
|
||||
stage_id=stage_id,
|
||||
model=model,
|
||||
stage_config=stage_config,
|
||||
metadata=metadata,
|
||||
**kwargs,
|
||||
)
|
||||
return client
|
||||
|
||||
mocker.patch.object(startup_module, "initialize_diffusion_stage", side_effect=_initialize_typed_stage)
|
||||
|
||||
runtime = StageRuntime(
|
||||
stage_configs=list(resolved.stage_configs),
|
||||
model="generic-diffusion",
|
||||
config_path="",
|
||||
stage_init_timeout=10,
|
||||
async_chunk=False,
|
||||
)
|
||||
runtime.initialize()
|
||||
|
||||
assert launched["stage_config"] is stage
|
||||
assert isinstance(launched["stage_config"], VllmOmniDiffusionStageConfig)
|
||||
assert launched["metadata"].stage_type == "diffusion"
|
||||
assert launched["metadata"].model_stage == "diffusion"
|
||||
assert launched["use_inline"] is True
|
||||
assert launched["model"] == "generic-diffusion"
|
||||
assert launched["stage_id"] == 0
|
||||
assert runtime.stage_pools[0].clients == [client]
|
||||
@@ -1,601 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexOutputAction,
|
||||
DuplexOutputDecision,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.engine.messages import OutputMessage
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.client_request_state import ClientRequestState
|
||||
from vllm_omni.entrypoints.duplex_request_client import DuplexRequestClient
|
||||
from vllm_omni.metrics.stats import OrchestratorAggregator, StageRequestStats, StageStats
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.outputs.duplex import attach_duplex_output_decision
|
||||
|
||||
|
||||
def test_async_omni_uses_extracted_duplex_request_client():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = SimpleNamespace(num_stages=2)
|
||||
app.request_states = {}
|
||||
app._duplex_request_client = None
|
||||
|
||||
client = app._get_duplex_request_client()
|
||||
|
||||
assert isinstance(client, DuplexRequestClient)
|
||||
assert not hasattr(client, "_async_omni")
|
||||
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_open_duplex_forwards_session_config_and_timeout():
|
||||
calls = []
|
||||
output_handler_starts = 0
|
||||
|
||||
async def open_duplex_session_async(session_id, **kwargs):
|
||||
calls.append((session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = SimpleNamespace(open_duplex_session_async=open_duplex_session_async)
|
||||
|
||||
def start_output_handler():
|
||||
nonlocal output_handler_starts
|
||||
output_handler_starts += 1
|
||||
|
||||
app._final_output_handler = start_output_handler
|
||||
|
||||
result = await app.open_duplex_session_async(
|
||||
"sid",
|
||||
fence=DuplexFence("sid"),
|
||||
capabilities={"implementation_level": "model_native_duplex"},
|
||||
session_config={"instructions": "Be brief.", "idle_timeout_s": 30},
|
||||
timeout=7.5,
|
||||
)
|
||||
|
||||
assert result == {"ok": True}
|
||||
assert output_handler_starts == 1
|
||||
assert calls == [
|
||||
(
|
||||
"sid",
|
||||
{
|
||||
"session_mode": "duplex",
|
||||
"capabilities": {"implementation_level": "model_native_duplex"},
|
||||
"session_config": {"instructions": "Be brief.", "idle_timeout_s": 30},
|
||||
"fence": DuplexFence("sid"),
|
||||
"timeout": 7.5,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_runtime_controls_forward_timeout():
|
||||
calls = []
|
||||
cancelled_fence = DuplexFence("sid", epoch=2, turn_id=3)
|
||||
next_fence = DuplexFence("sid", epoch=3, turn_id=3)
|
||||
|
||||
async def append_duplex_input_async(session_id, **kwargs):
|
||||
calls.append(("append", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def signal_duplex_turn_async(session_id, **kwargs):
|
||||
calls.append(("signal", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def close_duplex_session_async(session_id, **kwargs):
|
||||
calls.append(("close", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def touch_duplex_session_async(session_id, **kwargs):
|
||||
calls.append(("touch", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def resume_duplex_session_async(session_id, **kwargs):
|
||||
calls.append(("resume", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
app.engine = SimpleNamespace(
|
||||
append_duplex_input_async=append_duplex_input_async,
|
||||
signal_duplex_turn_async=signal_duplex_turn_async,
|
||||
close_duplex_session_async=close_duplex_session_async,
|
||||
touch_duplex_session_async=touch_duplex_session_async,
|
||||
resume_duplex_session_async=resume_duplex_session_async,
|
||||
)
|
||||
|
||||
await app.append_duplex_input_async(
|
||||
"sid",
|
||||
mode="append_audio_chunk",
|
||||
payload={},
|
||||
fence=cancelled_fence,
|
||||
timeout=12.5,
|
||||
)
|
||||
await app.signal_duplex_turn_async(
|
||||
"sid",
|
||||
event="barge_in",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
timeout=13.5,
|
||||
)
|
||||
await app.close_duplex_session_async("sid", reason="done", fence=next_fence, timeout=14.5)
|
||||
await app.touch_duplex_session_async(
|
||||
"sid",
|
||||
fence=next_fence,
|
||||
activity=DuplexLeaseActivity.HEARTBEAT,
|
||||
timeout=15.5,
|
||||
)
|
||||
await app.resume_duplex_session_async(
|
||||
"sid",
|
||||
fence=next_fence,
|
||||
expected_lease_generation=3,
|
||||
timeout=16.5,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
"append",
|
||||
"sid",
|
||||
{
|
||||
"mode": "append_audio_chunk",
|
||||
"payload": {},
|
||||
"final": False,
|
||||
"expected_epoch": None,
|
||||
"fence": cancelled_fence,
|
||||
"timeout": 12.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"signal",
|
||||
"sid",
|
||||
{
|
||||
"event": "barge_in",
|
||||
"fence": cancelled_fence,
|
||||
"next_fence": next_fence,
|
||||
"timeout": 13.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"close",
|
||||
"sid",
|
||||
{
|
||||
"reason": "done",
|
||||
"fence": next_fence,
|
||||
"timeout": 14.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"touch",
|
||||
"sid",
|
||||
{
|
||||
"fence": next_fence,
|
||||
"activity": DuplexLeaseActivity.HEARTBEAT,
|
||||
"timeout": 15.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"resume",
|
||||
"sid",
|
||||
{
|
||||
"fence": next_fence,
|
||||
"expected_lease_generation": 3,
|
||||
"timeout": 16.5,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_append_can_defer_data_plane_output_collection():
|
||||
app = object.__new__(AsyncOmni)
|
||||
request_id = duplex_resource_request_id(DuplexFence("sid"), "stage0")
|
||||
|
||||
async def append_duplex_input_async(session_id, **kwargs):
|
||||
del session_id, kwargs
|
||||
request_state = app.request_states.get(request_id)
|
||||
assert request_state is not None
|
||||
assert request_state.metrics is not None
|
||||
return {
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": request_id,
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
app.engine = SimpleNamespace(append_duplex_input_async=append_duplex_input_async)
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
|
||||
result = await app.append_duplex_input_async(
|
||||
"sid",
|
||||
mode="append_audio_chunk",
|
||||
payload={},
|
||||
fence=DuplexFence("sid"),
|
||||
collect_outputs=False,
|
||||
)
|
||||
|
||||
assert "data_plane_outputs" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_append_forwards_fence():
|
||||
calls = []
|
||||
|
||||
async def append_duplex_input_async(session_id, **kwargs):
|
||||
calls.append((session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
app.engine = SimpleNamespace(append_duplex_input_async=append_duplex_input_async)
|
||||
fence = DuplexFence("sid", epoch=1, turn_id=2)
|
||||
|
||||
await app.append_duplex_input_async(
|
||||
"sid",
|
||||
mode="append_audio_chunk",
|
||||
payload={},
|
||||
fence=fence,
|
||||
collect_outputs=False,
|
||||
)
|
||||
|
||||
assert calls[0][1]["fence"] is fence
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_collect_waits_for_response_stage():
|
||||
app = object.__new__(AsyncOmni)
|
||||
req_state = ClientRequestState("duplex-sid-e0-stage0")
|
||||
stage0_output = OmniRequestOutput(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=0,
|
||||
finished=False,
|
||||
)
|
||||
stage1_output = OmniRequestOutput(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=1,
|
||||
final_output_type="audio",
|
||||
finished=False,
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=0,
|
||||
engine_outputs=stage0_output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=1,
|
||||
engine_outputs=stage1_output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
outputs = await app._collect_duplex_data_plane_outputs(
|
||||
"duplex-sid-e0-stage0",
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert outputs == [stage1_output]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_request_client_retains_output_route_at_segment_terminal():
|
||||
request_id = "duplex-sid-e0-stage0"
|
||||
request_state = ClientRequestState(request_id)
|
||||
request_states = {request_id: request_state}
|
||||
output = OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=1,
|
||||
final_output_type="audio",
|
||||
finished=True,
|
||||
)
|
||||
await request_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=1,
|
||||
engine_outputs=output,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
client = DuplexRequestClient(
|
||||
SimpleNamespace(),
|
||||
SimpleNamespace(
|
||||
request_states=request_states,
|
||||
num_stages=2,
|
||||
log_stats=False,
|
||||
),
|
||||
)
|
||||
|
||||
outputs = await client.collect_outputs(
|
||||
request_id,
|
||||
request_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert outputs == [output]
|
||||
assert request_states[request_id] is request_state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_direct_output_keeps_stage_metrics():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app._duplex_request_client = None
|
||||
app._final_output_handler = lambda: None
|
||||
request_id = "duplex-sid-e0-stage0"
|
||||
req_state = ClientRequestState(request_id)
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={
|
||||
"duplex_direct_response": True,
|
||||
"duplex_native_decision": "speak",
|
||||
},
|
||||
)
|
||||
direct_output = attach_duplex_output_decision(
|
||||
OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
final_output_type="text",
|
||||
finished=True,
|
||||
),
|
||||
decision,
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
engine_outputs=direct_output,
|
||||
metrics=StageRequestStats(
|
||||
batch_id=1,
|
||||
batch_size=1,
|
||||
num_tokens_in=160,
|
||||
num_tokens_out=8,
|
||||
stage_gen_time_ms=80.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(total_token=8, total_gen_time_ms=80.0),
|
||||
vllm_ttft_ms=12.5,
|
||||
vllm_tpot_ms=9.0,
|
||||
vllm_itl_ms=8.75,
|
||||
vllm_itls_ms=[8.0, 9.5],
|
||||
),
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
|
||||
outputs = await app._collect_duplex_data_plane_outputs(
|
||||
request_id,
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert outputs == [direct_output]
|
||||
assert direct_output.metrics["stage_metrics"]["0"]["vllm_ttft_ms"] == 12.5
|
||||
assert direct_output.metrics["stage_metrics"]["0"]["vllm_tpot_ms"] == 9.0
|
||||
assert direct_output.metrics["stage_metrics"]["0"]["vllm_itls_ms"] == [8.0, 9.5]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_metrics_cursor_isolates_resumable_collect_windows():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app._duplex_request_client = None
|
||||
app._final_output_handler = lambda: None
|
||||
request_id = "duplex-sid-e0-stage0"
|
||||
req_state = ClientRequestState(request_id)
|
||||
req_state.metrics = OrchestratorAggregator(
|
||||
num_stages=2,
|
||||
log_stats=False,
|
||||
wall_start_ts=0.0,
|
||||
final_stage_id_for_e2e=1,
|
||||
)
|
||||
|
||||
def stage_message(*, batch_id: int, tokens: int, ttft_ms: float, itls: list[float]) -> OutputMessage:
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={"duplex_direct_response": True, "duplex_native_decision": "speak"},
|
||||
)
|
||||
output = attach_duplex_output_decision(
|
||||
OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
final_output_type="text",
|
||||
finished=True,
|
||||
),
|
||||
decision,
|
||||
)
|
||||
return OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
engine_outputs=output,
|
||||
metrics=StageRequestStats(
|
||||
batch_id=batch_id,
|
||||
batch_size=1,
|
||||
num_tokens_in=160,
|
||||
num_tokens_out=tokens,
|
||||
stage_gen_time_ms=80.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(total_token=tokens, total_gen_time_ms=80.0),
|
||||
vllm_ttft_ms=ttft_ms,
|
||||
vllm_tpot_ms=9.0,
|
||||
vllm_itl_ms=sum(itls) / len(itls),
|
||||
vllm_itls_ms=itls,
|
||||
),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
await req_state.queue.put(stage_message(batch_id=1, tokens=71, ttft_ms=62.5, itls=[10.0, 11.0]))
|
||||
first = await app._collect_duplex_data_plane_outputs(
|
||||
request_id,
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
await req_state.queue.put(stage_message(batch_id=2, tokens=155, ttft_ms=81.0, itls=[12.0, 13.0, 14.0]))
|
||||
second = await app._collect_duplex_data_plane_outputs(
|
||||
request_id,
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert first[0].metrics["stage_metrics"]["0"]["num_tokens_out"] == 71
|
||||
second_stage0 = second[0].metrics["stage_metrics"]["0"]
|
||||
assert second_stage0["num_tokens_out"] == 155
|
||||
assert second_stage0["vllm_ttft_ms"] == 81.0
|
||||
assert second_stage0["vllm_itls_ms"] == [12.0, 13.0, 14.0]
|
||||
|
||||
|
||||
def test_async_omni_duplex_direct_output_prefers_outer_control_metadata():
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={
|
||||
"duplex_direct_response": True,
|
||||
"duplex_native_decision": "listen",
|
||||
},
|
||||
)
|
||||
output = attach_duplex_output_decision(
|
||||
OmniRequestOutput(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=0,
|
||||
outputs=[],
|
||||
_multimodal_output={"kind": "processed-payload"},
|
||||
_custom_output={"inner": "completion"},
|
||||
),
|
||||
decision,
|
||||
)
|
||||
|
||||
assert AsyncOmni._is_direct_duplex_data_plane_response(output)
|
||||
|
||||
|
||||
def test_async_omni_drops_unregistered_duplex_prefixed_output():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.request_states = {}
|
||||
output = OmniRequestOutput(request_id="duplex-unregistered-e0-stage0", stage_id=0)
|
||||
|
||||
keep_processing, request_id, stage_id, request_state = app._handle_output_message(
|
||||
OutputMessage(
|
||||
request_id=output.request_id,
|
||||
stage_id=0,
|
||||
engine_outputs=output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert keep_processing is True
|
||||
assert request_id is stage_id is request_state is None
|
||||
assert app.request_states == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_collect_wraps_raw_response_stage_output():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.log_stats = False
|
||||
app._enable_ar_profiler = False
|
||||
app.request_states = {}
|
||||
app.prom_metrics = SimpleNamespace(
|
||||
set_running=lambda value: None,
|
||||
set_waiting=lambda value: None,
|
||||
)
|
||||
app.engine = SimpleNamespace(
|
||||
num_stages=2,
|
||||
get_stage_metadata=lambda stage_id: SimpleNamespace(
|
||||
final_output=stage_id == 1,
|
||||
final_output_type="audio",
|
||||
),
|
||||
)
|
||||
req_state = ClientRequestState("duplex-sid-e0-stage0")
|
||||
raw_stage1_output = SimpleNamespace(
|
||||
finished=False,
|
||||
outputs=[],
|
||||
stage_durations={},
|
||||
peak_memory_mb=0.0,
|
||||
final_output_type="audio",
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=1,
|
||||
engine_outputs=raw_stage1_output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
outputs = await app._collect_duplex_data_plane_outputs(
|
||||
"duplex-sid-e0-stage0",
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].request_id == "duplex-sid-e0-stage0"
|
||||
assert outputs[0].finished is False
|
||||
assert outputs[0].stage_id == 1
|
||||
assert outputs[0].final_output_type == "audio"
|
||||
assert outputs[0].peak_memory_mb == raw_stage1_output.peak_memory_mb
|
||||
|
||||
|
||||
def test_async_omni_duplex_request_info_includes_response_stage():
|
||||
request_id, response_stage_id = AsyncOmni._duplex_data_plane_request_info(
|
||||
{
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": "duplex-sid-e0-stage0",
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert request_id == "duplex-sid-e0-stage0"
|
||||
assert response_stage_id == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_request_client_retains_output_route_when_close_fails():
|
||||
fence = DuplexFence("sid-close-route")
|
||||
request_id = duplex_resource_request_id(fence, "stage0")
|
||||
request_states = {request_id: ClientRequestState(request_id)}
|
||||
|
||||
async def close_duplex_session_async(session_id, **kwargs):
|
||||
del session_id, kwargs
|
||||
raise RuntimeError("close failed")
|
||||
|
||||
client = DuplexRequestClient(
|
||||
SimpleNamespace(close_duplex_session_async=close_duplex_session_async),
|
||||
SimpleNamespace(request_states=request_states),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="close failed"):
|
||||
await client.close(fence.session_id, reason="test", fence=fence, timeout=1.0)
|
||||
|
||||
assert request_id in request_states
|
||||
@@ -53,7 +53,7 @@ def test_async_engine_consumes_disable_log_stats_before_resolution(monkeypatch:
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.engine.async_omni_engine.load_and_resolve_stage_configs",
|
||||
"vllm_omni.engine.omni_engine_base.load_and_resolve_stage_configs",
|
||||
fake_resolve,
|
||||
)
|
||||
engine = SimpleNamespace(_apply_strategy_lb_policy=lambda *args, **kwargs: None)
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai_api.test_duplex_handler import FakeChatService, TimedWebSocket
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.duplex.protocol import DuplexSession, DuplexSessionConfig
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FenceRecordingEngine:
|
||||
output_modalities = ["text", "audio"]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, DuplexFence | None]] = []
|
||||
self.signal_next_fences: list[DuplexFence | None] = []
|
||||
|
||||
async def open_duplex_session_async(self, session_id, *, fence=None, **kwargs):
|
||||
del session_id, kwargs
|
||||
self.calls.append(("open", fence))
|
||||
return {"ok": True}
|
||||
|
||||
async def append_duplex_input_async(self, session_id, *, fence=None, **kwargs):
|
||||
del session_id, kwargs
|
||||
self.calls.append(("append", fence))
|
||||
return {"ok": True}
|
||||
|
||||
async def signal_duplex_turn_async(
|
||||
self,
|
||||
session_id,
|
||||
*,
|
||||
event,
|
||||
fence=None,
|
||||
next_fence=None,
|
||||
timeout=None,
|
||||
):
|
||||
del session_id, event, timeout
|
||||
self.calls.append(("signal", fence))
|
||||
self.signal_next_fences.append(next_fence)
|
||||
return {"ok": True}
|
||||
|
||||
async def close_duplex_session_async(self, session_id, *, fence=None, **kwargs):
|
||||
del session_id, kwargs
|
||||
self.calls.append(("close", fence))
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_forwards_fence_for_all_runtime_operations():
|
||||
engine = FenceRecordingEngine()
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = engine
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
fence = DuplexFence("sid", epoch=1, turn_id=2)
|
||||
|
||||
await app.open_duplex_session_async("sid", fence=fence)
|
||||
await app.append_duplex_input_async(
|
||||
"sid", mode="append_audio_chunk", payload={}, fence=fence, collect_outputs=False
|
||||
)
|
||||
await app.signal_duplex_turn_async("sid", event="turn.end", fence=fence)
|
||||
await app.close_duplex_session_async("sid", reason="done", fence=fence)
|
||||
|
||||
assert engine.calls == [(operation, fence) for operation in ("open", "append", "signal", "close")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_handler_passes_current_session_fence_for_runtime_controls():
|
||||
engine = FenceRecordingEngine()
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
|
||||
session = DuplexSession(
|
||||
session_id="sid",
|
||||
config=DuplexSessionConfig(extra_body={"native_duplex": True}),
|
||||
epoch=1,
|
||||
turn_id=2,
|
||||
)
|
||||
ws = TimedWebSocket()
|
||||
fence = DuplexFence("sid", epoch=1, turn_id=2)
|
||||
|
||||
await handler._open_runtime_session(session, ws.send_json)
|
||||
await handler._signal_runtime_session(session, "turn.end", send_json=ws.send_json)
|
||||
await handler._close_runtime_session(session, reason="done", send_json=ws.send_json)
|
||||
|
||||
assert engine.calls == [(operation, fence) for operation in ("open", "signal", "close")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_signal_uses_fence_captured_before_session_turn_advances():
|
||||
engine = FenceRecordingEngine()
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
|
||||
session = DuplexSession(session_id="sid", config=DuplexSessionConfig(extra_body={"native_duplex": True}))
|
||||
captured_fence = DuplexFence("sid", epoch=0, turn_id=0)
|
||||
session.turn_id = 1
|
||||
|
||||
await handler._signal_runtime_session(
|
||||
session,
|
||||
"input.commit",
|
||||
fence=captured_fence,
|
||||
)
|
||||
|
||||
assert engine.calls == [("signal", captured_fence)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_handler_forwards_cancel_fence_through_async_omni_facade():
|
||||
engine = FenceRecordingEngine()
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = engine
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(app))
|
||||
session = DuplexSession(session_id="sid-cancel", config=DuplexSessionConfig(extra_body={"native_duplex": True}))
|
||||
cancelled_fence = DuplexFence("sid-cancel", epoch=2, turn_id=3)
|
||||
next_fence = DuplexFence("sid-cancel", epoch=3, turn_id=3)
|
||||
|
||||
assert await handler._signal_runtime_session(
|
||||
session,
|
||||
"input.cancel",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
)
|
||||
|
||||
assert engine.calls == [("signal", cancelled_fence)]
|
||||
assert engine.signal_next_fences == [next_fence]
|
||||
@@ -0,0 +1,326 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``DuplexOmni`` / ``DuplexSessionHandle``: the thin Python API over a fake engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.test_omni_entrypoints import FakeAsyncOmniEngine
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex import commands
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.events import (
|
||||
AudioDelta,
|
||||
DuplexEvent,
|
||||
SessionClosed,
|
||||
SessionCreated,
|
||||
SessionExpired,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
DuplexControlResultMessage,
|
||||
DuplexSessionError,
|
||||
DuplexSessionEventMessage,
|
||||
)
|
||||
from vllm_omni.engine.messages import ErrorMessage
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeDuplexEngine(FakeAsyncOmniEngine):
|
||||
"""The engine surface ``DuplexOmni`` uses; every call is recorded."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.duplex_session_config = DuplexSessionRuntimeConfig(max_sessions=2)
|
||||
self.duplex_capabilities = DuplexCapabilities(supports_session_resume=True)
|
||||
self.opened: list[tuple[str, DuplexSessionConfig]] = []
|
||||
self.closed: list[tuple[str, str]] = []
|
||||
self.resumed: list[tuple[str, int]] = []
|
||||
self.touched: list[tuple[str, str]] = []
|
||||
self.commands: list[tuple[str, commands.DuplexCommand]] = []
|
||||
self.open_error: DuplexSessionError | None = None
|
||||
self.close_error: DuplexSessionError | None = None
|
||||
|
||||
def _result(self, operation: str, session_id: str, **fields: Any) -> DuplexControlResultMessage:
|
||||
return DuplexControlResultMessage(control_id="c", operation=operation, session_id=session_id, ok=True, **fields)
|
||||
|
||||
async def open_session_async(self, session_id, session_config, *, timeout=None):
|
||||
self.opened.append((session_id, session_config))
|
||||
if self.open_error is not None:
|
||||
raise self.open_error
|
||||
return self._result(
|
||||
"open",
|
||||
session_id,
|
||||
capabilities=self.duplex_capabilities,
|
||||
public_session={"id": session_id, "idle_timeout_s": 7},
|
||||
lease_generation=0,
|
||||
)
|
||||
|
||||
async def close_session_async(self, session_id, *, reason="client_close", timeout=None):
|
||||
self.closed.append((session_id, reason))
|
||||
if self.close_error is not None:
|
||||
raise self.close_error
|
||||
# The runner emits session.closed before answering the RPC.
|
||||
self.emit(session_id, SessionClosed(reason=reason))
|
||||
return self._result("close", session_id)
|
||||
|
||||
async def resume_session_async(self, session_id, *, expected_lease_generation, timeout=None):
|
||||
self.resumed.append((session_id, expected_lease_generation))
|
||||
return self._result("resume", session_id, lease_generation=expected_lease_generation + 1)
|
||||
|
||||
async def touch_session_async(self, session_id, *, activity, timeout=None):
|
||||
self.touched.append((session_id, activity))
|
||||
return self._result("touch", session_id)
|
||||
|
||||
async def submit_command_async(self, session_id, command):
|
||||
self.commands.append((session_id, command))
|
||||
|
||||
def emit(self, session_id: str, event: DuplexEvent) -> None:
|
||||
# The session manager binds the session identity to every event it emits.
|
||||
event = replace(event, session_id=session_id)
|
||||
self.output_q.put(DuplexSessionEventMessage(session_id=session_id, event=event))
|
||||
|
||||
|
||||
def _make_omni(monkeypatch: pytest.MonkeyPatch) -> tuple[DuplexOmni, FakeDuplexEngine]:
|
||||
engine = FakeDuplexEngine()
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.duplex_omni.DuplexOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
|
||||
return DuplexOmni("dummy-model"), engine
|
||||
|
||||
|
||||
async def _collect(handle: DuplexSessionHandle, count: int, *, timeout_s: float = 2.0) -> list[DuplexEvent]:
|
||||
events: list[DuplexEvent] = []
|
||||
|
||||
async def consume() -> None:
|
||||
async for event in handle.events():
|
||||
events.append(event)
|
||||
if len(events) >= count:
|
||||
break
|
||||
|
||||
await asyncio.wait_for(consume(), timeout=timeout_s)
|
||||
return events
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_session_allocates_the_id_and_ignores_client_ids(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session({"session_id": "mine", "id": "mine-too", "instructions": "hi"})
|
||||
|
||||
assert handle.session_id.startswith("duplex-") and len(handle.session_id) == len("duplex-") + 32
|
||||
assert handle.session_id != "mine"
|
||||
((session_id, config),) = engine.opened
|
||||
assert session_id == handle.session_id
|
||||
assert isinstance(config, DuplexSessionConfig)
|
||||
assert config.model == "dummy-model" and config.instructions == "hi"
|
||||
assert handle.capabilities is engine.duplex_capabilities
|
||||
assert handle.public_session["id"] == handle.session_id
|
||||
assert omni.sessions == {handle.session_id: handle}
|
||||
assert omni.get_session(handle.session_id) is handle
|
||||
assert omni.active_session_count() == 1
|
||||
|
||||
second = await omni.open_session(DuplexSessionConfig(instructions="typed"))
|
||||
assert second.session_id != handle.session_id
|
||||
assert engine.opened[1][1].model == "dummy-model"
|
||||
assert omni.duplex_session_config.max_sessions == 2
|
||||
assert omni.duplex_capabilities.supports_session_resume is True
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_failure_drops_the_pending_handle_and_raises_the_engine_code(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
engine.open_error = DuplexSessionError("full", code="resource_exhausted", retryable=True)
|
||||
try:
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await omni.open_session()
|
||||
assert excinfo.value.code == "resource_exhausted"
|
||||
assert omni.sessions == {}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_are_routed_to_their_handle_and_end_on_session_closed(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.emit("duplex-unknown", AudioDelta(delta="zzz")) # dropped
|
||||
engine.emit(handle.session_id, SessionCreated(session={"id": handle.session_id}))
|
||||
engine.emit(handle.session_id, AudioDelta(response_id="r1", delta="abc"))
|
||||
engine.emit(handle.session_id, SessionClosed(reason="client_close"))
|
||||
|
||||
events = await _collect(handle, 10)
|
||||
|
||||
assert [event.type for event in events] == ["session.created", "response.output_audio.delta", "session.closed"]
|
||||
assert events[0].session_id == handle.session_id
|
||||
assert handle.closed and handle.close_reason == "client_close"
|
||||
assert await handle.wait_closed() == "client_close"
|
||||
assert handle.session_id not in omni.sessions
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await handle.submit(commands.Heartbeat())
|
||||
assert excinfo.value.code == "session_closed"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_is_single_consumer_but_may_be_reentered(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.emit(handle.session_id, AudioDelta(delta="one"))
|
||||
first = handle.events()
|
||||
assert (await first.__anext__()).type == "response.output_audio.delta"
|
||||
with pytest.raises(RuntimeError, match="active events"):
|
||||
await handle.events().__anext__()
|
||||
await first.aclose()
|
||||
|
||||
engine.emit(handle.session_id, SessionExpired(reason="idle_ttl_expired"))
|
||||
events = await _collect(handle, 1)
|
||||
assert events[0].type == "session.expired" and events[0].is_terminal
|
||||
assert handle.closed and handle.close_reason == "idle_ttl_expired"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_session_waits_for_the_typed_session_closed_event(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
consumer = asyncio.create_task(_collect(handle, 1))
|
||||
await handle.close(reason="done")
|
||||
|
||||
assert engine.closed == [(handle.session_id, "done")]
|
||||
assert handle.closed and handle.close_reason == "done"
|
||||
assert (await consumer)[0].type == "session.closed"
|
||||
# Idempotent: a second close is a no-op.
|
||||
await omni.close_session(handle.session_id)
|
||||
assert len(engine.closed) == 1
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_failure_marks_the_handle_closed_and_propagates(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
engine.close_error = DuplexSessionError("boom", code="engine_error")
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
with pytest.raises(DuplexSessionError, match="boom"):
|
||||
await omni.close_session(handle.session_id)
|
||||
assert handle.closed
|
||||
# A failed close must not leave a dead handle shadowing the id.
|
||||
assert omni.sessions == {}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_death_ends_every_session_handle(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.output_q.put(ErrorMessage(error="orchestrator died", fatal=True))
|
||||
|
||||
await asyncio.wait_for(handle.wait_closed(), timeout=2.0)
|
||||
assert handle.closed
|
||||
assert "engine_dead" in (handle.close_reason or "")
|
||||
assert omni.sessions == {}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_detach_and_touch_forward_to_the_engine(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
resumed = await omni.resume_session(handle.session_id, expected_lease_generation=0)
|
||||
assert resumed is handle and handle.lease_generation == 1
|
||||
assert engine.resumed == [(handle.session_id, 0)]
|
||||
|
||||
await omni.detach_session(handle.session_id)
|
||||
await omni.touch_session(handle.session_id, activity="heartbeat")
|
||||
assert engine.touched == [(handle.session_id, "detach"), (handle.session_id, "heartbeat")]
|
||||
|
||||
for call in (
|
||||
omni.resume_session("duplex-nope", expected_lease_generation=0),
|
||||
omni.detach_session("duplex-nope"),
|
||||
):
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await call
|
||||
assert excinfo.value.code == "unknown_session"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_wrappers_build_typed_commands_in_caller_order(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
await handle.append_audio(b"\x00\x01", sample_rate_hz=16000, is_speech=True)
|
||||
await handle.commit(create_response=True)
|
||||
await handle.create_response({"instructions": "x"})
|
||||
await handle.cancel_response("resp-1")
|
||||
await handle.barge_in()
|
||||
await handle.ack_playback(120.7, response_id="resp-1", committed_ms=100)
|
||||
await handle.update({"temperature": 0.2})
|
||||
await handle.create_item({"id": "item_1", "type": "message"}, previous_item_id="root")
|
||||
await handle.delete_item("item_1")
|
||||
await handle.truncate_item("item_1", audio_end_ms=50)
|
||||
await handle.signal_turn("user_started", {"x": 1})
|
||||
await handle.heartbeat()
|
||||
|
||||
submitted = [command for session_id, command in engine.commands if session_id == handle.session_id]
|
||||
assert [type(command) for command in submitted] == [
|
||||
commands.AppendAudio,
|
||||
commands.Commit,
|
||||
commands.CreateResponse,
|
||||
commands.CancelResponse,
|
||||
commands.BargeIn,
|
||||
commands.AckPlayback,
|
||||
commands.UpdateSession,
|
||||
commands.CreateItem,
|
||||
commands.DeleteItem,
|
||||
commands.TruncateItem,
|
||||
commands.SignalTurn,
|
||||
commands.Heartbeat,
|
||||
]
|
||||
append, commit, create, cancel, _, ack, update, item, delete, truncate, signal, _ = submitted
|
||||
assert append.audio == b"\x00\x01" and append.sample_rate_hz == 16000 and append.is_speech is True
|
||||
assert commit.create_response is True and commit.final is True
|
||||
assert create.options == {"instructions": "x"}
|
||||
assert cancel.response_id == "resp-1"
|
||||
assert ack.played_ms == 120 and ack.committed_ms == 100 and ack.response_id == "resp-1"
|
||||
assert update.patch == {"temperature": 0.2}
|
||||
assert item.previous_item_id == "root" and item.item["id"] == "item_1"
|
||||
assert delete.item_id == "item_1"
|
||||
assert truncate.audio_end_ms == 50
|
||||
assert signal.event == "user_started" and signal.signal_payload == {"x": 1}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_closes_every_handle(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
first = await omni.open_session()
|
||||
second = await omni.open_session()
|
||||
|
||||
omni.shutdown()
|
||||
|
||||
assert first.closed and second.closed
|
||||
assert first.close_reason == "shutdown"
|
||||
assert omni.sessions == {}
|
||||
assert engine.shutdown_called
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Unit tests for OmniBase and AsyncOmni profiler methods."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -28,10 +31,14 @@ class TestOmniBaseProfiler:
|
||||
@pytest.fixture
|
||||
def omni_base_instance(self, mock_engine, mocker: MockerFixture):
|
||||
"""Create an OmniBase instance with mocked dependencies."""
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", return_value=mock_engine)
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
# ``OmniBase`` builds its engine through the ``_create_engine`` seam that
|
||||
# ``AsyncOmni`` / ``DuplexOmni`` implement; there is no module-level
|
||||
# engine class to patch any more.
|
||||
mocker.patch.object(OmniBase, "_create_engine", return_value=mock_engine)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", side_effect=lambda x: x)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.weakref.finalize")
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
instance = OmniBase(model="test-model")
|
||||
return instance
|
||||
@@ -48,10 +55,14 @@ class TestOmniBaseProfiler:
|
||||
|
||||
def test_removed_diffusion_batch_size_fails_loudly(self, mock_engine, mocker: MockerFixture):
|
||||
"""Legacy diffusion_batch_size callers receive an actionable error."""
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", return_value=mock_engine)
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
# ``OmniBase`` builds its engine through the ``_create_engine`` seam that
|
||||
# ``AsyncOmni`` / ``DuplexOmni`` implement; there is no module-level
|
||||
# engine class to patch any more.
|
||||
mocker.patch.object(OmniBase, "_create_engine", return_value=mock_engine)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", side_effect=lambda x: x)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.weakref.finalize")
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
with pytest.raises(TypeError, match="diffusion_batch_size.*max_num_seqs"):
|
||||
OmniBase(model="test-model", diffusion_batch_size=8)
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import queue
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
@@ -15,6 +17,8 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.config.stage_config import StageConfig
|
||||
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
|
||||
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
|
||||
@@ -117,8 +121,6 @@ class FakeAsyncOmniEngine:
|
||||
self.output_processors = [SimpleNamespace(tokenizer=None) for _ in range(self.num_stages)]
|
||||
self.input_processor = None
|
||||
self.endpoint_restrictions = ()
|
||||
self.duplex_session_config = None
|
||||
self.duplex_serving_adapter_path = None
|
||||
|
||||
self.output_q: queue.Queue[Any] = queue.Queue()
|
||||
self.submitted: list[dict[str, Any]] = []
|
||||
@@ -166,7 +168,8 @@ class FakeAsyncOmniEngine:
|
||||
def abort(self, request_ids: list[str]) -> None:
|
||||
self.aborted.append(list(request_ids))
|
||||
|
||||
async def abort_async(self, request_ids: list[str]) -> None:
|
||||
async def abort_async(self, request_ids: list[str], timeout=None) -> None:
|
||||
del timeout
|
||||
self.abort(request_ids)
|
||||
|
||||
async def collective_rpc_async(self, **_: Any) -> list[Any]:
|
||||
@@ -181,7 +184,8 @@ class FakeAsyncOmniEngine:
|
||||
|
||||
|
||||
def _patch_engine(monkeypatch: pytest.MonkeyPatch, engine: FakeAsyncOmniEngine) -> None:
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni.AsyncOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.async_omni.AsyncOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
|
||||
# Don't add random UUIDs to requests calling .generate since we usually
|
||||
# just want to check for present requests anyway, and would need to just
|
||||
@@ -200,33 +204,75 @@ def _make_base():
|
||||
return obj
|
||||
|
||||
|
||||
def test_resolve_sampling_params_list_preserves_stage_constraints():
|
||||
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||
def test_resolve_sampling_params_list_preserves_stage_constraints(typed):
|
||||
from vllm_omni.config.omni_config import VllmOmniARStageConfig
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
|
||||
base = _make_base()
|
||||
base.engine.num_stages = 1
|
||||
base.default_sampling_params_list = [SamplingParams(max_tokens=1000, detokenize=False, stop_token_ids=[42])]
|
||||
base.engine.stage_configs = [
|
||||
StageConfig(
|
||||
if typed:
|
||||
stage = VllmOmniARStageConfig(
|
||||
stage_pipeline_config=StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="dummy-model",
|
||||
sampling_constraints={"detokenize": False, "stop_token_ids": [42]},
|
||||
)
|
||||
)
|
||||
else:
|
||||
stage = StageConfig(
|
||||
stage_id=0,
|
||||
model_stage="dummy-model",
|
||||
sampling_constraints={"detokenize": False, "stop_token_ids": [42]},
|
||||
).to_omegaconf()
|
||||
]
|
||||
base.engine.stage_configs = [stage]
|
||||
base.sampling_constraints_list = base._get_sampling_constraints_list(base.engine.stage_configs)
|
||||
assert base.sampling_constraints_list == [{"detokenize": False, "stop_token_ids": [42]}]
|
||||
caller_params = SamplingParams(seed=1234, max_tokens=7, detokenize=True, stop_token_ids=[7])
|
||||
|
||||
resolved_defaults = base.resolve_sampling_params_list(None)
|
||||
assert resolved_defaults[0].stop_token_ids == [42]
|
||||
|
||||
caller_params = SamplingParams(seed=1234, max_tokens=7, detokenize=True, stop_token_ids=[7])
|
||||
resolved = base.resolve_sampling_params_list(caller_params)
|
||||
|
||||
assert resolved[0] is not caller_params
|
||||
assert resolved[0].seed == 1234
|
||||
assert resolved[0].max_tokens == 7
|
||||
assert resolved[0].detokenize is False
|
||||
assert resolved[0].stop_token_ids == [42]
|
||||
assert resolved[0].stop_token_ids == [7, 42]
|
||||
assert 7 in resolved[0]._all_stop_token_ids
|
||||
assert 42 in resolved[0]._all_stop_token_ids
|
||||
assert caller_params.detokenize is True
|
||||
assert caller_params.stop_token_ids == [7]
|
||||
|
||||
|
||||
def test_resolve_sampling_params_list_merges_required_stop_tokens():
|
||||
base = _make_base()
|
||||
base.engine.num_stages = 1
|
||||
required_stop_ids = [151704, 151645]
|
||||
base.default_sampling_params_list = [
|
||||
SamplingParams(max_tokens=1000, detokenize=False, stop_token_ids=[99, *required_stop_ids])
|
||||
]
|
||||
base.engine.stage_configs = [
|
||||
StageConfig(
|
||||
stage_id=0,
|
||||
model_stage="dummy-model",
|
||||
sampling_constraints={"detokenize": False, "stop_token_ids": required_stop_ids},
|
||||
).to_omegaconf()
|
||||
]
|
||||
base.sampling_constraints_list = base._get_sampling_constraints_list(base.engine.stage_configs)
|
||||
assert base.sampling_constraints_list == [{"detokenize": False, "stop_token_ids": required_stop_ids}]
|
||||
|
||||
|
||||
def test_sampling_constraints_are_forwarded_by_typed_stage_configs():
|
||||
config = VllmOmniConfig.from_pipeline_config(OMNI_PIPELINES["qwen3_tts"])
|
||||
|
||||
constraints = OmniBase._get_sampling_constraints_list(config.stage_configs)
|
||||
|
||||
assert constraints == [dict(stage.stage_pipeline_config.sampling_constraints) for stage in config.stage_configs]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_defaults", [False, True])
|
||||
def test_moss_local_output_policy_preserves_codec_streaming(use_defaults):
|
||||
from vllm_omni.model_executor.models.moss_tts.pipeline import MOSS_TTS_LOCAL_PIPELINE
|
||||
@@ -625,6 +671,43 @@ async def test_async_omni_llm_diffusion_yields_text_stream_then_image(monkeypatc
|
||||
assert "req-1" not in app.request_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_abort_yields_terminal_from_requested_final_stage(monkeypatch: pytest.MonkeyPatch):
|
||||
stage_metadata = [
|
||||
_stage_meta(stage_type="llm", final_output=False, final_output_type=None),
|
||||
_stage_meta(stage_type="llm", final_output=True, final_output_type="audio"),
|
||||
]
|
||||
engine = FakeAsyncOmniEngine(stage_metadata=stage_metadata)
|
||||
_patch_engine(monkeypatch, engine)
|
||||
app = AsyncOmni("dummy-model")
|
||||
|
||||
async def collect_outputs() -> list[OmniRequestOutput]:
|
||||
return [
|
||||
output
|
||||
async for output in app.generate(
|
||||
prompt="hello",
|
||||
request_id="req-1",
|
||||
output_modalities=["audio"],
|
||||
)
|
||||
]
|
||||
|
||||
try:
|
||||
generate_task = asyncio.create_task(collect_outputs())
|
||||
while not engine.submitted:
|
||||
await asyncio.sleep(0)
|
||||
await app.abort("req-1")
|
||||
outputs = await asyncio.wait_for(generate_task, timeout=1)
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].stage_id == 1
|
||||
assert outputs[0].final_output_type == "audio"
|
||||
assert outputs[0].finished is True
|
||||
assert outputs[0].outputs[0].finish_reason == "abort"
|
||||
assert "req-1" not in app.request_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_abort_forwards_to_engine(monkeypatch: pytest.MonkeyPatch):
|
||||
engine = FakeAsyncOmniEngine(stage_metadata=THREE_STAGE_META)
|
||||
@@ -758,7 +841,7 @@ def test_omni_generate_py_generator_yields_final_outputs_for_each_request(monkey
|
||||
f"{engine.submitted[1]['request_id']}-stage0-0",
|
||||
f"{engine.submitted[1]['request_id']}-stage2-final",
|
||||
]
|
||||
assert engine.shutdown_called is True
|
||||
assert engine.shutdown_called is not True
|
||||
|
||||
|
||||
def test_omni_generate_returns_list_when_not_using_generator(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -804,7 +887,7 @@ def test_omni_generate_diffusion_only_yields_single_image_per_request(monkeypatc
|
||||
[f"{engine.submitted[0]['request_id']}-image"],
|
||||
[f"{engine.submitted[1]['request_id']}-image"],
|
||||
]
|
||||
assert engine.shutdown_called is True
|
||||
assert engine.shutdown_called is not True
|
||||
|
||||
|
||||
def test_omni_generate_llm_diffusion_yields_final_text_then_image_per_request(
|
||||
@@ -837,7 +920,7 @@ def test_omni_generate_llm_diffusion_yields_final_text_then_image_per_request(
|
||||
[f"{engine.submitted[1]['request_id']}-image"],
|
||||
]
|
||||
assert engine.submitted[0]["sampling_params_list"][0].output_kind == RequestOutputKind.FINAL_ONLY
|
||||
assert engine.shutdown_called is True
|
||||
assert engine.shutdown_called is not True
|
||||
|
||||
|
||||
def test_omni_abort_forwards_to_engine(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -1318,3 +1401,57 @@ def test_omni_errored_property_dead_stage(monkeypatch: pytest.MonkeyPatch):
|
||||
assert app.errored is False
|
||||
finally:
|
||||
app.shutdown()
|
||||
|
||||
|
||||
def test_omni_pygenerator_does_not_kill_engine(monkeypatch: pytest.MonkeyPatch):
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=_enqueue_async_three_stage_outputs,
|
||||
)
|
||||
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
# Evaluating the generator should not shut down the engine
|
||||
list(app.generate(["hello"], py_generator=True))
|
||||
assert not engine.shutdown_called
|
||||
|
||||
|
||||
def test_omni_generator_close_cleans_up(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Ensure that a closed generator cleans things up properly."""
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=_enqueue_async_three_stage_outputs,
|
||||
)
|
||||
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
|
||||
# Create a generator and start to evaluate it to make sure the request isn't aborted yet
|
||||
my_gen = app.generate(["hello"], py_generator=True)
|
||||
next(my_gen)
|
||||
request_id = engine.submitted[0]["request_id"]
|
||||
assert engine.aborted == []
|
||||
assert request_id in app.request_states
|
||||
|
||||
# Close it and make sure the sure it's aborted, but without killing engine
|
||||
my_gen.close()
|
||||
assert engine.aborted == [[request_id]]
|
||||
assert request_id not in app.request_states
|
||||
assert not engine.shutdown_called
|
||||
|
||||
|
||||
def test_del_shutsdown_engine(monkeypatch: pytest.MonkeyPatch):
|
||||
engine = FakeAsyncOmniEngine(
|
||||
stage_metadata=THREE_STAGE_META,
|
||||
on_add_request=_enqueue_async_three_stage_outputs,
|
||||
)
|
||||
|
||||
_patch_engine(monkeypatch, engine)
|
||||
|
||||
app = Omni("dummy-model")
|
||||
assert not engine.shutdown_called
|
||||
del app
|
||||
gc.collect()
|
||||
assert engine.shutdown_called
|
||||
@@ -18,6 +18,7 @@ import torch
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -98,16 +99,6 @@ MODEL = "ByteDance-Seed/BAGEL-7B-MoT"
|
||||
MODEL_DIFF = "riverclouds/qwen_image_random"
|
||||
|
||||
|
||||
def get_dynamic_devices(stage_idx: int, num_stages: int, tp_size: int) -> str:
|
||||
total_gpus = torch.accelerator.device_count()
|
||||
gpus_per_stage = tp_size
|
||||
start_idx = stage_idx * gpus_per_stage
|
||||
if start_idx + gpus_per_stage > total_gpus:
|
||||
start_idx = start_idx % total_gpus
|
||||
device_ids = [str(start_idx + i) for i in range(gpus_per_stage)]
|
||||
return ",".join(device_ids)
|
||||
|
||||
|
||||
async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None:
|
||||
"""Best-effort full wake + resume so the next shared-engine test starts active."""
|
||||
try:
|
||||
@@ -120,33 +111,28 @@ async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None:
|
||||
logger.warning("ensure_resume failed (stage_ids=%s): %s", stage_ids, e)
|
||||
|
||||
|
||||
def _build_llm_stages() -> tuple[list[dict], list[dict]]:
|
||||
common_args = {
|
||||
"worker_type": "ar",
|
||||
"enable_sleep_mode": True,
|
||||
"dtype": "bfloat16",
|
||||
"trust_remote_code": True,
|
||||
"max_model_len": 2048,
|
||||
"max_num_batched_tokens": 8192,
|
||||
"enforce_eager": True,
|
||||
}
|
||||
stages = [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"stage_type": "llm",
|
||||
"runtime": {"process": True, "devices": "0", "max_batch_size": 1},
|
||||
"engine_args": {**common_args, "model_stage": "thinker", "gpu_memory_utilization": 0.1},
|
||||
def _sleep_deploy_config(*, diffusion_only: bool = False, tp_size: int = 1) -> str:
|
||||
"""Use registered BAGEL topology and override only deployment settings."""
|
||||
filename = "bagel_single_stage.yaml" if diffusion_only else "bagel.yaml"
|
||||
stage_ids = (0,) if diffusion_only else (0, 1)
|
||||
return modify_stage_config(
|
||||
get_deploy_config_path(filename),
|
||||
updates={
|
||||
"stages": {
|
||||
stage_id: {
|
||||
"devices": ",".join(str(stage_id * tp_size + rank) for rank in range(tp_size)),
|
||||
"tensor_parallel_size": tp_size,
|
||||
"enable_sleep_mode": True,
|
||||
"enforce_eager": True,
|
||||
"dtype": "bfloat16",
|
||||
# Stages use disjoint GPUs. On an 80 GiB H100, 40% cannot
|
||||
# fit TP=1 Thinker weights, activation peaks and KV cache.
|
||||
"gpu_memory_utilization": 0.8 if not diffusion_only and stage_id == 0 else 0.4,
|
||||
}
|
||||
for stage_id in stage_ids
|
||||
}
|
||||
},
|
||||
{
|
||||
"stage_id": 1,
|
||||
"stage_type": "llm",
|
||||
"engine_input_source": [0],
|
||||
"runtime": {"process": True, "devices": "1", "max_batch_size": 1, "connector_type": "queue"},
|
||||
"engine_args": {**common_args, "model_stage": "talker", "gpu_memory_utilization": 0.1},
|
||||
},
|
||||
]
|
||||
connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
|
||||
return stages, connectors
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
@@ -163,11 +149,10 @@ def _module_device_cleanup():
|
||||
|
||||
@pytest_asyncio.fixture(scope="class", loop_scope="class")
|
||||
async def llm_engine():
|
||||
"""Shared 2-stage BAGEL LLM engine for sleep/wake protocol tests."""
|
||||
"""Shared BAGEL Thinker + DiT engine for LLM sleep/wake protocol tests."""
|
||||
if current_omni_platform.is_rocm():
|
||||
clean_device_envs()
|
||||
stages, connectors = _build_llm_stages()
|
||||
engine = AsyncOmni(model=MODEL, stages=stages, connectors=connectors, init_timeout=600, enable_sleep_mode=True)
|
||||
engine = AsyncOmni(model=MODEL, deploy_config=_sleep_deploy_config(), init_timeout=600, enable_sleep_mode=True)
|
||||
yield engine
|
||||
engine.shutdown()
|
||||
await asyncio.sleep(1.5)
|
||||
@@ -178,27 +163,12 @@ async def diffusion_engine():
|
||||
"""Shared BAGEL diffusion TP=2 engine for sleep/wake + generate checks."""
|
||||
if current_omni_platform.is_rocm():
|
||||
clean_device_envs()
|
||||
stages = [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"stage_type": "diffusion",
|
||||
"runtime": {"process": True, "devices": "0,1", "max_batch_size": 1},
|
||||
"engine_args": {
|
||||
"model_stage": "base",
|
||||
"gpu_memory_utilization": 0.1,
|
||||
"model_class_name": "BagelPipeline",
|
||||
"enable_sleep_mode": True,
|
||||
"enforce_eager": True,
|
||||
"max_num_batched_tokens": 8192,
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 2,
|
||||
},
|
||||
},
|
||||
"final_output": True,
|
||||
"final_output_type": "image",
|
||||
}
|
||||
]
|
||||
engine = AsyncOmni(model=MODEL, stages=stages, init_timeout=600, enable_sleep_mode=True)
|
||||
engine = AsyncOmni(
|
||||
model=MODEL,
|
||||
deploy_config=_sleep_deploy_config(diffusion_only=True, tp_size=2),
|
||||
init_timeout=600,
|
||||
enable_sleep_mode=True,
|
||||
)
|
||||
yield engine
|
||||
engine.shutdown()
|
||||
await asyncio.sleep(1.5)
|
||||
@@ -208,7 +178,7 @@ class TestOmniLlmSleepMode:
|
||||
"""LLM sleep/wake protocol tests sharing one class-scoped engine."""
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="class")
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
async def test_llm_sleep_ack(self, llm_engine: AsyncOmni):
|
||||
"""LLM Thinker (GPU0) Signal and Physical Recycling Audit"""
|
||||
device_id = 0
|
||||
@@ -333,11 +303,10 @@ class TestOmniDiffusionSleepMode:
|
||||
# --- integrity: baseline generate at original resolution/steps ---
|
||||
prompt = "A huge swimming pool, with many people swimming."
|
||||
sp = OmniDiffusionSamplingParams(num_inference_steps=4, height=512, width=512, seed=42)
|
||||
llm_sp = SamplingParams()
|
||||
|
||||
logger.info("Running Baseline Generation...")
|
||||
base_output = None
|
||||
async for output in diffusion_engine.generate(prompt, request_id="base", sampling_params_list=[llm_sp, sp]):
|
||||
async for output in diffusion_engine.generate(prompt, request_id="base", sampling_params=sp):
|
||||
base_output = output
|
||||
assert base_output is not None and len(base_output.images) > 0
|
||||
logger.info("Baseline Generation successful.")
|
||||
@@ -384,7 +353,7 @@ class TestOmniDiffusionSleepMode:
|
||||
# --- integrity + lifecycle: post-wake generate ---
|
||||
logger.info("Running Post-Wakeup Generation...")
|
||||
post_output = None
|
||||
async for output in diffusion_engine.generate(prompt, request_id="post", sampling_params_list=[llm_sp, sp]):
|
||||
async for output in diffusion_engine.generate(prompt, request_id="post", sampling_params=sp):
|
||||
post_output = output
|
||||
assert post_output is not None
|
||||
assert len(base_output.images) == len(post_output.images)
|
||||
@@ -458,41 +427,22 @@ class TestOmniCoordinatedSleepMode:
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.asyncio
|
||||
async def test_multistage_llm_diffusion_sleep_wake(tp_size: int):
|
||||
"""Explicit 2-stage (llm + diffusion) + connectors; sleep/wake both stages."""
|
||||
"""Registered BAGEL Thinker + DiT topology; sleep/wake both stages."""
|
||||
if current_omni_platform.is_rocm():
|
||||
clean_device_envs()
|
||||
num_gpus = torch.accelerator.device_count()
|
||||
if num_gpus < tp_size * 2:
|
||||
pytest.skip("Not enough GPUs")
|
||||
|
||||
stages = []
|
||||
for i in range(2):
|
||||
devs = get_dynamic_devices(i, 2, tp_size)
|
||||
stages.append(
|
||||
{
|
||||
"stage_id": i,
|
||||
"stage_type": "llm" if i == 0 else "diffusion",
|
||||
"runtime": {"process": True, "devices": devs},
|
||||
"engine_args": {
|
||||
"model": MODEL,
|
||||
"model_stage": "thinker" if i == 0 else "base",
|
||||
"tensor_parallel_size": tp_size,
|
||||
"gpu_memory_utilization": 0.4,
|
||||
"dtype": "bfloat16",
|
||||
"enable_sleep_mode": True,
|
||||
"trust_remote_code": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
connectors = [{"src_stage_id": 0, "dst_stage_id": 1, "connector_type": "queue"}]
|
||||
|
||||
engine = AsyncOmni(
|
||||
model=MODEL, stages=stages, connectors=connectors, enable_sleep_mode=True, stage_init_timeout=1200
|
||||
model=MODEL,
|
||||
deploy_config=_sleep_deploy_config(tp_size=tp_size),
|
||||
enable_sleep_mode=True,
|
||||
stage_init_timeout=1200,
|
||||
)
|
||||
try:
|
||||
sp = OmniDiffusionSamplingParams(num_inference_steps=2)
|
||||
async for _ in engine.generate("warmup", sampling_params_list=[SamplingParams(), sp]):
|
||||
async for _ in engine.generate("warmup", sampling_params_list=[engine.default_sampling_params_list[0], sp]):
|
||||
pass
|
||||
|
||||
acks = await engine.sleep(stage_ids=[0, 1], level=1)
|
||||
@@ -500,7 +450,7 @@ async def test_multistage_llm_diffusion_sleep_wake(tp_size: int):
|
||||
|
||||
await engine.wake_up(stage_ids=[0, 1])
|
||||
await engine.resume_generation(stage_ids=[0, 1])
|
||||
async for _ in engine.generate("verify", sampling_params_list=[SamplingParams(), sp]):
|
||||
async for _ in engine.generate("verify", sampling_params_list=[engine.default_sampling_params_list[0], sp]):
|
||||
pass
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
@@ -32,15 +32,18 @@ def test_dreamzero_resolves_through_registry_with_model_defaults(monkeypatch):
|
||||
stage_overrides=None,
|
||||
strategy_config_path=None,
|
||||
)
|
||||
engine_args = resolved.stage_configs[0].engine_args
|
||||
stage = resolved.stage_configs[0]
|
||||
|
||||
assert resolved.config_path is not None
|
||||
assert resolved.config_path.endswith("vllm_omni/deploy/dreamzero.yaml")
|
||||
assert engine_args.model_class_name == "DreamZeroPipeline"
|
||||
assert engine_args.model_config.policy_server_config.action_space == "joint_position"
|
||||
assert stage.diffusion_config.model_class_name == "DreamZeroPipeline"
|
||||
assert stage.diffusion_config.model_config["policy_server_config"]["action_space"] == "joint_position"
|
||||
|
||||
|
||||
def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch):
|
||||
def _get_hf_config(path, _model):
|
||||
return None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.config.get_hf_file_to_dict",
|
||||
lambda path, _model, **_kwargs: (
|
||||
|
||||
@@ -6,12 +6,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.v1.engine.utils import EngineZmqAddresses
|
||||
|
||||
from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig
|
||||
from vllm_omni.config.resolver import OmniConfigResolution
|
||||
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
|
||||
from vllm_omni.engine.stage_runtime import StageEngineLaunch
|
||||
from vllm_omni.entrypoints.cli.serve import (
|
||||
OmniServeCommand,
|
||||
_parse_stage_overrides,
|
||||
@@ -23,6 +30,31 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingName
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeProcess:
|
||||
sentinel: str
|
||||
exitcode: int | None
|
||||
name: str
|
||||
pid: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeProcessOwner:
|
||||
processes: list[_FakeProcess]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeStageEngineArgs:
|
||||
async_chunk: bool
|
||||
enable_sleep_mode: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeStageConfig:
|
||||
stage_id: int
|
||||
engine_args: _FakeStageEngineArgs
|
||||
|
||||
|
||||
def _resolved(*stages: SimpleNamespace) -> OmniConfigResolution:
|
||||
return OmniConfigResolution(config_path="/fake/stages.yaml", stage_configs=tuple(stages))
|
||||
|
||||
@@ -42,6 +74,113 @@ def test_serve_parser_accepts_no_async_chunk_and_marks_it_explicit() -> None:
|
||||
explicit = args.get_explicit_kwargs_dict()
|
||||
assert args.get_explicit_kwargs_dict()
|
||||
assert not explicit["async_chunk"]
|
||||
assert (args.api_server_count or 1) == 1
|
||||
assert "api_server_count" not in explicit
|
||||
|
||||
|
||||
def _parse_serve_args(argv: list[str]) -> TrackingNamespace:
|
||||
"""Parse a ``serve`` argv through the real Omni parser, returning the
|
||||
TrackingNamespace (with ``explicit_keys``) that ``validate`` receives."""
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def test_omni_serve_requires_model_when_none_provided() -> None:
|
||||
"""Regression for https://github.com/vllm-project/vllm-omni/issues/4158:
|
||||
``vllm serve --omni`` with no model must fail fast instead of silently
|
||||
falling back to vLLM's default model and crashing in the diffusion worker.
|
||||
"""
|
||||
args = _parse_serve_args(["serve", "--omni"])
|
||||
cmd = OmniServeCommand()
|
||||
with pytest.raises(ValueError, match="requires an explicit model"):
|
||||
cmd.validate(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
# A deploy YAML supplied *without* an explicit model must still trip
|
||||
# the guard. It carries per-stage engine args only -- the
|
||||
# checkpoint is always threaded in from ``args.model`` -- so its mere
|
||||
# presence does not establish that a model was provided. Without an
|
||||
# explicit model ``args.model`` stays at vLLM's default and the launch
|
||||
# reproduces the issue-4158 diffusion-registry crash.
|
||||
["serve", "--omni", "--deploy-config", "deploy.yaml"],
|
||||
# An empty/whitespace model must not satisfy the guard either -- this is
|
||||
# the ``--model "$MODEL"``-with-unset-var footgun. Otherwise the empty
|
||||
# value flows downstream and crashes with the same confusing error.
|
||||
["serve", "", "--omni"], # empty positional
|
||||
["serve", "--omni", "--model", ""], # empty --model
|
||||
["serve", "--omni", "--model", " "], # whitespace-only --model
|
||||
],
|
||||
)
|
||||
def test_omni_serve_rejects_missing_or_empty_model(argv: list[str]) -> None:
|
||||
"""Regression for review feedback on PR #4167: a deploy YAML is not a
|
||||
model source, so ``--deploy-config`` alone must not bypass the
|
||||
require-a-model guard; neither may an empty/whitespace model value."""
|
||||
args = _parse_serve_args(argv)
|
||||
cmd = OmniServeCommand()
|
||||
with pytest.raises(ValueError, match="requires an explicit model"):
|
||||
cmd.validate(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"argv",
|
||||
[
|
||||
["serve", "fake-model", "--omni"], # positional model
|
||||
["serve", "--omni", "--model", "fake-model"], # --model flag
|
||||
# An explicit model with a deploy YAML is the legitimate multi-stage
|
||||
# path and must pass the guard.
|
||||
["serve", "fake-model", "--omni", "--deploy-config", "deploy.yaml"],
|
||||
],
|
||||
)
|
||||
def test_omni_serve_accepts_explicit_model(argv: list[str], mocker: MockerFixture) -> None:
|
||||
"""A model supplied positionally or via ``--model`` -- with or without a
|
||||
deploy YAML -- must NOT trip the require-a-model guard. Downstream
|
||||
validation is mocked so only the new guard is exercised."""
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.utils.hf_utils.is_diffusion_model",
|
||||
return_value=False,
|
||||
)
|
||||
mocker.patch("vllm_omni.entrypoints.cli.serve.validate_parsed_serve_args")
|
||||
|
||||
args = _parse_serve_args(argv)
|
||||
cmd = OmniServeCommand()
|
||||
# Must not raise the "requires a model" error for any of these.
|
||||
cmd.validate(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_class,exists,native",
|
||||
[
|
||||
("AnimaPipeline", True, True),
|
||||
("AnimaModularPipeline", True, True),
|
||||
("AnimaPipeline", False, False),
|
||||
("FluxPipeline", True, False),
|
||||
(None, True, False),
|
||||
],
|
||||
)
|
||||
def test_serve_native_checkpoint_validation(model_class, exists, native, tmp_path, mocker):
|
||||
checkpoint = tmp_path / "anima.safetensors"
|
||||
if exists:
|
||||
checkpoint.touch()
|
||||
argv = ["serve", str(checkpoint), "--omni"]
|
||||
if model_class is not None:
|
||||
argv.extend(["--model-class-name", model_class])
|
||||
args = _parse_serve_args(argv)
|
||||
detect_model = mocker.patch("vllm_omni.diffusion.utils.hf_utils.is_diffusion_model", return_value=False)
|
||||
validate_llm = mocker.patch("vllm_omni.entrypoints.cli.serve.validate_parsed_serve_args")
|
||||
|
||||
OmniServeCommand().validate(args)
|
||||
|
||||
if native:
|
||||
detect_model.assert_not_called()
|
||||
validate_llm.assert_not_called()
|
||||
else:
|
||||
detect_model.assert_called_once_with(str(checkpoint))
|
||||
validate_llm.assert_called_once_with(args)
|
||||
|
||||
|
||||
def test_serve_parser_accepts_strategy_config() -> None:
|
||||
@@ -69,6 +208,277 @@ def test_serve_parser_accepts_deploy_config() -> None:
|
||||
assert args.get_explicit_kwargs_dict()["deploy_config"] == "/tmp/deploy.yaml"
|
||||
|
||||
|
||||
def test_serve_parser_accepts_video_output_transport() -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
|
||||
args = parser.parse_args(
|
||||
["serve", "fake-model", "--omni", "--video-output-transport", '{"enable_device_postprocess": true}']
|
||||
)
|
||||
|
||||
expected = {"enable_device_postprocess": True}
|
||||
assert args.video_output_transport == expected
|
||||
assert args.get_explicit_kwargs_dict()["video_output_transport"] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["{not json", "[]"])
|
||||
def test_serve_parser_rejects_invalid_video_output_transport(value: str) -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["serve", "fake-model", "--omni", "--video-output-transport", value])
|
||||
|
||||
|
||||
def test_tracking_namespace_is_picklable_for_spawned_api_workers() -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
args = parser.parse_args(["serve", "fake-model", "--omni", "--api-server-count", "2"])
|
||||
args._omni_stage_client_configs = [{"stage_addresses": {0: {0: {"input_address": "ipc://input"}}}}]
|
||||
|
||||
restored = ForkingPickler.loads(ForkingPickler.dumps(args))
|
||||
|
||||
assert restored.api_server_count == 2
|
||||
assert restored.get_explicit_kwargs_dict()["api_server_count"] == 2
|
||||
assert restored._omni_stage_client_configs == args._omni_stage_client_configs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native_checkpoint", [False, True])
|
||||
def test_serve_validate_rejects_multiple_api_servers_for_diffusion(
|
||||
mocker: MockerFixture, tmp_path, native_checkpoint: bool
|
||||
) -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
cmd = OmniServeCommand()
|
||||
cmd.subparser_init(subparsers)
|
||||
argv = ["serve", "fake-diffusion-model", "--omni", "--api-server-count", "2"]
|
||||
if native_checkpoint:
|
||||
checkpoint = tmp_path / "anima.safetensors"
|
||||
checkpoint.touch()
|
||||
argv[1] = str(checkpoint)
|
||||
argv.extend(["--model-class-name", "AnimaPipeline"])
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
mocker.patch("vllm_omni.diffusion.utils.hf_utils.is_diffusion_model", return_value=not native_checkpoint)
|
||||
validate = mocker.patch("vllm_omni.entrypoints.cli.serve.validate_parsed_serve_args")
|
||||
|
||||
with pytest.raises(ValueError, match="not supported for diffusion"):
|
||||
cmd.validate(args)
|
||||
|
||||
validate.assert_not_called()
|
||||
|
||||
|
||||
def test_serve_validate_rejects_sleep_mode_with_multiple_api_servers(mocker: MockerFixture) -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
cmd = OmniServeCommand()
|
||||
cmd.subparser_init(subparsers)
|
||||
args = parser.parse_args(["serve", "fake-model", "--omni", "--api-server-count", "2", "--enable-sleep-mode"])
|
||||
|
||||
mocker.patch("vllm_omni.diffusion.utils.hf_utils.is_diffusion_model", return_value=False)
|
||||
mocker.patch("vllm_omni.entrypoints.cli.serve.validate_parsed_serve_args")
|
||||
|
||||
with pytest.raises(ValueError, match="enable-sleep-mode"):
|
||||
cmd.validate(args)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True])
|
||||
@pytest.mark.parametrize("downstream_async_chunk", [False, True])
|
||||
def test_build_multi_api_stage_runtime_matches_current_constructor(
|
||||
mocker: MockerFixture, typed: bool, downstream_async_chunk: bool
|
||||
) -> None:
|
||||
from vllm_omni.entrypoints.cli import serve as serve_module
|
||||
|
||||
args = TrackingNamespace(
|
||||
argparse.Namespace(
|
||||
model="dummy-model",
|
||||
model_tag=None,
|
||||
stage_init_timeout=1,
|
||||
tokenizer=None,
|
||||
disable_log_stats=False,
|
||||
),
|
||||
frozenset({"model"}),
|
||||
)
|
||||
if typed:
|
||||
from vllm_omni.config.omni_config import VllmOmniARStageConfig
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
|
||||
stages = [
|
||||
VllmOmniARStageConfig(stage_pipeline_config=StagePipelineConfig(stage_id=i, model_stage="ar"))
|
||||
for i in range(2)
|
||||
]
|
||||
stages[1].connector_config.async_chunk = downstream_async_chunk
|
||||
else:
|
||||
stages = [
|
||||
_FakeStageConfig(
|
||||
stage_id=i,
|
||||
engine_args=_FakeStageEngineArgs(
|
||||
async_chunk=bool(i and downstream_async_chunk), enable_sleep_mode=False
|
||||
),
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", return_value="dummy-model")
|
||||
mocker.patch(
|
||||
"vllm_omni.config.config_factory.with_trust_remote_code_override",
|
||||
side_effect=lambda kwargs, _trust: kwargs,
|
||||
)
|
||||
mocker.patch("vllm_omni.entrypoints.utils.parse_stage_overrides", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver.resolve_omni_config",
|
||||
return_value=OmniConfigResolution(config_path="dummy-config", stage_configs=tuple(stages)),
|
||||
)
|
||||
|
||||
runtime = serve_module._build_multi_api_stage_runtime(args, 2)
|
||||
|
||||
assert runtime._client_count == 1
|
||||
assert runtime._client_index == 0
|
||||
assert runtime._async_chunk is downstream_async_chunk
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True])
|
||||
def test_build_multi_api_stage_runtime_rejects_sleep_enabled_in_stage_config(
|
||||
mocker: MockerFixture, typed: bool
|
||||
) -> None:
|
||||
from vllm_omni.entrypoints.cli import serve as serve_module
|
||||
|
||||
args = TrackingNamespace(
|
||||
argparse.Namespace(
|
||||
model="dummy-model",
|
||||
model_tag=None,
|
||||
stage_init_timeout=1,
|
||||
tokenizer=None,
|
||||
disable_log_stats=False,
|
||||
),
|
||||
frozenset({"model"}),
|
||||
)
|
||||
stage_config = _FakeStageConfig(
|
||||
stage_id=3,
|
||||
engine_args=_FakeStageEngineArgs(async_chunk=False, enable_sleep_mode=True),
|
||||
)
|
||||
if typed:
|
||||
from vllm_omni.config.omni_config import VllmOmniARStageConfig
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
|
||||
typed_stage = VllmOmniARStageConfig(stage_pipeline_config=StagePipelineConfig(stage_id=3, model_stage="ar"))
|
||||
typed_stage.model_config.enable_sleep_mode = True
|
||||
stage_config = typed_stage
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", return_value="dummy-model")
|
||||
mocker.patch(
|
||||
"vllm_omni.config.config_factory.with_trust_remote_code_override",
|
||||
side_effect=lambda kwargs, _trust: kwargs,
|
||||
)
|
||||
mocker.patch("vllm_omni.entrypoints.utils.parse_stage_overrides", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver.resolve_omni_config",
|
||||
return_value=OmniConfigResolution(config_path="dummy-config", stage_configs=(stage_config,)),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=r"sleep mode.*stage\(s\) \[3\]"):
|
||||
serve_module._build_multi_api_stage_runtime(args, 2)
|
||||
|
||||
|
||||
def test_run_multi_api_server_omni_starts_workers_after_shared_engine_launch(mocker: MockerFixture) -> None:
|
||||
from vllm_omni.entrypoints.cli import serve as serve_module
|
||||
|
||||
args = TrackingNamespace(
|
||||
argparse.Namespace(api_server_count=2, shutdown_timeout=1),
|
||||
frozenset({"api_server_count", "shutdown_timeout"}),
|
||||
)
|
||||
socket = mocker.Mock()
|
||||
primary_addresses = EngineZmqAddresses(
|
||||
inputs=["ipc://input-0", "ipc://input-1"],
|
||||
outputs=["ipc://output-0", "ipc://output-1"],
|
||||
)
|
||||
engine_launch = StageEngineLaunch(
|
||||
client_configs=[{"client_count": 2, "client_index": index, "stage_addresses": {}} for index in range(2)],
|
||||
resources=[StageReplicaResources(addresses=primary_addresses)],
|
||||
)
|
||||
runtime = mocker.MagicMock()
|
||||
runtime.launch_stage_engines.return_value.__enter__.return_value = engine_launch
|
||||
runtime.launch_stage_engines.return_value.__exit__.return_value = False
|
||||
|
||||
manager = mocker.Mock()
|
||||
manager.processes = [mocker.Mock()]
|
||||
manager.gather_actual_addresses.return_value = (primary_addresses.inputs, primary_addresses.outputs)
|
||||
|
||||
mocker.patch.object(serve_module, "_build_multi_api_stage_runtime", return_value=runtime)
|
||||
mocker.patch.object(serve_module, "_wait_for_multi_api_server_completion")
|
||||
mocker.patch("signal.signal")
|
||||
mocker.patch("vllm.entrypoints.openai.api_server.setup_server", return_value=("127.0.0.1:8000", socket))
|
||||
mocker.patch("vllm.v1.metrics.prometheus.setup_multiprocess_prometheus")
|
||||
start_manager = mocker.patch.object(serve_module, "_start_api_server_process_manager", return_value=manager)
|
||||
|
||||
serve_module.run_multi_api_server_omni(args)
|
||||
|
||||
assert args._omni_stage_client_configs is engine_launch.client_configs
|
||||
start_manager.assert_called_once()
|
||||
manager_kwargs = start_manager.call_args.kwargs
|
||||
assert manager_kwargs["cleanup_timeout"] == 1
|
||||
assert manager_kwargs["num_servers"] == 2
|
||||
assert manager_kwargs["input_addresses"] == ["ipc://input-0", "ipc://input-1"]
|
||||
assert manager_kwargs["target_server_fn"] is serve_module.run_omni_api_server_worker_proc
|
||||
assert engine_launch.watched_frontend_processes == manager.processes
|
||||
runtime.shutdown.assert_called_once_with()
|
||||
socket.close.assert_called_once_with()
|
||||
|
||||
|
||||
def test_start_api_server_process_manager_cleans_up_partial_start(mocker: MockerFixture) -> None:
|
||||
from vllm_omni.entrypoints.cli import serve as serve_module
|
||||
|
||||
first_process = mocker.Mock(name="first_process")
|
||||
second_process = mocker.Mock(name="second_process")
|
||||
first_process.pid = 1234
|
||||
second_process.pid = None
|
||||
second_process.start.side_effect = RuntimeError("spawn failed")
|
||||
parent_pipes = [mocker.Mock(name="parent_pipe_0"), mocker.Mock(name="parent_pipe_1")]
|
||||
child_pipes = [mocker.Mock(name="child_pipe_0"), mocker.Mock(name="child_pipe_1")]
|
||||
spawn_context = mocker.Mock()
|
||||
spawn_context.Process.side_effect = [first_process, second_process]
|
||||
spawn_context.Pipe.side_effect = list(zip(parent_pipes, child_pipes))
|
||||
mocker.patch("vllm.v1.utils.multiprocessing.get_context", return_value=spawn_context)
|
||||
shutdown = mocker.patch("vllm.v1.utils.shutdown")
|
||||
|
||||
with pytest.raises(RuntimeError, match="spawn failed"):
|
||||
serve_module._start_api_server_process_manager(
|
||||
cleanup_timeout=1.5,
|
||||
listen_address="127.0.0.1:8000",
|
||||
sock=mocker.Mock(),
|
||||
args=argparse.Namespace(),
|
||||
num_servers=2,
|
||||
input_addresses=["ipc://input-0", "ipc://input-1"],
|
||||
output_addresses=["ipc://output-0", "ipc://output-1"],
|
||||
target_server_fn=mocker.Mock(),
|
||||
)
|
||||
|
||||
first_process.start.assert_called_once_with()
|
||||
second_process.start.assert_called_once_with()
|
||||
for pipe in parent_pipes:
|
||||
pipe.close.assert_called_once_with()
|
||||
shutdown.assert_called_once_with([first_process], timeout=1.5)
|
||||
|
||||
|
||||
def test_wait_for_multi_api_server_completion_rejects_engine_failure(
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
from vllm_omni.entrypoints.cli import serve as serve_module
|
||||
|
||||
api_process = _FakeProcess(sentinel="api-0", exitcode=None, name="api-0", pid=10)
|
||||
engine_process = _FakeProcess(sentinel="engine-0", exitcode=4, name="engine-0", pid=20)
|
||||
manager = _FakeProcessOwner(processes=[api_process])
|
||||
engine_launch = StageEngineLaunch(
|
||||
client_configs=[],
|
||||
resources=[StageReplicaResources(manager=_FakeProcessOwner(processes=[engine_process]))],
|
||||
)
|
||||
mocker.patch("multiprocessing.connection.wait", return_value=["engine-0"])
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"Shared stage engine process engine-0 .* exited with code 4"):
|
||||
serve_module._wait_for_multi_api_server_completion(manager, engine_launch)
|
||||
|
||||
|
||||
def test_serve_parser_rejects_stage_configs_path() -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
@@ -88,6 +498,26 @@ def test_serve_parser_accepts_four_way_cfg_parallelism() -> None:
|
||||
assert args.cfg_parallel_size == 4
|
||||
|
||||
|
||||
def test_serve_parser_accepts_robot_openpi_idle_timeout() -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
|
||||
args = parser.parse_args(["serve", "fake-model", "--omni", "--robot-openpi-idle-timeout", "0"])
|
||||
|
||||
assert args.robot_openpi_idle_timeout == 0
|
||||
assert args.get_explicit_kwargs_dict()["robot_openpi_idle_timeout"] == 0
|
||||
|
||||
|
||||
def test_serve_parser_rejects_negative_robot_openpi_idle_timeout() -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
OmniServeCommand().subparser_init(subparsers)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["serve", "fake-model", "--omni", "--robot-openpi-idle-timeout", "-1"])
|
||||
|
||||
|
||||
def test_serve_parser_accepts_ulysses_a2a_permute() -> None:
|
||||
parser = TrackingArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="subcommand")
|
||||
@@ -216,6 +646,26 @@ def test_parse_stage_overrides_invalid_json_raises() -> None:
|
||||
assert f"Got: {bad!r}" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
['{"abc": {}}', '{"-1": {}}', '{"1.5": {}}', '{"\uff10": {}}'],
|
||||
)
|
||||
def test_parse_stage_overrides_rejects_invalid_stage_ids(payload: str) -> None:
|
||||
with pytest.raises(argparse.ArgumentTypeError, match="non-negative integer stage ids"):
|
||||
_parse_stage_overrides(payload)
|
||||
|
||||
|
||||
def test_parse_stage_overrides_preserves_arbitrary_override_fields() -> None:
|
||||
parsed = _parse_stage_overrides(
|
||||
'{"0": {"extras": {"ltx2_use_conv_vae": true}, "kv_cache_dtype": "fp8", "typo_field_xyz": 1}}'
|
||||
)
|
||||
assert parsed["0"] == {
|
||||
"extras": {"ltx2_use_conv_vae": True},
|
||||
"kv_cache_dtype": "fp8",
|
||||
"typo_field_xyz": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_parse_stage_overrides_rejects_non_dict_top_level() -> None:
|
||||
"""Top-level must be a JSON object (dict). A list, scalar, or non-dict
|
||||
mapping is rejected with a ValueError naming ``--stage-overrides`` and
|
||||
@@ -588,6 +1038,77 @@ def test_run_headless_diffusion_registers_and_spawns_proc(mocker: MockerFixture)
|
||||
assert manager_kwargs["omni_replica_id"] == 0
|
||||
|
||||
|
||||
def test_run_headless_generic_diffusion_launches_structured_stage(mocker: MockerFixture) -> None:
|
||||
"""Headless resolution starts the typed stage through the real group launcher."""
|
||||
from vllm_omni.engine import stage_engine_startup as startup_module
|
||||
|
||||
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||
return_value=(True, "FakeDiffusionPipeline"),
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||
mocker.patch.object(startup_module.stage_init_utils, "load_omni_transfer_config_for_model", return_value=None)
|
||||
mocker.patch.object(
|
||||
startup_module.initialization,
|
||||
"resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
captured: dict[str, Any] = {}
|
||||
od_config = SimpleNamespace()
|
||||
|
||||
def _build_diffusion_config(model, stage_config, metadata):
|
||||
captured.update(model=model, stage_config=stage_config, metadata=metadata)
|
||||
return od_config
|
||||
|
||||
def _launch_replica_group(**kwargs):
|
||||
captured.update(group_kwargs=kwargs)
|
||||
captured["manager"] = kwargs["launch_one"](0)
|
||||
|
||||
def _launch_replica(**kwargs):
|
||||
captured.update(replica_kwargs=kwargs)
|
||||
return SimpleNamespace(exitcode=None)
|
||||
|
||||
mocker.patch.object(startup_module.stage_init_utils, "build_diffusion_config", side_effect=_build_diffusion_config)
|
||||
mocker.patch.object(startup_module, "launch_headless_replica_group", side_effect=_launch_replica_group)
|
||||
mocker.patch.object(startup_module, "launch_headless_diffusion_replica", side_effect=_launch_replica)
|
||||
|
||||
explicit_keys = frozenset(
|
||||
{
|
||||
"model",
|
||||
"stage_id",
|
||||
"omni_master_address",
|
||||
"omni_master_port",
|
||||
"worker_backend",
|
||||
"model_class_name",
|
||||
"num_gpus",
|
||||
}
|
||||
)
|
||||
args = _make_headless_args(
|
||||
explicit_keys=explicit_keys,
|
||||
model="generic-diffusion",
|
||||
model_class_name="FakeDiffusionPipeline",
|
||||
num_gpus=1,
|
||||
)
|
||||
|
||||
run_headless(args)
|
||||
|
||||
stage = captured["stage_config"]
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert captured["metadata"].stage_type == "diffusion"
|
||||
assert captured["metadata"].model_stage == "diffusion"
|
||||
assert captured["model"] == "generic-diffusion"
|
||||
assert captured["group_kwargs"]["stage_id"] == 0
|
||||
assert captured["group_kwargs"]["omni_dp_size_local"] == 1
|
||||
assert captured["group_kwargs"]["per_replica_devices"] == ["0"]
|
||||
assert captured["replica_kwargs"]["stage_config"] is stage
|
||||
assert captured["replica_kwargs"]["stage_id"] == 0
|
||||
assert captured["replica_kwargs"]["omni_master_address"] == "127.0.0.1"
|
||||
assert captured["replica_kwargs"]["omni_master_port"] == 26000
|
||||
assert captured["replica_kwargs"]["od_config"] is od_config
|
||||
assert stage.runtime_config.devices == "0"
|
||||
|
||||
|
||||
def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixture) -> None:
|
||||
"""A diffusion replica that exits with a non-zero code must surface as a
|
||||
RuntimeError from ``run_headless`` (the head needs the signal to roll
|
||||
@@ -636,3 +1157,22 @@ def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixtur
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"exited with code 137"):
|
||||
run_headless(_make_headless_args(stage_id=1))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["unfiltered_ns", "explicit_keys", "missing", "__dict__"])
|
||||
def test_tracking_namespace_uninitialized_access_raises_attribute_error(name):
|
||||
namespace = object.__new__(TrackingNamespace)
|
||||
with pytest.raises(AttributeError):
|
||||
getattr(namespace, name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deep", [False, True])
|
||||
def test_tracking_namespace_copy_preserves_tracking(deep):
|
||||
import copy
|
||||
|
||||
namespace = TrackingNamespace(
|
||||
argparse.Namespace(model="example", api_server_count=2), frozenset({"api_server_count"})
|
||||
)
|
||||
restored = copy.deepcopy(namespace) if deep else copy.copy(namespace)
|
||||
assert restored.get_explicit_kwargs_dict() == {"api_server_count": 2}
|
||||
assert restored.model == "example"
|
||||
+206
-50
@@ -13,6 +13,14 @@ import torch
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
|
||||
from vllm_omni.config.composable_parallel import (
|
||||
Broadcast,
|
||||
FanInByStage,
|
||||
MeshAxisSpec,
|
||||
RouteByStage,
|
||||
StrategySpec,
|
||||
TakeRank,
|
||||
)
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.config.resolver import (
|
||||
OmniConfigResolution,
|
||||
@@ -21,7 +29,6 @@ from vllm_omni.config.resolver import (
|
||||
resolve_omni_config,
|
||||
)
|
||||
from vllm_omni.config.stage_config import PipelineConfig
|
||||
from vllm_omni.config.yaml_util import create_config
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
from vllm_omni.entrypoints.utils import (
|
||||
@@ -296,6 +303,70 @@ class TestFilterDataclassKwargs:
|
||||
|
||||
|
||||
class TestResolveOmniConfig:
|
||||
@pytest.mark.parametrize("model_class_name", ["AnimaPipeline", "AnimaModularPipeline"])
|
||||
def test_native_anima_checkpoint_uses_default_diffusion_stage_without_model_config(
|
||||
self, tmp_path, mocker: MockerFixture, model_class_name: str
|
||||
):
|
||||
checkpoint = tmp_path / "anima.safetensors"
|
||||
checkpoint.write_text("dummy")
|
||||
custom_args = {"components_path": "/tmp/anima-components"}
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
||||
side_effect=AssertionError("native Anima checkpoints should not require model config discovery"),
|
||||
)
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
str(checkpoint),
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=None,
|
||||
cli_overrides={"model_class_name": model_class_name, "custom_pipeline_args": custom_args},
|
||||
stage_overrides=None,
|
||||
strategy_config_path=None,
|
||||
)
|
||||
|
||||
assert resolved.config_path is None
|
||||
assert len(resolved.stage_configs) == 1
|
||||
stage_config = resolved.stage_configs[0]
|
||||
assert stage_config.model_stage == "diffusion"
|
||||
assert stage_config.diffusion_config.model_class_name == "AnimaPipeline"
|
||||
assert stage_config.diffusion_config.custom_pipeline_args == custom_args
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_class_name", "checkpoint_exists", "deploy_config_path"),
|
||||
[
|
||||
("AnimaPipeline", True, "deploy.yaml"),
|
||||
("AnimaPipeline", False, None),
|
||||
("FluxPipeline", True, None),
|
||||
],
|
||||
)
|
||||
def test_native_single_file_fallback_preserves_config_discovery(
|
||||
self, tmp_path, mocker: MockerFixture, model_class_name, checkpoint_exists, deploy_config_path
|
||||
):
|
||||
checkpoint = tmp_path / "model.safetensors"
|
||||
if checkpoint_exists:
|
||||
checkpoint.write_text("dummy")
|
||||
create_from_model = mocker.patch(
|
||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None
|
||||
)
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
str(checkpoint),
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=deploy_config_path,
|
||||
cli_overrides={"model_class_name": model_class_name},
|
||||
stage_overrides=None,
|
||||
strategy_config_path=None,
|
||||
)
|
||||
|
||||
create_from_model.assert_called_once_with(
|
||||
str(checkpoint),
|
||||
trust_remote_code=False,
|
||||
cli_overrides={"model_class_name": model_class_name, "trust_remote_code": False},
|
||||
deploy_config_path=deploy_config_path,
|
||||
strategy_specs=None,
|
||||
)
|
||||
assert resolved.config_path == deploy_config_path
|
||||
|
||||
def test_bare_deploy_name_returns_packaged_resolved_path(self, tmp_path, mocker: MockerFixture):
|
||||
deploy_name = "qwen3_omni_moe.yaml"
|
||||
deploy_path = tmp_path / deploy_name
|
||||
@@ -356,16 +427,12 @@ class TestResolveOmniConfig:
|
||||
)
|
||||
assert resolved.config_path is None
|
||||
assert len(resolved.stage_configs) == 1
|
||||
engine_args = resolved.stage_configs[0]["engine_args"]
|
||||
assert "dtype" in engine_args
|
||||
assert engine_args["engine_backend"] == engine_backend
|
||||
assert engine_args["revision"] == "pinned-revision"
|
||||
assert resolved.stage_configs[0].diffusion_config.dtype is torch.float32
|
||||
assert resolved.stage_configs[0].diffusion_config.engine_backend == engine_backend
|
||||
assert resolved.stage_configs[0].diffusion_config.revision == "pinned-revision"
|
||||
|
||||
def test_generic_diffusion_uses_registered_model_metadata(self, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
||||
return_value=None,
|
||||
)
|
||||
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||
resolve_model_class = mocker.patch(
|
||||
"vllm_omni.config.resolver.resolve_model_class_name",
|
||||
return_value="WanImageToVideoPipeline",
|
||||
@@ -374,59 +441,98 @@ class TestResolveOmniConfig:
|
||||
"vllm_omni.config.resolver.DiffusionModelRegistry.get_supported_archs",
|
||||
return_value={"WanImageToVideoPipeline"},
|
||||
)
|
||||
load_model_class = mocker.patch("vllm_omni.config.resolver.DiffusionModelRegistry._try_load_model_cls")
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
"/models/Wan2.2-I2V",
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=None,
|
||||
cli_overrides={
|
||||
"diffusion_load_format": "diffusers",
|
||||
"revision": "pinned-revision",
|
||||
},
|
||||
cli_overrides={"diffusion_load_format": "diffusers", "revision": "pinned-revision"},
|
||||
stage_overrides=None,
|
||||
strategy_config_path=None,
|
||||
)
|
||||
|
||||
resolve_model_class.assert_called_once_with("/models/Wan2.2-I2V", "diffusers", "pinned-revision")
|
||||
load_model_class.assert_not_called()
|
||||
stage = resolved.stage_configs[0]
|
||||
resolve_model_class.assert_called_once_with(
|
||||
"/models/Wan2.2-I2V",
|
||||
"diffusers",
|
||||
"pinned-revision",
|
||||
)
|
||||
assert stage.engine_args.model_class_name == "WanImageToVideoPipeline"
|
||||
assert stage.diffusion_config.model_class_name == "WanImageToVideoPipeline"
|
||||
assert stage.diffusion_config.revision == "pinned-revision"
|
||||
assert stage.final_output_type == "video"
|
||||
|
||||
def test_generic_diffusion_stage_overrides_reach_typed_backend(self, mocker: MockerFixture):
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||
|
||||
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||
return_value=(True, "LTX2Pipeline"),
|
||||
)
|
||||
resolved = resolve_omni_config(
|
||||
"/models/LTX-2.5-Diffusers",
|
||||
trust_remote_code=False,
|
||||
deploy_config_path=None,
|
||||
cli_overrides={
|
||||
"extras": {"ltx2_use_conv_vae": False, "keep": "global"},
|
||||
"tensor_parallel_size": 1,
|
||||
"diffusion_streaming_output": True,
|
||||
},
|
||||
stage_overrides={
|
||||
"0": {
|
||||
"extras": {"ltx2_use_conv_vae": True},
|
||||
"tensor_parallel_size": 2,
|
||||
"devices": "2,3",
|
||||
"engine_backend": "custom.backend",
|
||||
},
|
||||
},
|
||||
strategy_config_path=None,
|
||||
)
|
||||
stage = resolved.stage_by_id(0)
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(stage, model="/models/LTX-2.5-Diffusers")
|
||||
|
||||
assert stage.runtime_config.devices == "2,3"
|
||||
assert stage.parallel_config.world_size == 2
|
||||
assert engine_args["engine_backend"] == "custom.backend"
|
||||
assert engine_args["extras"]["ltx2_use_conv_vae"] is True
|
||||
assert engine_args["extras"]["keep"] == "global"
|
||||
assert engine_args["streaming_output"] is True
|
||||
|
||||
def test_registered_pipeline_uses_structured_metadata_and_preserves_override_trust(self, mocker: MockerFixture):
|
||||
endpoint_restriction = SimpleNamespace(name="chat")
|
||||
typed_stage = SimpleNamespace(stage_id=1)
|
||||
structured_config = SimpleNamespace(
|
||||
orchestrator_config=SimpleNamespace(deploy_config_path="/resolved/deploy.yaml"),
|
||||
orchestrator_config=SimpleNamespace(
|
||||
deploy_config_path="/resolved/deploy.yaml",
|
||||
omni_lb_policy="round_robin",
|
||||
),
|
||||
pipeline_config=SimpleNamespace(endpoint_restrictions=(endpoint_restriction,)),
|
||||
stage_configs=(typed_stage,),
|
||||
strategy_omni_lb_policy="round-robin",
|
||||
)
|
||||
runtime_stage = create_config(
|
||||
{
|
||||
"stage_id": 1,
|
||||
"runtime": {"devices": "1,2,3", "num_replicas": 3},
|
||||
"engine_args": {"model": "dummy-model"},
|
||||
}
|
||||
)
|
||||
legacy_stage = SimpleNamespace(to_omegaconf=lambda: runtime_stage)
|
||||
create_structured = mocker.patch(
|
||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
||||
return_value=structured_config,
|
||||
)
|
||||
create_legacy = mocker.patch(
|
||||
"vllm_omni.config.resolver.StageConfigFactory._resolve_legacy_from_registry",
|
||||
return_value=SimpleNamespace(
|
||||
stage_configs=[legacy_stage],
|
||||
pipeline_config=structured_config.pipeline_config,
|
||||
omni_lb_policy="round_robin",
|
||||
),
|
||||
"vllm_omni.config.resolver.StageConfigFactory.create_legacy_stage_configs_from_model",
|
||||
)
|
||||
strategy_specs = {"stage_1": {"dp": 3}}
|
||||
strategy_specs = {
|
||||
"stage_1": [
|
||||
StrategySpec(
|
||||
"stage_replica",
|
||||
MeshAxisSpec("stage_replica", 2),
|
||||
RouteByStage("round_robin"),
|
||||
FanInByStage(),
|
||||
)
|
||||
]
|
||||
}
|
||||
load_strategy = mocker.patch(
|
||||
"vllm_omni.config.resolver._load_strategy_specs",
|
||||
return_value=strategy_specs,
|
||||
)
|
||||
apply_strategy = mocker.patch(
|
||||
"vllm_omni.config.composable_parallel.apply_strategy_specs",
|
||||
return_value=SimpleNamespace(omni_lb_policy="round_robin"),
|
||||
)
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
"dummy-model",
|
||||
@@ -447,31 +553,63 @@ class TestResolveOmniConfig:
|
||||
trust_remote_code=None,
|
||||
cli_overrides=expected_overrides,
|
||||
deploy_config_path="deploy.yaml",
|
||||
)
|
||||
create_legacy.assert_called_once_with(
|
||||
structured_config.pipeline_config,
|
||||
expected_overrides,
|
||||
"/resolved/deploy.yaml",
|
||||
strategy_specs=strategy_specs,
|
||||
)
|
||||
create_legacy.assert_not_called()
|
||||
load_strategy.assert_called_once_with("strategy.yaml")
|
||||
apply_strategy.assert_not_called()
|
||||
assert resolved.config_path == "/resolved/deploy.yaml"
|
||||
assert resolved.pipeline_config is structured_config.pipeline_config
|
||||
assert resolved.omni_lb_policy == "round_robin"
|
||||
assert resolved.omni_lb_policy == "round-robin"
|
||||
assert resolved.endpoint_restrictions == (endpoint_restriction,)
|
||||
assert resolved.stage_configs == (runtime_stage,)
|
||||
assert resolved.stage_configs == (typed_stage,)
|
||||
|
||||
def test_registered_resolution_exposes_forced_aligner_topology(self, mocker: MockerFixture):
|
||||
pipeline = OMNI_PIPELINES["qwen3_tts"]
|
||||
assert isinstance(pipeline, PipelineConfig)
|
||||
def test_tp_only_strategy_does_not_report_default_lb_policy_as_derived(self, mocker: MockerFixture):
|
||||
structured_config = SimpleNamespace(
|
||||
orchestrator_config=SimpleNamespace(deploy_config_path=None),
|
||||
pipeline_config=pipeline,
|
||||
orchestrator_config=SimpleNamespace(
|
||||
deploy_config_path="/resolved/deploy.yaml",
|
||||
omni_lb_policy="round-robin",
|
||||
),
|
||||
pipeline_config=SimpleNamespace(endpoint_restrictions=()),
|
||||
stage_configs=(SimpleNamespace(stage_id=0),),
|
||||
strategy_omni_lb_policy=None,
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
||||
return_value=structured_config,
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.resolver._load_strategy_specs",
|
||||
return_value={
|
||||
"thinker": [
|
||||
StrategySpec(
|
||||
"tp",
|
||||
MeshAxisSpec("tp", 2),
|
||||
Broadcast(),
|
||||
TakeRank(),
|
||||
)
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
"dummy-model",
|
||||
trust_remote_code=True,
|
||||
deploy_config_path="deploy.yaml",
|
||||
cli_overrides={"omni_lb_policy": "round-robin"},
|
||||
stage_overrides=None,
|
||||
strategy_config_path="strategy.yaml",
|
||||
)
|
||||
|
||||
assert resolved.omni_lb_policy is None
|
||||
|
||||
def test_registered_resolution_exposes_forced_aligner_topology(self, mocker: MockerFixture):
|
||||
pipeline = OMNI_PIPELINES["qwen3_tts"]
|
||||
assert isinstance(pipeline, PipelineConfig)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.config_factory.StageConfigFactory.get_pipeline_config",
|
||||
return_value=pipeline,
|
||||
)
|
||||
|
||||
resolved = resolve_omni_config(
|
||||
"Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
@@ -483,9 +621,9 @@ class TestResolveOmniConfig:
|
||||
)
|
||||
|
||||
assert resolved.pipeline_config is not None
|
||||
pipeline_stage_ids = [stage.stage_id for stage in resolved.pipeline_config.stages]
|
||||
runtime_stage_ids = [stage.stage_id for stage in resolved.stage_configs]
|
||||
assert pipeline_stage_ids == runtime_stage_ids
|
||||
assert [stage.stage_id for stage in resolved.pipeline_config.stages] == [
|
||||
stage.stage_id for stage in resolved.stage_configs
|
||||
]
|
||||
assert resolved.pipeline_config.stages[-1].model_stage == "forced_aligner"
|
||||
assert len(resolved.pipeline_config.stages) == len(pipeline.stages) + 1
|
||||
|
||||
@@ -529,3 +667,21 @@ class TestCumulativeStreamingCoercion:
|
||||
assert all([isinstance(r, SamplingParams) for r in result])
|
||||
assert result[0].output_kind == RequestOutputKind.DELTA
|
||||
assert result[1].output_kind == RequestOutputKind.DELTA
|
||||
|
||||
|
||||
@pytest.mark.parametrize("api_server_count", [1, 2])
|
||||
def test_prepare_stage_config_inputs_filters_frontend_options(api_server_count):
|
||||
from vllm_omni.entrypoints.utils import prepare_stage_config_inputs
|
||||
|
||||
kwargs = {
|
||||
"api_server_count": api_server_count,
|
||||
"disable_log_stats": True,
|
||||
"model_tag": "example",
|
||||
"max_num_seqs": 8,
|
||||
"stage_overrides": '{"0": {"max_num_seqs": 4}}',
|
||||
}
|
||||
inputs = prepare_stage_config_inputs("example", kwargs, trust_remote_code=None)
|
||||
|
||||
assert inputs.kwargs == {"max_num_seqs": 8}
|
||||
assert inputs.stage_overrides == {"0": {"max_num_seqs": 4}}
|
||||
assert kwargs["api_server_count"] == api_server_count
|
||||
Reference in new issue
Block a user