[BugFix] Optimize turn-based /v1/realtime auto truncation (#8566)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Nick CaoandCodex authored and GitHub committed 2026-10-07 15:57:09 -04:00
1 parent f3743ebdb6
commit 744c202d86
4 files changed
+601 -138

No files matched your search

@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import base64
import io
import json
@@ -12,6 +13,8 @@ from typing import Any
import numpy as np
import pytest
from openai.types import realtime as types
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import RequestOutputKind, SamplingParams
from tests.helpers.serving_chat import build_serving_chat
@@ -20,7 +23,10 @@ from vllm_omni.entrypoints.openai.realtime.connection import (
OpenAIFullDuplexConnection,
_ResolvedResponse,
)
from vllm_omni.entrypoints.openai.realtime.session import ActiveResponse
from vllm_omni.entrypoints.openai.realtime.session import (
ActiveResponse,
AudioFullDuplexSessionState,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -31,9 +37,13 @@ class _FakeRenderer:
def __init__(self) -> None:
self.raw_prompt_lengths: list[int] = []
self.message_counts: list[int] = []
self.skip_mm_cache: list[bool] = []
self.tokenization_params: list[Any] = []
self.engine_inputs: list[dict[str, Any]] = []
self.conversations: list[list[dict[str, Any]]] = []
self.token_counts: dict[int, int] = {}
self.errors: dict[int, VLLMValidationError] = {}
async def render_chat_async(
self,
@@ -41,12 +51,24 @@ class _FakeRenderer:
_chat_params: Any,
tok_params: Any,
):
message_count = len(conversations[0])
self.message_counts.append(message_count)
self.conversations.append(conversations[0])
raw_prompt_length = 2 * len(conversations[0])
self.raw_prompt_lengths.append(raw_prompt_length)
self.tokenization_params.append(tok_params)
error = self.errors.get(message_count)
if error is not None:
raise error
# Model-side expansion makes the rendered prompt much longer than its text tokens.
engine_input = {"prompt_token_ids": [0] * (10 + 20 * raw_prompt_length)}
token_count = self.token_counts.get(
message_count,
10 + 20 * raw_prompt_length,
)
engine_input = {
"prompt_token_ids": [0] * token_count,
"message_count": message_count,
}
self.engine_inputs.append(engine_input)
return conversations, [engine_input]
@@ -70,9 +92,8 @@ def _make_connection(
model_config = SimpleNamespace(max_model_len=max_model_len, multimodal_config=None)
async def preprocess_chat(request: Any, messages: list[dict[str, Any]], **kwargs: Any):
tok_params = kwargs.get("tok_params")
if tok_params is None:
tok_params = request.build_tok_params(model_config)
tok_params = request.build_tok_params(model_config)
renderer.skip_mm_cache.append(bool(kwargs.get("skip_mm_cache", False)))
return await renderer.render_chat_async(
[messages],
kwargs.get("default_template_kwargs"),
@@ -97,22 +118,79 @@ def _make_connection(
return connection, renderer
def _message_item(
item_id: str,
text: str | None = None,
) -> types.RealtimeConversationItemUserMessage:
return types.RealtimeConversationItemUserMessage(
id=item_id,
type="message",
role="user",
content=[{"type": "input_text", "text": text or item_id}],
)
def _audio_item(item_id: str, audio: bytes) -> types.RealtimeConversationItemUserMessage:
return types.RealtimeConversationItemUserMessage(
id=item_id,
type="message",
role="user",
content=[
{
"type": "input_audio",
"audio": base64.b64encode(audio).decode("ascii"),
}
],
)
def _assistant_item(item_id: str, text: str) -> types.RealtimeConversationItemAssistantMessage:
return types.RealtimeConversationItemAssistantMessage(
id=item_id,
type="message",
role="assistant",
status="completed",
content=[{"type": "output_text", "text": text}],
)
def _websocket_events(websocket: _FakeWebSocket) -> list[dict[str, Any]]:
return [json.loads(message) for message in websocket.messages]
def _prompt_texts(conversation: list[dict[str, Any]]) -> list[str]:
texts: list[str] = []
for message in conversation:
content = message["content"]
if isinstance(content, str):
texts.append(content)
continue
texts.extend(part["text"] for part in content if part.get("type") == "text")
return texts
async def _create_response(
connection: OpenAIFullDuplexConnection,
) -> list[dict[str, Any]]:
submitted: list[dict[str, Any]] = []
async def run_response(_response_id: str, _response: Any, engine_input: dict[str, Any]) -> None:
submitted.append(engine_input)
connection.session.active_response = None
connection._run_response = run_response
await connection._handle_response_create(SimpleNamespace(event_id="evt_create", response=None))
if connection._response_task is not None:
await connection._response_task
return submitted
@pytest.mark.asyncio
async def test_preflight_truncates_by_rendered_token_count() -> None:
connection, renderer = _make_connection()
first = SimpleNamespace(
id="first",
type="message",
role="user",
content=[SimpleNamespace(type="input_text", text="first")],
)
second = SimpleNamespace(
id="second",
type="message",
role="user",
content=[SimpleNamespace(type="input_text", text="second")],
)
first = _message_item("first")
second = _message_item("second")
items = [first, second]
response = _ResolvedResponse(
input=items,
@@ -124,16 +202,405 @@ async def test_preflight_truncates_by_rendered_token_count() -> None:
metadata=None,
)
engine_input = await connection._prepare_engine_input_with_auto_truncation(response)
prompt_items = await connection._truncate_prompt_items(response)
engine_input = await connection._build_full_prompt(
tools=response.tools,
instructions=response.instructions,
items=prompt_items,
)
assert renderer.raw_prompt_lengths == [4, 2, 2]
assert [len(prompt["prompt_token_ids"]) for prompt in renderer.engine_inputs] == [90, 50, 50]
assert all(params.max_total_tokens is None for params in renderer.tokenization_params)
assert all(params.max_total_tokens == 100 for params in renderer.tokenization_params)
assert all(params.max_output_tokens == 0 for params in renderer.tokenization_params)
assert items == [second]
assert items == [first, second]
assert prompt_items == [second]
assert engine_input is renderer.engine_inputs[-1]
@pytest.mark.asyncio
async def test_persistent_model_context_cursor_reuses_truncated_window() -> None:
connection, renderer = _make_connection()
first = _message_item("first")
second = _message_item("second")
connection.session.insert_item(first)
connection.session.insert_item(second)
response = _ResolvedResponse(
input=None,
instructions=None,
modalities=["text"],
max_output_tokens="inf",
tools=None,
tool_choice="none",
metadata=None,
)
prompt_items = await connection._truncate_prompt_items(response)
await connection._build_full_prompt(
tools=response.tools,
instructions=response.instructions,
items=prompt_items,
)
connection.session.commit_model_context_items(prompt_items)
renderer.raw_prompt_lengths.clear()
prompt_items = await connection._truncate_prompt_items(response)
assert renderer.raw_prompt_lengths == [2]
assert prompt_items == [second]
assert connection.session.model_context_first_item_id == "second"
assert not connection.session.model_context_cursor_at_end
@pytest.mark.asyncio
async def test_empty_model_context_allows_a_later_appended_item() -> None:
connection, renderer = _make_connection(max_model_len=20)
response = _ResolvedResponse(
input=None,
instructions=None,
modalities=["text"],
max_output_tokens="inf",
tools=None,
tool_choice="none",
metadata=None,
)
prompt_items = await connection._truncate_prompt_items(response)
connection.session.commit_model_context_items(prompt_items)
connection.session.insert_item(_message_item("appended"))
assert renderer.raw_prompt_lengths == [0]
assert prompt_items == []
assert [item.id for item in connection.session.model_context_items()] == ["appended"]
assert connection.session.model_context_first_item_id == "appended"
assert not connection.session.model_context_cursor_at_end
@pytest.mark.asyncio
async def test_truncation_to_empty_selects_no_items_when_overhead_fits() -> None:
connection, renderer = _make_connection(max_model_len=20)
connection.session.insert_item(_message_item("item"))
response = _ResolvedResponse(
input=None,
instructions=None,
modalities=["text"],
max_output_tokens="inf",
tools=None,
tool_choice="none",
metadata=None,
)
prompt_items = await connection._truncate_prompt_items(response)
connection.session.commit_model_context_items(prompt_items)
connection.session.insert_item(_message_item("appended"))
assert renderer.raw_prompt_lengths == [2, 0]
assert prompt_items == []
assert [item.id for item in connection.session.model_context_items()] == ["appended"]
def test_model_context_cursor_moves_only_forward() -> None:
session = AudioFullDuplexSessionState()
session.insert_item(_message_item("first"))
session.insert_item(_message_item("second"))
session.insert_item(_message_item("third"))
assert session.model_context_first_item_id == "first"
assert session.model_context_items() == session.items
session.commit_model_context_items(session.items[1:])
session.insert_item(_message_item("before-second"), previous_item_id="first")
assert session.model_context_first_item_id == "second"
assert [item.id for item in session.model_context_items()] == ["second", "third"]
session.insert_item(_message_item("after-second"), previous_item_id="second")
assert session.model_context_first_item_id == "second"
assert [item.id for item in session.model_context_items()] == [
"second",
"after-second",
"third",
]
session.remove_item("second")
assert session.model_context_first_item_id == "after-second"
assert [item.id for item in session.model_context_items()] == ["after-second", "third"]
session.commit_model_context_items([])
session.insert_item(_message_item("fourth"))
session.insert_item(_message_item("root"), previous_item_id="root")
assert session.model_context_first_item_id == "fourth"
assert not session.model_context_cursor_at_end
assert [item.id for item in session.model_context_items()] == ["fourth"]
@pytest.mark.asyncio
async def test_prompt_truncation_bisects_the_model_context_cursor() -> None:
connection, renderer = _make_connection()
items = [
SimpleNamespace(
id=f"item_{index}",
type="message",
role="user",
content=[SimpleNamespace(type="input_text", text=str(index))],
)
for index in range(8)
]
response = _ResolvedResponse(
input=items,
instructions=None,
modalities=["text"],
max_output_tokens="inf",
tools=None,
tool_choice="none",
metadata=None,
)
prompt_items = await connection._truncate_prompt_items(response)
await connection._build_full_prompt(
tools=response.tools,
instructions=response.instructions,
items=prompt_items,
)
assert renderer.raw_prompt_lengths == [16, 8, 4, 2, 2]
assert prompt_items == [items[-1]]
@pytest.mark.asyncio
async def test_realtime_auto_truncation_end_to_end() -> None:
websocket = _FakeWebSocket()
connection, renderer = _make_connection(websocket=websocket)
audio = b"\x01\x00\x02\x00\x03\x00\x04\x00"
items: list[Any] = [_message_item(f"item_{index}") for index in range(7)]
items.append(_audio_item("item_audio", audio))
for item in items:
connection.session.insert_item(item)
submitted = await _create_response(connection)
assert len(submitted) == 1
assert submitted[0]["message_count"] == 1
assert renderer.message_counts == [8, 4, 2, 1, 1]
assert renderer.skip_mm_cache == [True, True, True, True, False]
assert connection.session.model_context_first_item_id == "item_audio"
assert not connection.session.model_context_cursor_at_end
assert len(connection.session.items) == 8
events = _websocket_events(websocket)
assert not [event for event in events if event["type"] == "conversation.item.deleted"]
assert [event for event in events if event["type"] == "response.created"]
await connection._handle_item_retrieve(
types.ConversationItemRetrieveEvent(
type="conversation.item.retrieve",
event_id="evt_retrieve",
item_id="item_0",
)
)
retrieved = _websocket_events(websocket)[-1]
assert retrieved["type"] == "conversation.item.retrieved"
assert retrieved["item"]["id"] == "item_0"
final_conversation = renderer.conversations[-1]
assert len(final_conversation) == 1
audio_content = final_conversation[0]["content"][0]["input_audio"]
with wave.open(io.BytesIO(base64.b64decode(audio_content["data"]))) as wav:
assert wav.readframes(wav.getnframes()) == audio
@pytest.mark.parametrize(
("truncation", "max_output_tokens", "expected_texts"),
[
("auto", "inf", ["item_1", "item_2", "item_3"]),
("disabled", "inf", None),
(
types.RealtimeTruncationRetentionRatio(
type="retention_ratio",
retention_ratio=0.5,
token_limits={"post_instructions": 70},
),
"inf",
["item_3"],
),
(
types.RealtimeTruncationRetentionRatio(
type="retention_ratio",
retention_ratio=0.3,
),
"inf",
["item_3"],
),
("auto", 25, ["item_3"]),
],
)
@pytest.mark.asyncio
async def test_realtime_truncation_configurations(
truncation: Any,
max_output_tokens: Any,
expected_texts: list[str] | None,
) -> None:
websocket = _FakeWebSocket()
connection, renderer = _make_connection(websocket=websocket)
connection.session.config.truncation = truncation
connection.session.config.max_output_tokens = max_output_tokens
renderer.token_counts = {0: 10, 1: 30, 2: 45, 3: 50, 4: 110}
for index in range(4):
connection.session.insert_item(_message_item(f"item_{index}"))
submitted = await _create_response(connection)
if expected_texts is None:
assert submitted == []
assert connection._response_task is None
assert "exceeds the model's input token limit" in websocket.messages[-1]
return
assert len(submitted) == 1
assert _prompt_texts(renderer.conversations[-1]) == expected_texts
assert connection.session.model_context_first_item_id == f"item_{4 - len(expected_texts)}"
@pytest.mark.asyncio
async def test_realtime_truncation_render_errors() -> None:
websocket = _FakeWebSocket()
connection, renderer = _make_connection(websocket=websocket)
renderer.token_counts = {0: 10}
renderer.errors = {
2: VLLMValidationError("Prompt is too long", parameter="input_tokens"),
1: VLLMValidationError("Prompt is too long", parameter="input_tokens"),
}
connection.session.insert_item(_message_item("first"))
connection.session.insert_item(_message_item("second"))
submitted = await _create_response(connection)
assert len(submitted) == 1
assert submitted[0]["message_count"] == 0
assert renderer.message_counts == [2, 1, 0, 0]
assert renderer.skip_mm_cache == [True, True, True, False]
assert connection.session.model_context_cursor_at_end
websocket = _FakeWebSocket()
connection, renderer = _make_connection(websocket=websocket)
renderer.errors = {
2: VLLMValidationError("Tool rendering failed", parameter="tools"),
}
connection.session.insert_item(_message_item("first"))
connection.session.insert_item(_message_item("second"))
submitted = await _create_response(connection)
assert submitted == []
assert connection._response_task is None
assert renderer.message_counts == [2]
assert "Tool rendering failed" in websocket.messages[-1]
@pytest.mark.asyncio
async def test_realtime_truncation_survives_history_mutations() -> None:
websocket = _FakeWebSocket()
connection, renderer = _make_connection(websocket=websocket)
renderer.token_counts = {0: 10, 1: 30, 2: 45, 3: 50, 4: 110}
async def add_item(item: Any, previous_item_id: str | None = None) -> None:
await connection._handle_item_create(
types.ConversationItemCreateEvent(
type="conversation.item.create",
event_id=f"evt_create_{item.id}",
item=item,
previous_item_id=previous_item_id,
)
)
await add_item(_message_item("item_0"))
await add_item(_message_item("item_1"))
await add_item(_assistant_item("assistant", "assistant text"))
await add_item(_message_item("item_3"))
submitted = await _create_response(connection)
assert len(submitted) == 1
assert _prompt_texts(renderer.conversations[-1]) == [
"item_1",
"assistant text",
"item_3",
]
assert connection.session.model_context_first_item_id == "item_1"
renderer.message_counts.clear()
renderer.skip_mm_cache.clear()
await add_item(_message_item("before"), previous_item_id="item_0")
await add_item(_message_item("after"), previous_item_id="item_3")
assert connection.session.model_context_first_item_id == "item_1"
await connection._handle_item_truncate(
types.ConversationItemTruncateEvent(
type="conversation.item.truncate",
event_id="evt_truncate",
item_id="assistant",
content_index=0,
audio_end_ms=0,
)
)
truncated_part = connection.session.find_item("assistant").content[0]
assert truncated_part.type == "output_audio"
assert truncated_part.transcript == ""
await connection._handle_item_delete(
types.ConversationItemDeleteEvent(
type="conversation.item.delete",
event_id="evt_delete",
item_id="item_1",
)
)
assert connection.session.model_context_first_item_id == "assistant"
active_item = _assistant_item("active", "active text")
connection.session.insert_item(active_item)
active = ActiveResponse(response_id="resp_active", request_id="req_active")
active.item_id = active_item.id
connection.session.active_response = active
cancellation_cleanup: list[str] = []
async def active_response_task() -> None:
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
connection.session.remove_item(active_item.id)
connection.session.active_response = None
cancellation_cleanup.append(active_item.id)
raise
connection._response_task = asyncio.create_task(active_response_task())
await asyncio.sleep(0)
aborted_requests: list[str] = []
async def abort(request_id: str) -> None:
aborted_requests.append(request_id)
connection.engine.abort = abort
renderer.message_counts.clear()
renderer.skip_mm_cache.clear()
submitted = await _create_response(connection)
assert aborted_requests == ["req_active"]
assert cancellation_cleanup == ["active"]
assert renderer.message_counts == [3, 2, 2]
assert renderer.skip_mm_cache == [True, True, False]
assert len(submitted) == 1
assert _prompt_texts(renderer.conversations[-1]) == ["item_3", "after"]
assert [item.id for item in connection.session.items] == [
"item_0",
"before",
"assistant",
"item_3",
"after",
]
assert connection.session.model_context_first_item_id == "assistant"
assert connection.session.active_response is None
@pytest.mark.asyncio
@pytest.mark.asyncio
async def test_audio_prompt_uses_standard_chat_audio_content() -> None:
audio = b"\x00\x00\x00\x40"