[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-17 23:32:31 +08:00
1 parent eb1abbadf8
commit 882bf8bc95
53 files changed
+3438 -633

No files matched your search

@@ -110,8 +110,7 @@ class AppendAttempt:
try:
predecessor_ok = await predecessor
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
if helpers.task_is_cancelling(asyncio.current_task()):
raise
predecessor_ok = False
except Exception:
@@ -159,8 +158,10 @@ class AppendAttempt:
self.ctx.run.runtime_closed = True
return False
if not emitted_response and session.epoch == self.epoch:
if session.active_request_id == helpers.stage0_request_id(session, self.epoch):
session.clear_request(self.request_id)
# Only if the session still points at this append's request:
# ``clear_request`` compares before it clears, and the id
# carries a turn suffix when the core request is not resumable.
session.clear_request(self.request_id)
if self.final:
self.out.emit_events([session.signal_turn(DuplexTurnEventType.USER_STARTED.value)])
return append_ok
@@ -113,6 +113,19 @@ class SessionControl:
except ValueError as exc:
self._out.emit_error("bad_audio", str(exc))
return None
if result.speech_started or result.speech_stopped:
logger.info(
"Duplex VAD session=%s started=%s stopped=%s active=%s probability=%.3f "
"start_ms=%s end_ms=%s commit=%s",
self._ctx.session.session_id,
result.speech_started,
result.speech_stopped,
result.speech_active,
result.speech_probability,
result.audio_start_ms,
result.audio_end_ms,
result.should_commit,
)
apply_turn_detection_result(event, result)
return result
@@ -310,6 +323,47 @@ class SessionControl:
name="duplex-continue",
)
return
item_payload = item if isinstance(item, dict) else None
raw_parts = item_payload.get("content", []) if item_payload is not None else []
parts: list[object] = [part for part in raw_parts] if isinstance(raw_parts, list) else []
images = [p for p in parts if isinstance(p, dict) and p.get("type") == "input_image"]
if images:
if item_payload is None or item_payload.get("role") != "user":
self._out.emit_error("invalid_image", "input_image is supported only in user messages")
return
if not session.capabilities.supports_image_input:
self._out.emit_error("unsupported", "This model does not support input_image conversation items")
return
from vllm_omni.engine.duplex.realtime_commands import validate_realtime_video_frames
urls: list[str] = []
for part in images:
url = part.get("image_url")
if not isinstance(url, str) or not url.startswith(
("data:image/jpeg;base64,", "data:image/png;base64,")
):
self._out.emit_error("invalid_image", "input_image requires a JPEG or PNG base64 data URL")
return
error = validate_realtime_video_frames([url.split(",", 1)[1]], None)
if error:
self._out.emit_error("invalid_image", error)
return
urls.append(url)
existing: list[str] = []
for history_message in session.history:
content = history_message.get("content")
if not isinstance(content, list):
continue
for part in content:
if not isinstance(part, dict) or part.get("type") != "image_url":
continue
image_url = part.get("image_url")
nested_url = image_url.get("url") if isinstance(image_url, dict) else None
if isinstance(nested_url, str):
existing.append(nested_url)
if len(existing) + len(urls) > 8 or sum(map(len, existing + urls)) > 4 * 1024 * 1024:
self._out.emit_error("input_backpressure", "Image context exceeds 8 images or 4 MiB; delete old items")
return
message = realtime_item_to_history_message(item)
item_id = item.get("id") if isinstance(item, dict) else None
if message is not None:
+117 -45
View File
@@ -20,7 +20,7 @@ import time
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from uuid import uuid4
from vllm_omni.engine.duplex.config import (
@@ -53,6 +53,42 @@ def _default_lease() -> DuplexLeaseState:
return DuplexLeaseState(config=DuplexLeaseConfig(), generation=0, last_activity=time.monotonic())
def _object_dict(**fields: object) -> dict[str, object]:
return dict(fields)
def _as_int(value: object, default: int = 0) -> int:
if isinstance(value, bool):
return default
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return default
def _as_optional_int(value: object) -> int | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return None
def _copy_mapping(value: object) -> dict[str, object] | None:
if not isinstance(value, dict):
return None
return {str(key): item for key, item in value.items()}
def _copy_list(value: object) -> list[object] | None:
if not isinstance(value, list):
return None
return [item for item in value]
@dataclass
class InputBufferState:
commit_seq: int = 0
@@ -624,7 +660,7 @@ class DuplexEngineSession:
input_audio_part["transcript"] = transcript
self._bind_active_response_to_input_commit(self._input.commit_seq + 1)
self._input.commit_seq += 1
message = {"role": "user", "content": [input_audio_part]}
message = _object_dict(role="user", content=[input_audio_part])
if transcript:
message["transcript"] = transcript
self._conversation.messages.append(message)
@@ -694,7 +730,7 @@ class DuplexEngineSession:
self._response.stage_metric_tpot_weighted_ms.clear()
self._response.stage_metric_tpot_weight.clear()
def stash_stage_metrics(self, stage_metrics: Mapping[object, object] | None) -> None:
def stash_stage_metrics(self, stage_metrics: Mapping[Any, Any] | None) -> None:
"""Hold a stage snapshot until a response exists to attribute it to.
A stage that hands its output to the next stage instead of the client
@@ -706,9 +742,10 @@ class DuplexEngineSession:
"""
if not isinstance(stage_metrics, Mapping):
return
snapshot = {
str(stage_id): dict(values) for stage_id, values in stage_metrics.items() if isinstance(values, Mapping)
}
snapshot: dict[str, dict[str, object]] = {}
for stage_id, values in stage_metrics.items():
if isinstance(values, Mapping):
snapshot[str(stage_id)] = {str(key): value for key, value in values.items()}
if not snapshot:
return
if self.active_response_id is not None:
@@ -718,7 +755,7 @@ class DuplexEngineSession:
def accumulate_response_stage_metrics(
self,
stage_metrics: Mapping[object, object] | None,
stage_metrics: Mapping[Any, Any] | None,
) -> dict[str, dict[str, object]]:
if self.active_response_id is None:
return copy.deepcopy(self._response.stage_metrics)
@@ -762,7 +799,9 @@ class DuplexEngineSession:
for name in additive_fields:
value = raw_values.get(name)
if isinstance(value, int | float) and not isinstance(value, bool):
current[name] = current.get(name, 0) + value
previous = current.get(name, 0)
addend = previous if isinstance(previous, int | float) and not isinstance(previous, bool) else 0
current[name] = addend + value
for name in first_positive_fields:
value = raw_values.get(name)
current_value = current.get(name)
@@ -776,7 +815,8 @@ class DuplexEngineSession:
for list_name, mean_name in interval_fields:
values = raw_values.get(list_name)
if isinstance(values, list):
combined = list(current.get(list_name, []))
existing = current.get(list_name, [])
combined = list(existing) if isinstance(existing, list) else []
combined.extend(
value for value in values if isinstance(value, int | float) and not isinstance(value, bool)
)
@@ -805,7 +845,7 @@ class DuplexEngineSession:
def replace_response_stage_metric_snapshots(
self,
stage_metrics: Mapping[object, object] | None,
stage_metrics: Mapping[Any, Any] | None,
) -> dict[str, dict[str, object]]:
"""Merge cumulative chat snapshots by replacing each stage's latest value."""
if self.active_response_id is None or not isinstance(stage_metrics, Mapping):
@@ -839,8 +879,12 @@ class DuplexEngineSession:
*,
text_chars: int | None = None,
audio_text_marks: list[dict[str, object]] | None = None,
text_requires_complete_audio: bool = False,
audio_complete: bool = False,
) -> None:
playback = self._playback.current
playback.text_requires_complete_audio |= text_requires_complete_audio
playback.audio_complete |= audio_complete
if duration_ms is not None:
playback.generated_ms = max(playback.generated_ms, duration_ms)
playback.sent_ms = max(playback.sent_ms, duration_ms)
@@ -925,9 +969,9 @@ class DuplexEngineSession:
response_input_commit_seq = self.input_commit_seq
response_history_is_late = self.input_commit_seq > response_input_commit_seq
assistant_text = "".join(self._response.assistant_text_buffer).strip()
message = None
message: dict[str, object] | None = None
if assistant_text:
self._conversation.last_assistant_full_message = {"role": "assistant", "content": assistant_text}
self._conversation.last_assistant_full_message = _object_dict(role="assistant", content=assistant_text)
self._conversation.last_assistant_audio_text_marks = list(self._response.assistant_audio_text_marks)
if response_id is not None:
self._conversation.assistant_response_snapshots[response_id] = (
@@ -943,7 +987,7 @@ class DuplexEngineSession:
else:
committed_text = ""
if commit_text and committed_text:
message = {"role": "assistant", "content": committed_text}
message = _object_dict(role="assistant", content=committed_text)
item_id = f"item_{response_id}" if response_id is not None else None
if item_id is not None and item_id in self._conversation.history_item_placeholders:
self._store_history_item_message(item_id, message)
@@ -1078,8 +1122,8 @@ class DuplexEngineSession:
self._conversation.pending_truncations_ms.pop(item_id, None)
return True
message = self._conversation.item_ids.get(item_id)
if message is None:
stored_message = self._conversation.item_ids.get(item_id)
if stored_message is None:
pending = self._conversation.pending_item_ids.get(item_id)
if pending is None:
self._conversation.pending_truncations_ms[item_id] = max(0, int(audio_end_ms))
@@ -1113,6 +1157,7 @@ class DuplexEngineSession:
if item_id.startswith("item_"):
self._conversation.assistant_response_snapshots.pop(item_id.removeprefix("item_"), None)
return True
message = stored_message
changed = self._truncate_message_to_audio_ms(
message,
audio_end_ms=audio_end_ms,
@@ -1209,12 +1254,20 @@ class DuplexEngineSession:
sent_ms = max(self.playback.sent_ms, self.playback.generated_ms)
committed_ms = self.playback.committed_ms
policy = playback_commit_policy or self.config.playback_commit_policy
commit_all = policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value
if self.playback.text_requires_complete_audio:
# Unaligned text has no spoken prefix to cut at, so history gets
# the whole answer or none of it. Under commit_all_on_done that
# choice is the whole answer --- which is also the only thing a
# client that never acks playback can be credited with.
if commit_all:
return assistant_text
keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text))
return assistant_text[:keep_chars].rstrip()
if sent_ms <= 0 or committed_ms >= sent_ms:
return assistant_text
if committed_ms <= 0:
if policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value:
return assistant_text
return ""
return assistant_text if commit_all else ""
keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text))
if keep_chars <= 0:
return ""
@@ -1233,6 +1286,11 @@ class DuplexEngineSession:
audio_end_ms = max(0, int(audio_end_ms))
marks = marks if marks is not None else self._response.assistant_audio_text_marks
playback = playback or self._playback.current
if playback.text_requires_complete_audio:
# Unaligned streams cannot establish a spoken prefix. A terminal
# boundary plus its playback ACK can establish the whole answer.
total_ms = max(playback.sent_ms, playback.generated_ms)
return text_len if playback.audio_complete and total_ms > 0 and audio_end_ms >= total_ms else 0
if not marks:
sent_ms = max(1, playback.sent_ms, playback.generated_ms)
return int(text_len * max(0.0, min(1.0, audio_end_ms / sent_ms)))
@@ -1307,11 +1365,9 @@ class DuplexEngineSession:
elif event_type == DuplexTurnEventType.ASSISTANT_DONE.value:
self.transition_turn(DuplexTurnState.IDLE)
elif event_type == DuplexTurnEventType.PLAYBACK_ACK.value:
played_ms = int(payload.get("played_ms", 0) or 0)
committed_ms = payload.get("committed_ms")
self.acknowledge_playback(
played_ms,
int(committed_ms) if isinstance(committed_ms, int | float) else None,
_as_int(payload.get("played_ms", 0)),
_as_optional_int(payload.get("committed_ms")),
)
elif event_type == DuplexTurnEventType.BARGE_IN.value:
self.transition_turn(DuplexTurnState.BARGE_IN)
@@ -1346,29 +1402,45 @@ class DuplexEngineSession:
"playback": self.playback.as_dict(),
"capabilities": self.capabilities.as_dict(),
}
if isinstance(self.config.extra_body.get("realtime_tools"), list):
payload["tools"] = self.config.extra_body["realtime_tools"]
if isinstance(self.config.extra_body.get("realtime_tool_choice"), str | dict):
payload["tool_choice"] = self.config.extra_body["realtime_tool_choice"]
if isinstance(self.config.extra_body.get("realtime_metadata"), dict):
payload["metadata"] = dict(self.config.extra_body["realtime_metadata"])
if isinstance(self.config.extra_body.get("realtime_include"), list):
payload["include"] = list(self.config.extra_body["realtime_include"])
if isinstance(self.config.extra_body.get("realtime_prompt"), dict):
payload["prompt"] = dict(self.config.extra_body["realtime_prompt"])
if isinstance(self.config.extra_body.get("realtime_input_audio_transcription"), dict):
payload["input_audio_transcription"] = dict(self.config.extra_body["realtime_input_audio_transcription"])
if isinstance(self.config.extra_body.get("realtime_input_audio_noise_reduction"), dict):
payload["input_audio_noise_reduction"] = dict(
self.config.extra_body["realtime_input_audio_noise_reduction"]
)
if isinstance(self.config.extra_body.get("realtime_audio"), dict):
payload["audio"] = dict(self.config.extra_body["realtime_audio"])
if isinstance(self.config.extra_body.get("realtime_tracing"), str | dict):
payload["tracing"] = self.config.extra_body["realtime_tracing"]
raw_realtime_session = self.config.extra_body.get("realtime_session_payload")
if isinstance(raw_realtime_session, dict):
for key, value in raw_realtime_session.items():
extra_body = self.config.extra_body
realtime_tools = _copy_list(extra_body.get("realtime_tools"))
if realtime_tools is not None:
payload["tools"] = realtime_tools
realtime_tool_choice = extra_body.get("realtime_tool_choice")
if isinstance(realtime_tool_choice, str):
payload["tool_choice"] = realtime_tool_choice
else:
copied_tool_choice = _copy_mapping(realtime_tool_choice)
if copied_tool_choice is not None:
payload["tool_choice"] = copied_tool_choice
realtime_metadata = _copy_mapping(extra_body.get("realtime_metadata"))
if realtime_metadata is not None:
payload["metadata"] = realtime_metadata
realtime_include = _copy_list(extra_body.get("realtime_include"))
if realtime_include is not None:
payload["include"] = realtime_include
realtime_prompt = _copy_mapping(extra_body.get("realtime_prompt"))
if realtime_prompt is not None:
payload["prompt"] = realtime_prompt
realtime_transcription = _copy_mapping(extra_body.get("realtime_input_audio_transcription"))
if realtime_transcription is not None:
payload["input_audio_transcription"] = realtime_transcription
realtime_noise_reduction = _copy_mapping(extra_body.get("realtime_input_audio_noise_reduction"))
if realtime_noise_reduction is not None:
payload["input_audio_noise_reduction"] = realtime_noise_reduction
realtime_audio = _copy_mapping(extra_body.get("realtime_audio"))
if realtime_audio is not None:
payload["audio"] = realtime_audio
realtime_tracing = extra_body.get("realtime_tracing")
if isinstance(realtime_tracing, str):
payload["tracing"] = realtime_tracing
else:
copied_tracing = _copy_mapping(realtime_tracing)
if copied_tracing is not None:
payload["tracing"] = copied_tracing
copied_session = _copy_mapping(extra_body.get("realtime_session_payload"))
if copied_session is not None:
for key, value in copied_session.items():
if key not in payload and key != "extra_body":
payload[key] = value
return payload
+30 -3
View File
@@ -13,14 +13,16 @@ Functions, not a component, because there is nothing here to own.
from __future__ import annotations
import asyncio
import binascii
from typing import TYPE_CHECKING
import pybase64 as base64
from vllm_omni.engine.duplex.config import DuplexPlaybackCommitPolicy
from vllm_omni.engine.duplex.contracts import DuplexFence, duplex_resource_request_id
from vllm_omni.engine.duplex.contracts import DuplexFence
from vllm_omni.engine.duplex.events import ErrorEvent, OverlapDecision, error_event
from vllm_omni.engine.duplex.plugin import payload_turn_id
if TYPE_CHECKING:
from collections.abc import Mapping
@@ -29,13 +31,38 @@ if TYPE_CHECKING:
from vllm_omni.engine.duplex.session.engine_session import DuplexCommittedInput, DuplexEngineSession
def task_is_cancelling(task: asyncio.Task[object] | None) -> bool:
"""True when *task* has a pending cancellation (Python 3.11+ ``Task.cancelling``)."""
if task is None:
return False
cancelling = getattr(task, "cancelling", None)
return bool(cancelling()) if callable(cancelling) else False
# --------------------------------------------------------------------------- #
# Questions about the session #
# --------------------------------------------------------------------------- #
def stage0_request_id(session: DuplexEngineSession, epoch: int) -> str:
return duplex_resource_request_id(DuplexFence(session.session_id, epoch=epoch), "stage0")
def append_fence(session: DuplexEngineSession, payload: object, *, epoch: int | None = None) -> DuplexFence:
"""The fence one append is submitted under.
A model without a resumable core request gets one stage request per turn,
so the turn this resolves to is part of the request id. The runner names
that id when it queues the append and the model channel names it again
when it submits, and the two must not disagree --- otherwise the session
binds a request id it never submitted.
"""
turn_id = payload_turn_id(payload)
if turn_id is None:
turn_id = session.active_response_turn_id
if turn_id is None:
turn_id = session.turn_id
return DuplexFence(
session.session_id,
epoch=session.epoch if epoch is None else epoch,
turn_id=turn_id,
)
def response_in_progress(session: DuplexEngineSession, tasks: DuplexSessionTasks) -> bool:
+8 -3
View File
@@ -408,8 +408,11 @@ class DuplexSessionManager:
return configured
@staticmethod
def stage_request_id(fence: DuplexFence, *, stage_id: int) -> str:
return duplex_resource_request_id(fence, f"stage{stage_id}")
def stage_request_id(fence: DuplexFence, *, stage_id: int, resumable: bool = True) -> str:
role = f"stage{stage_id}"
if not resumable:
role += f"-turn{fence.turn_id}"
return duplex_resource_request_id(fence, role)
def ensure_stage_request(
self,
@@ -422,7 +425,9 @@ class DuplexSessionManager:
if stage_id >= self.stage_port.stage_count:
return None
effective_fence = fence or session.fence
request_id = self.stage_request_id(effective_fence, stage_id=stage_id)
request_id = self.stage_request_id(
effective_fence, stage_id=stage_id, resumable=session.capabilities.supports_core_resumable_request
)
session.reserve_stage_request(stage_id, request_id, fence=effective_fence)
context = DuplexStageRequestContext(
request_id=request_id,
@@ -22,6 +22,7 @@ tracked task, and aborting a stage request in the background.
from __future__ import annotations
import asyncio
import copy
import time
import uuid
from collections.abc import Awaitable, Callable, Mapping
@@ -43,8 +44,8 @@ from vllm_omni.engine.duplex.contracts import (
from vllm_omni.engine.duplex.plugin import (
DuplexRuntimeConfigError,
coerce_int,
payload_turn_id,
)
from vllm_omni.engine.duplex.session import helpers
from vllm_omni.engine.duplex.session.context import DuplexSessionContext, StageOutput
from vllm_omni.engine.duplex.session.emitter import SessionEmitter
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession, DuplexFenceMismatchError
@@ -114,6 +115,13 @@ class ModelChannel:
) -> dict[str, object]:
session = self._ctx.session
response_config = session.response_config
if not session.capabilities.supports_core_resumable_request and self.should_commit_response_to_history(
session, response_id
):
# Committed-turn models can receive the next utterance before the
# speaker drains. Reserve the response's chronological slot now;
# an ACK fills it later without moving it past newer user input.
session.reserve_history_item(f"item_{response_id}")
payload: dict[str, object] = {
"type": "response.created",
"session_id": session.session_id,
@@ -202,22 +210,13 @@ class ModelChannel:
session = self._ctx.session
if self._ctx.stage_port.stage_count == 0:
raise RuntimeError("duplex_data_plane_has_no_stage")
payload_turn = payload_turn_id(payload)
fence = DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=(
payload_turn
if payload_turn is not None
else (
session.active_response_turn_id if session.active_response_turn_id is not None else session.turn_id
)
),
)
fence = helpers.append_fence(session, payload)
lease_operation_id = f"append:{operation_id or uuid.uuid4().hex}"
operation_started = False
stage_id = 0
request_id = self._ctx.manager.stage_request_id(fence, stage_id=stage_id)
request_id = self._ctx.manager.stage_request_id(
fence, stage_id=stage_id, resumable=session.capabilities.supports_core_resumable_request
)
try:
session.begin_lease_operation(fence, lease_operation_id)
operation_started = True
@@ -226,10 +225,17 @@ class ModelChannel:
request_context = self._ctx.manager.ensure_stage_request(session, stage_id=stage_id, fence=fence)
if request_context is None:
raise RuntimeError("duplex_data_plane_has_no_stage")
append_plan = self._ctx.plugin.plan_append(
prompt_payload: dict[str, object] = (
{str(key): value for key, value in payload.items()} if isinstance(payload, Mapping) else {}
)
append_plan = await self._ctx.plugin.prepare_append_plan(
request_id=request_id,
fence=fence,
session_config=dict(request_context.session_config),
session_config=self._ctx.plugin.prepare_prompt_config(
{**request_context.session_config, "conversation": list(session.history)},
state=self._ctx.model_state,
payload=prompt_payload,
),
runtime_config=dict(request_context.runtime_config),
seq=reservation.update.seq,
turn_seq=reservation.update.turn_seq,
@@ -237,6 +243,10 @@ class ModelChannel:
final=final,
sampling_params=request_context.stage_sampling_params,
)
# Preparation may yield while cancellation or session teardown runs.
# Never submit the worker's stale result to the model.
if session.epoch != fence.epoch or session.state != DuplexSessionState.OPEN:
raise DuplexFenceMismatchError(session.fence, fence)
if not isinstance(append_plan, DuplexAppendPlan):
raise TypeError("duplex plugin plan_append() must return DuplexAppendPlan")
submission = DuplexStageSubmission(
@@ -279,7 +289,7 @@ class ModelChannel:
"seq": update.seq,
"turn_id": update.turn_id,
"turn_seq": update.turn_seq,
"resumable": True,
"resumable": session.capabilities.supports_core_resumable_request,
},
}
],
@@ -413,7 +423,7 @@ class ModelChannel:
engine_output.finished = True
engine_output = attach_duplex_output_decision(engine_output, item.decision)
elif isinstance(output, OmniRequestOutput):
engine_output = output
engine_output = copy.copy(output)
if finished:
engine_output.finished = True
else:
@@ -425,6 +435,9 @@ class ModelChannel:
stage_id=item.stage_id,
final_output_type=final_output_type if isinstance(final_output_type, str) else "audio",
)
engine_output.stage_id = item.stage_id
if self._ctx.plugin.projects_intermediate_outputs and item.stage_id < item.context.final_stage_id:
engine_output.finished = False
snapshot = self.stage_metrics_snapshot(item.stage_id, item.metrics, output)
if snapshot is not None:
existing = engine_output.metrics if isinstance(engine_output.metrics, dict) else {}
@@ -461,6 +474,11 @@ class ModelChannel:
await self._close_from_runtime(close_reason)
return
finished = self._data_plane_outputs_finished(drain_result)
if finished and not session.capabilities.supports_core_resumable_request:
if self._ctx.run.stream_request_id == item.request_id:
self._ctx.run.stream_request_id = None
await self._ctx.stage_port.cleanup([item.request_id])
return
if finished and emitted_response and not self._out.auto_responds():
# A finished, emitted response releases the per-request projector
# cursor on its way out and offers the model another
@@ -759,8 +777,9 @@ class ModelChannel:
response_id = session.begin_response(turn_id=model_turn_id)
response_created = True
self._out.emit(self.response_created_payload(response_id, epoch=session.epoch))
stage_metrics = model_result.get("stage_metrics")
response_stage_metrics = session.accumulate_response_stage_metrics(
model_result.get("stage_metrics") if isinstance(model_result.get("stage_metrics"), Mapping) else None
stage_metrics if isinstance(stage_metrics, Mapping) else None
)
if response_created:
speak_payload = {
@@ -805,6 +824,8 @@ class ModelChannel:
mark_duration_ms,
text_chars=mark_text_chars if mark_duration_ms is not None else None,
audio_text_marks=audio_text_marks,
text_requires_complete_audio=model_result.get("text_requires_complete_audio") is True,
audio_complete=model_result.get("audio_complete") is True,
)
payload = {
"type": "response.output_audio.delta",
@@ -904,10 +925,10 @@ class ModelChannel:
*,
audio_offset_ms: int,
text_offset_chars: int,
) -> list[dict[str, int]] | None:
) -> list[dict[str, object]] | None:
if not audio_text_marks:
return None
normalized: list[dict[str, int]] = []
normalized: list[dict[str, object]] = []
for raw_mark in audio_text_marks:
if not isinstance(raw_mark, dict):
continue
+65 -34
View File
@@ -71,7 +71,6 @@ from vllm_omni.engine.duplex.plugin import (
DuplexModelPlugin,
DuplexModelSessionState,
PcmAppendReservation,
payload_turn_id,
)
from vllm_omni.engine.duplex.realtime_events import (
RealtimeProjectionState,
@@ -251,7 +250,8 @@ class DuplexSessionRunner:
if stage_id < context.final_stage_id:
decision = self.model.decide_output(stage_id, output, context)
consume = decision is not None or stage_id >= context.final_stage_id
if not consume:
project_intermediate = self.plugin.projects_intermediate_outputs and stage_id == 0
if not consume and not project_intermediate:
# Stage0 text without a direct decision feeds the TTS stage as before.
# Its metrics still have to reach the client: before sessions moved
# into the engine the orchestrator published them as a standalone
@@ -274,7 +274,7 @@ class DuplexSessionRunner:
decision=decision,
)
)
return True
return consume
def on_stage_failure(self, stage_id: int, exc: BaseException) -> None:
"""A stage rejected this session's request: fail the active response now.
@@ -434,12 +434,11 @@ class DuplexSessionRunner:
worker.cancel()
await asyncio.gather(worker, return_exceptions=True)
def spawn(self, coro: Awaitable[None], *, name: str) -> asyncio.Task[None]:
def spawn(self, coro: Awaitable[None], *, name: str) -> None:
task = asyncio.ensure_future(coro)
task.set_name(name)
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)
return task
async def offload(self, fn: Callable[..., _OffloadT], *args: object, **kwargs: object) -> _OffloadT:
loop = self._loop or asyncio.get_running_loop()
@@ -539,14 +538,14 @@ class DuplexSessionRunner:
elif isinstance(command, ClearInput):
self._on_clear_input()
elif isinstance(command, CancelResponse):
resolved = resolve_cancel_response(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_cancel_response(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._on_cancel(payload)
elif isinstance(command, ClearOutputAudio):
resolved = resolve_clear_output_audio(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_clear_output_audio(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._on_cancel(payload)
elif isinstance(command, CancelInput | BargeIn):
await self._on_cancel(command.payload())
@@ -564,7 +563,8 @@ class DuplexSessionRunner:
return
payload = command.payload()
if command.event in _CANCEL_EVENTS:
normalized = dict(payload.get("payload") or {})
inner = payload.get("payload")
normalized: dict[str, object] = dict(inner) if isinstance(inner, Mapping) else {}
normalized.update(payload)
normalized["type"] = command.event
await self._on_cancel(normalized)
@@ -577,19 +577,19 @@ class DuplexSessionRunner:
elif isinstance(command, Heartbeat):
self._on_heartbeat(command)
elif isinstance(command, CreateItem):
resolved = resolve_create_item(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_create_item(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._run_internal_payload(payload)
elif isinstance(command, DeleteItem):
resolved = resolve_delete_item(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_delete_item(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self.control.on_turn_signal(payload)
elif isinstance(command, TruncateItem):
resolved = resolve_truncate_item(projector, command)
self._emit_events(resolved.events)
for payload in resolved.payloads:
control = resolve_truncate_item(projector, command)
self._emit_events(control.events)
for payload in control.payloads:
await self._run_internal_payload(payload)
elif isinstance(command, CloseSession):
await self._on_close_command(command.reason)
@@ -776,7 +776,7 @@ class DuplexSessionRunner:
sr_raw = event.get("sample_rate_hz") or event.get("sample_rate")
sample_rate_hz = sr_raw if isinstance(sr_raw, int | float) else 16000
try:
audio, fmt, sample_rate_hz = await self.offload(
converted: tuple[object, object, int | float | None] = await self.offload(
convert_input_audio_with_rate,
audio,
fmt,
@@ -785,13 +785,25 @@ class DuplexSessionRunner:
except ValueError as exc:
self._emit_error("bad_event", str(exc))
return
audio, fmt, converted_rate = converted
if converted_rate is not None:
sample_rate_hz = converted_rate
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le"}:
self._emit_error("bad_audio", "input_audio_buffer.append pcm16 audio could not be decoded")
return
event["audio"] = audio
event["format"] = fmt
event["sample_rate_hz"] = sample_rate_hz
client_force_listen = bool(event.get("force_listen", False))
vad_result = await self.control.run_turn_detection(event)
if (
vad_result is not None
and not session.capabilities.supports_core_resumable_request
and not client_force_listen
):
# VAD's force_listen hint controls native model decoding. Committed
# turn models already buffer speech; it must not suppress barge-in.
event.pop("force_listen", None)
projector = self._require_projector()
self._emit_events(note_input_append(projector, event, vad_result=vad_result))
if self.run.closing or session.state != DuplexSessionState.OPEN:
@@ -896,8 +908,11 @@ class DuplexSessionRunner:
return
if pcm_reservation.byte_count == 0:
session.release_input_bytes(raw_audio_bytes)
payload = pcm_reservation.payload
await self._start_append(payload, final=False, pcm_reservation=pcm_reservation)
append_payload = pcm_reservation.payload
if append_payload is None:
self._maybe_schedule_vad_commit(vad_result)
return
await self._start_append(append_payload, final=False, pcm_reservation=pcm_reservation)
self._maybe_schedule_vad_commit(vad_result)
def _maybe_schedule_vad_commit(self, vad_result: TurnDetectionResult | None) -> None:
@@ -968,10 +983,13 @@ class DuplexSessionRunner:
on_append_accepted = _reanchor_chain
append_epoch = session.epoch
append_turn_id = payload_turn_id(payload)
if append_turn_id is None:
append_turn_id = session.turn_id
request_id = helpers.stage0_request_id(self.session, append_epoch)
append_fence = helpers.append_fence(session, payload, epoch=append_epoch)
append_turn_id = append_fence.turn_id
request_id = self.ctx.manager.stage_request_id(
append_fence,
stage_id=0,
resumable=session.capabilities.supports_core_resumable_request,
)
if final or precreate_response:
session.bind_request(request_id)
if precreate_response:
@@ -979,6 +997,14 @@ class DuplexSessionRunner:
if precreate_response and session.active_response_id is None:
response_id = session.begin_response(turn_id=append_turn_id)
self.emit(self.model.response_created_payload(response_id, epoch=append_epoch))
if final and not session.capabilities.supports_core_resumable_request:
logger.info(
"Duplex committed turn session=%s request=%s response=%s audio_bytes=%s",
session.session_id,
request_id,
session.active_response_id,
helpers.audio_payload_size_bytes(payload),
)
attempt = AppendAttempt(
ctx=self.ctx,
out=self.out,
@@ -1034,8 +1060,7 @@ class DuplexSessionRunner:
try:
return await predecessor
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
if helpers.task_is_cancelling(asyncio.current_task()):
raise
return False
except Exception:
@@ -1063,8 +1088,7 @@ class DuplexSessionRunner:
if not await pending_silence:
return False
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
if helpers.task_is_cancelling(asyncio.current_task()):
raise
return False
except Exception:
@@ -1523,6 +1547,10 @@ class DuplexSessionRunner:
retained_committed_payload=committed_payload,
)
return
if session.unanswered_user_items() and session.capabilities.supports_text_only_turn:
session.reset_unanswered_user_items()
await self._start_append({"type": "conversation"}, final=True, precreate_response=True)
return
if session.unanswered_user_items():
# Conversation items are context, not a turn. A model-native model
# decides to speak from the audio it hears, and there is no audio
@@ -1601,13 +1629,16 @@ class DuplexSessionRunner:
if deferred_payload is None:
commit_reservation.commit()
return False
retained_payload: dict[str, object]
if model_state.committed_audio_payload is not None:
deferred_payload = overlap_policy.merge_audio_payloads(
retained_payload = overlap_policy.merge_audio_payloads(
model_state.committed_audio_payload,
deferred_payload,
)
else:
retained_payload = deferred_payload
model_state.retain_committed_audio(
deferred_payload,
retained_payload,
operation_id=commit_reservation.operation_id,
reserved_bytes=commit_reservation.byte_count,
)