Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
eb1abbadf8
commit
882bf8bc95
53 files changed
+3438
-633
No files matched your search
@@ -110,8 +110,7 @@ class AppendAttempt:
|
||||
try:
|
||||
predecessor_ok = await predecessor
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
if current is not None and current.cancelling():
|
||||
if helpers.task_is_cancelling(asyncio.current_task()):
|
||||
raise
|
||||
predecessor_ok = False
|
||||
except Exception:
|
||||
@@ -159,8 +158,10 @@ class AppendAttempt:
|
||||
self.ctx.run.runtime_closed = True
|
||||
return False
|
||||
if not emitted_response and session.epoch == self.epoch:
|
||||
if session.active_request_id == helpers.stage0_request_id(session, self.epoch):
|
||||
session.clear_request(self.request_id)
|
||||
# Only if the session still points at this append's request:
|
||||
# ``clear_request`` compares before it clears, and the id
|
||||
# carries a turn suffix when the core request is not resumable.
|
||||
session.clear_request(self.request_id)
|
||||
if self.final:
|
||||
self.out.emit_events([session.signal_turn(DuplexTurnEventType.USER_STARTED.value)])
|
||||
return append_ok
|
||||
|
||||
@@ -113,6 +113,19 @@ class SessionControl:
|
||||
except ValueError as exc:
|
||||
self._out.emit_error("bad_audio", str(exc))
|
||||
return None
|
||||
if result.speech_started or result.speech_stopped:
|
||||
logger.info(
|
||||
"Duplex VAD session=%s started=%s stopped=%s active=%s probability=%.3f "
|
||||
"start_ms=%s end_ms=%s commit=%s",
|
||||
self._ctx.session.session_id,
|
||||
result.speech_started,
|
||||
result.speech_stopped,
|
||||
result.speech_active,
|
||||
result.speech_probability,
|
||||
result.audio_start_ms,
|
||||
result.audio_end_ms,
|
||||
result.should_commit,
|
||||
)
|
||||
apply_turn_detection_result(event, result)
|
||||
return result
|
||||
|
||||
@@ -310,6 +323,47 @@ class SessionControl:
|
||||
name="duplex-continue",
|
||||
)
|
||||
return
|
||||
item_payload = item if isinstance(item, dict) else None
|
||||
raw_parts = item_payload.get("content", []) if item_payload is not None else []
|
||||
parts: list[object] = [part for part in raw_parts] if isinstance(raw_parts, list) else []
|
||||
images = [p for p in parts if isinstance(p, dict) and p.get("type") == "input_image"]
|
||||
if images:
|
||||
if item_payload is None or item_payload.get("role") != "user":
|
||||
self._out.emit_error("invalid_image", "input_image is supported only in user messages")
|
||||
return
|
||||
if not session.capabilities.supports_image_input:
|
||||
self._out.emit_error("unsupported", "This model does not support input_image conversation items")
|
||||
return
|
||||
from vllm_omni.engine.duplex.realtime_commands import validate_realtime_video_frames
|
||||
|
||||
urls: list[str] = []
|
||||
for part in images:
|
||||
url = part.get("image_url")
|
||||
if not isinstance(url, str) or not url.startswith(
|
||||
("data:image/jpeg;base64,", "data:image/png;base64,")
|
||||
):
|
||||
self._out.emit_error("invalid_image", "input_image requires a JPEG or PNG base64 data URL")
|
||||
return
|
||||
error = validate_realtime_video_frames([url.split(",", 1)[1]], None)
|
||||
if error:
|
||||
self._out.emit_error("invalid_image", error)
|
||||
return
|
||||
urls.append(url)
|
||||
existing: list[str] = []
|
||||
for history_message in session.history:
|
||||
content = history_message.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if not isinstance(part, dict) or part.get("type") != "image_url":
|
||||
continue
|
||||
image_url = part.get("image_url")
|
||||
nested_url = image_url.get("url") if isinstance(image_url, dict) else None
|
||||
if isinstance(nested_url, str):
|
||||
existing.append(nested_url)
|
||||
if len(existing) + len(urls) > 8 or sum(map(len, existing + urls)) > 4 * 1024 * 1024:
|
||||
self._out.emit_error("input_backpressure", "Image context exceeds 8 images or 4 MiB; delete old items")
|
||||
return
|
||||
message = realtime_item_to_history_message(item)
|
||||
item_id = item.get("id") if isinstance(item, dict) else None
|
||||
if message is not None:
|
||||
|
||||
@@ -20,7 +20,7 @@ import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from uuid import uuid4
|
||||
|
||||
from vllm_omni.engine.duplex.config import (
|
||||
@@ -53,6 +53,42 @@ def _default_lease() -> DuplexLeaseState:
|
||||
return DuplexLeaseState(config=DuplexLeaseConfig(), generation=0, last_activity=time.monotonic())
|
||||
|
||||
|
||||
def _object_dict(**fields: object) -> dict[str, object]:
|
||||
return dict(fields)
|
||||
|
||||
|
||||
def _as_int(value: object, default: int = 0) -> int:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
return default
|
||||
|
||||
|
||||
def _as_optional_int(value: object) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
return None
|
||||
|
||||
|
||||
def _copy_mapping(value: object) -> dict[str, object] | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
return {str(key): item for key, item in value.items()}
|
||||
|
||||
|
||||
def _copy_list(value: object) -> list[object] | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
return [item for item in value]
|
||||
|
||||
|
||||
@dataclass
|
||||
class InputBufferState:
|
||||
commit_seq: int = 0
|
||||
@@ -624,7 +660,7 @@ class DuplexEngineSession:
|
||||
input_audio_part["transcript"] = transcript
|
||||
self._bind_active_response_to_input_commit(self._input.commit_seq + 1)
|
||||
self._input.commit_seq += 1
|
||||
message = {"role": "user", "content": [input_audio_part]}
|
||||
message = _object_dict(role="user", content=[input_audio_part])
|
||||
if transcript:
|
||||
message["transcript"] = transcript
|
||||
self._conversation.messages.append(message)
|
||||
@@ -694,7 +730,7 @@ class DuplexEngineSession:
|
||||
self._response.stage_metric_tpot_weighted_ms.clear()
|
||||
self._response.stage_metric_tpot_weight.clear()
|
||||
|
||||
def stash_stage_metrics(self, stage_metrics: Mapping[object, object] | None) -> None:
|
||||
def stash_stage_metrics(self, stage_metrics: Mapping[Any, Any] | None) -> None:
|
||||
"""Hold a stage snapshot until a response exists to attribute it to.
|
||||
|
||||
A stage that hands its output to the next stage instead of the client
|
||||
@@ -706,9 +742,10 @@ class DuplexEngineSession:
|
||||
"""
|
||||
if not isinstance(stage_metrics, Mapping):
|
||||
return
|
||||
snapshot = {
|
||||
str(stage_id): dict(values) for stage_id, values in stage_metrics.items() if isinstance(values, Mapping)
|
||||
}
|
||||
snapshot: dict[str, dict[str, object]] = {}
|
||||
for stage_id, values in stage_metrics.items():
|
||||
if isinstance(values, Mapping):
|
||||
snapshot[str(stage_id)] = {str(key): value for key, value in values.items()}
|
||||
if not snapshot:
|
||||
return
|
||||
if self.active_response_id is not None:
|
||||
@@ -718,7 +755,7 @@ class DuplexEngineSession:
|
||||
|
||||
def accumulate_response_stage_metrics(
|
||||
self,
|
||||
stage_metrics: Mapping[object, object] | None,
|
||||
stage_metrics: Mapping[Any, Any] | None,
|
||||
) -> dict[str, dict[str, object]]:
|
||||
if self.active_response_id is None:
|
||||
return copy.deepcopy(self._response.stage_metrics)
|
||||
@@ -762,7 +799,9 @@ class DuplexEngineSession:
|
||||
for name in additive_fields:
|
||||
value = raw_values.get(name)
|
||||
if isinstance(value, int | float) and not isinstance(value, bool):
|
||||
current[name] = current.get(name, 0) + value
|
||||
previous = current.get(name, 0)
|
||||
addend = previous if isinstance(previous, int | float) and not isinstance(previous, bool) else 0
|
||||
current[name] = addend + value
|
||||
for name in first_positive_fields:
|
||||
value = raw_values.get(name)
|
||||
current_value = current.get(name)
|
||||
@@ -776,7 +815,8 @@ class DuplexEngineSession:
|
||||
for list_name, mean_name in interval_fields:
|
||||
values = raw_values.get(list_name)
|
||||
if isinstance(values, list):
|
||||
combined = list(current.get(list_name, []))
|
||||
existing = current.get(list_name, [])
|
||||
combined = list(existing) if isinstance(existing, list) else []
|
||||
combined.extend(
|
||||
value for value in values if isinstance(value, int | float) and not isinstance(value, bool)
|
||||
)
|
||||
@@ -805,7 +845,7 @@ class DuplexEngineSession:
|
||||
|
||||
def replace_response_stage_metric_snapshots(
|
||||
self,
|
||||
stage_metrics: Mapping[object, object] | None,
|
||||
stage_metrics: Mapping[Any, Any] | None,
|
||||
) -> dict[str, dict[str, object]]:
|
||||
"""Merge cumulative chat snapshots by replacing each stage's latest value."""
|
||||
if self.active_response_id is None or not isinstance(stage_metrics, Mapping):
|
||||
@@ -839,8 +879,12 @@ class DuplexEngineSession:
|
||||
*,
|
||||
text_chars: int | None = None,
|
||||
audio_text_marks: list[dict[str, object]] | None = None,
|
||||
text_requires_complete_audio: bool = False,
|
||||
audio_complete: bool = False,
|
||||
) -> None:
|
||||
playback = self._playback.current
|
||||
playback.text_requires_complete_audio |= text_requires_complete_audio
|
||||
playback.audio_complete |= audio_complete
|
||||
if duration_ms is not None:
|
||||
playback.generated_ms = max(playback.generated_ms, duration_ms)
|
||||
playback.sent_ms = max(playback.sent_ms, duration_ms)
|
||||
@@ -925,9 +969,9 @@ class DuplexEngineSession:
|
||||
response_input_commit_seq = self.input_commit_seq
|
||||
response_history_is_late = self.input_commit_seq > response_input_commit_seq
|
||||
assistant_text = "".join(self._response.assistant_text_buffer).strip()
|
||||
message = None
|
||||
message: dict[str, object] | None = None
|
||||
if assistant_text:
|
||||
self._conversation.last_assistant_full_message = {"role": "assistant", "content": assistant_text}
|
||||
self._conversation.last_assistant_full_message = _object_dict(role="assistant", content=assistant_text)
|
||||
self._conversation.last_assistant_audio_text_marks = list(self._response.assistant_audio_text_marks)
|
||||
if response_id is not None:
|
||||
self._conversation.assistant_response_snapshots[response_id] = (
|
||||
@@ -943,7 +987,7 @@ class DuplexEngineSession:
|
||||
else:
|
||||
committed_text = ""
|
||||
if commit_text and committed_text:
|
||||
message = {"role": "assistant", "content": committed_text}
|
||||
message = _object_dict(role="assistant", content=committed_text)
|
||||
item_id = f"item_{response_id}" if response_id is not None else None
|
||||
if item_id is not None and item_id in self._conversation.history_item_placeholders:
|
||||
self._store_history_item_message(item_id, message)
|
||||
@@ -1078,8 +1122,8 @@ class DuplexEngineSession:
|
||||
self._conversation.pending_truncations_ms.pop(item_id, None)
|
||||
return True
|
||||
|
||||
message = self._conversation.item_ids.get(item_id)
|
||||
if message is None:
|
||||
stored_message = self._conversation.item_ids.get(item_id)
|
||||
if stored_message is None:
|
||||
pending = self._conversation.pending_item_ids.get(item_id)
|
||||
if pending is None:
|
||||
self._conversation.pending_truncations_ms[item_id] = max(0, int(audio_end_ms))
|
||||
@@ -1113,6 +1157,7 @@ class DuplexEngineSession:
|
||||
if item_id.startswith("item_"):
|
||||
self._conversation.assistant_response_snapshots.pop(item_id.removeprefix("item_"), None)
|
||||
return True
|
||||
message = stored_message
|
||||
changed = self._truncate_message_to_audio_ms(
|
||||
message,
|
||||
audio_end_ms=audio_end_ms,
|
||||
@@ -1209,12 +1254,20 @@ class DuplexEngineSession:
|
||||
sent_ms = max(self.playback.sent_ms, self.playback.generated_ms)
|
||||
committed_ms = self.playback.committed_ms
|
||||
policy = playback_commit_policy or self.config.playback_commit_policy
|
||||
commit_all = policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value
|
||||
if self.playback.text_requires_complete_audio:
|
||||
# Unaligned text has no spoken prefix to cut at, so history gets
|
||||
# the whole answer or none of it. Under commit_all_on_done that
|
||||
# choice is the whole answer --- which is also the only thing a
|
||||
# client that never acks playback can be credited with.
|
||||
if commit_all:
|
||||
return assistant_text
|
||||
keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text))
|
||||
return assistant_text[:keep_chars].rstrip()
|
||||
if sent_ms <= 0 or committed_ms >= sent_ms:
|
||||
return assistant_text
|
||||
if committed_ms <= 0:
|
||||
if policy == DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value:
|
||||
return assistant_text
|
||||
return ""
|
||||
return assistant_text if commit_all else ""
|
||||
keep_chars = self._text_chars_for_audio_ms(committed_ms, len(assistant_text))
|
||||
if keep_chars <= 0:
|
||||
return ""
|
||||
@@ -1233,6 +1286,11 @@ class DuplexEngineSession:
|
||||
audio_end_ms = max(0, int(audio_end_ms))
|
||||
marks = marks if marks is not None else self._response.assistant_audio_text_marks
|
||||
playback = playback or self._playback.current
|
||||
if playback.text_requires_complete_audio:
|
||||
# Unaligned streams cannot establish a spoken prefix. A terminal
|
||||
# boundary plus its playback ACK can establish the whole answer.
|
||||
total_ms = max(playback.sent_ms, playback.generated_ms)
|
||||
return text_len if playback.audio_complete and total_ms > 0 and audio_end_ms >= total_ms else 0
|
||||
if not marks:
|
||||
sent_ms = max(1, playback.sent_ms, playback.generated_ms)
|
||||
return int(text_len * max(0.0, min(1.0, audio_end_ms / sent_ms)))
|
||||
@@ -1307,11 +1365,9 @@ class DuplexEngineSession:
|
||||
elif event_type == DuplexTurnEventType.ASSISTANT_DONE.value:
|
||||
self.transition_turn(DuplexTurnState.IDLE)
|
||||
elif event_type == DuplexTurnEventType.PLAYBACK_ACK.value:
|
||||
played_ms = int(payload.get("played_ms", 0) or 0)
|
||||
committed_ms = payload.get("committed_ms")
|
||||
self.acknowledge_playback(
|
||||
played_ms,
|
||||
int(committed_ms) if isinstance(committed_ms, int | float) else None,
|
||||
_as_int(payload.get("played_ms", 0)),
|
||||
_as_optional_int(payload.get("committed_ms")),
|
||||
)
|
||||
elif event_type == DuplexTurnEventType.BARGE_IN.value:
|
||||
self.transition_turn(DuplexTurnState.BARGE_IN)
|
||||
@@ -1346,29 +1402,45 @@ class DuplexEngineSession:
|
||||
"playback": self.playback.as_dict(),
|
||||
"capabilities": self.capabilities.as_dict(),
|
||||
}
|
||||
if isinstance(self.config.extra_body.get("realtime_tools"), list):
|
||||
payload["tools"] = self.config.extra_body["realtime_tools"]
|
||||
if isinstance(self.config.extra_body.get("realtime_tool_choice"), str | dict):
|
||||
payload["tool_choice"] = self.config.extra_body["realtime_tool_choice"]
|
||||
if isinstance(self.config.extra_body.get("realtime_metadata"), dict):
|
||||
payload["metadata"] = dict(self.config.extra_body["realtime_metadata"])
|
||||
if isinstance(self.config.extra_body.get("realtime_include"), list):
|
||||
payload["include"] = list(self.config.extra_body["realtime_include"])
|
||||
if isinstance(self.config.extra_body.get("realtime_prompt"), dict):
|
||||
payload["prompt"] = dict(self.config.extra_body["realtime_prompt"])
|
||||
if isinstance(self.config.extra_body.get("realtime_input_audio_transcription"), dict):
|
||||
payload["input_audio_transcription"] = dict(self.config.extra_body["realtime_input_audio_transcription"])
|
||||
if isinstance(self.config.extra_body.get("realtime_input_audio_noise_reduction"), dict):
|
||||
payload["input_audio_noise_reduction"] = dict(
|
||||
self.config.extra_body["realtime_input_audio_noise_reduction"]
|
||||
)
|
||||
if isinstance(self.config.extra_body.get("realtime_audio"), dict):
|
||||
payload["audio"] = dict(self.config.extra_body["realtime_audio"])
|
||||
if isinstance(self.config.extra_body.get("realtime_tracing"), str | dict):
|
||||
payload["tracing"] = self.config.extra_body["realtime_tracing"]
|
||||
raw_realtime_session = self.config.extra_body.get("realtime_session_payload")
|
||||
if isinstance(raw_realtime_session, dict):
|
||||
for key, value in raw_realtime_session.items():
|
||||
extra_body = self.config.extra_body
|
||||
realtime_tools = _copy_list(extra_body.get("realtime_tools"))
|
||||
if realtime_tools is not None:
|
||||
payload["tools"] = realtime_tools
|
||||
realtime_tool_choice = extra_body.get("realtime_tool_choice")
|
||||
if isinstance(realtime_tool_choice, str):
|
||||
payload["tool_choice"] = realtime_tool_choice
|
||||
else:
|
||||
copied_tool_choice = _copy_mapping(realtime_tool_choice)
|
||||
if copied_tool_choice is not None:
|
||||
payload["tool_choice"] = copied_tool_choice
|
||||
realtime_metadata = _copy_mapping(extra_body.get("realtime_metadata"))
|
||||
if realtime_metadata is not None:
|
||||
payload["metadata"] = realtime_metadata
|
||||
realtime_include = _copy_list(extra_body.get("realtime_include"))
|
||||
if realtime_include is not None:
|
||||
payload["include"] = realtime_include
|
||||
realtime_prompt = _copy_mapping(extra_body.get("realtime_prompt"))
|
||||
if realtime_prompt is not None:
|
||||
payload["prompt"] = realtime_prompt
|
||||
realtime_transcription = _copy_mapping(extra_body.get("realtime_input_audio_transcription"))
|
||||
if realtime_transcription is not None:
|
||||
payload["input_audio_transcription"] = realtime_transcription
|
||||
realtime_noise_reduction = _copy_mapping(extra_body.get("realtime_input_audio_noise_reduction"))
|
||||
if realtime_noise_reduction is not None:
|
||||
payload["input_audio_noise_reduction"] = realtime_noise_reduction
|
||||
realtime_audio = _copy_mapping(extra_body.get("realtime_audio"))
|
||||
if realtime_audio is not None:
|
||||
payload["audio"] = realtime_audio
|
||||
realtime_tracing = extra_body.get("realtime_tracing")
|
||||
if isinstance(realtime_tracing, str):
|
||||
payload["tracing"] = realtime_tracing
|
||||
else:
|
||||
copied_tracing = _copy_mapping(realtime_tracing)
|
||||
if copied_tracing is not None:
|
||||
payload["tracing"] = copied_tracing
|
||||
copied_session = _copy_mapping(extra_body.get("realtime_session_payload"))
|
||||
if copied_session is not None:
|
||||
for key, value in copied_session.items():
|
||||
if key not in payload and key != "extra_body":
|
||||
payload[key] = value
|
||||
return payload
|
||||
|
||||
@@ -13,14 +13,16 @@ Functions, not a component, because there is nothing here to own.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import binascii
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pybase64 as base64
|
||||
|
||||
from vllm_omni.engine.duplex.config import DuplexPlaybackCommitPolicy
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence, duplex_resource_request_id
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence
|
||||
from vllm_omni.engine.duplex.events import ErrorEvent, OverlapDecision, error_event
|
||||
from vllm_omni.engine.duplex.plugin import payload_turn_id
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
@@ -29,13 +31,38 @@ if TYPE_CHECKING:
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexCommittedInput, DuplexEngineSession
|
||||
|
||||
|
||||
def task_is_cancelling(task: asyncio.Task[object] | None) -> bool:
|
||||
"""True when *task* has a pending cancellation (Python 3.11+ ``Task.cancelling``)."""
|
||||
if task is None:
|
||||
return False
|
||||
cancelling = getattr(task, "cancelling", None)
|
||||
return bool(cancelling()) if callable(cancelling) else False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Questions about the session #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def stage0_request_id(session: DuplexEngineSession, epoch: int) -> str:
|
||||
return duplex_resource_request_id(DuplexFence(session.session_id, epoch=epoch), "stage0")
|
||||
def append_fence(session: DuplexEngineSession, payload: object, *, epoch: int | None = None) -> DuplexFence:
|
||||
"""The fence one append is submitted under.
|
||||
|
||||
A model without a resumable core request gets one stage request per turn,
|
||||
so the turn this resolves to is part of the request id. The runner names
|
||||
that id when it queues the append and the model channel names it again
|
||||
when it submits, and the two must not disagree --- otherwise the session
|
||||
binds a request id it never submitted.
|
||||
"""
|
||||
turn_id = payload_turn_id(payload)
|
||||
if turn_id is None:
|
||||
turn_id = session.active_response_turn_id
|
||||
if turn_id is None:
|
||||
turn_id = session.turn_id
|
||||
return DuplexFence(
|
||||
session.session_id,
|
||||
epoch=session.epoch if epoch is None else epoch,
|
||||
turn_id=turn_id,
|
||||
)
|
||||
|
||||
|
||||
def response_in_progress(session: DuplexEngineSession, tasks: DuplexSessionTasks) -> bool:
|
||||
|
||||
@@ -408,8 +408,11 @@ class DuplexSessionManager:
|
||||
return configured
|
||||
|
||||
@staticmethod
|
||||
def stage_request_id(fence: DuplexFence, *, stage_id: int) -> str:
|
||||
return duplex_resource_request_id(fence, f"stage{stage_id}")
|
||||
def stage_request_id(fence: DuplexFence, *, stage_id: int, resumable: bool = True) -> str:
|
||||
role = f"stage{stage_id}"
|
||||
if not resumable:
|
||||
role += f"-turn{fence.turn_id}"
|
||||
return duplex_resource_request_id(fence, role)
|
||||
|
||||
def ensure_stage_request(
|
||||
self,
|
||||
@@ -422,7 +425,9 @@ class DuplexSessionManager:
|
||||
if stage_id >= self.stage_port.stage_count:
|
||||
return None
|
||||
effective_fence = fence or session.fence
|
||||
request_id = self.stage_request_id(effective_fence, stage_id=stage_id)
|
||||
request_id = self.stage_request_id(
|
||||
effective_fence, stage_id=stage_id, resumable=session.capabilities.supports_core_resumable_request
|
||||
)
|
||||
session.reserve_stage_request(stage_id, request_id, fence=effective_fence)
|
||||
context = DuplexStageRequestContext(
|
||||
request_id=request_id,
|
||||
|
||||
@@ -22,6 +22,7 @@ tracked task, and aborting a stage request in the background.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
@@ -43,8 +44,8 @@ from vllm_omni.engine.duplex.contracts import (
|
||||
from vllm_omni.engine.duplex.plugin import (
|
||||
DuplexRuntimeConfigError,
|
||||
coerce_int,
|
||||
payload_turn_id,
|
||||
)
|
||||
from vllm_omni.engine.duplex.session import helpers
|
||||
from vllm_omni.engine.duplex.session.context import DuplexSessionContext, StageOutput
|
||||
from vllm_omni.engine.duplex.session.emitter import SessionEmitter
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession, DuplexFenceMismatchError
|
||||
@@ -114,6 +115,13 @@ class ModelChannel:
|
||||
) -> dict[str, object]:
|
||||
session = self._ctx.session
|
||||
response_config = session.response_config
|
||||
if not session.capabilities.supports_core_resumable_request and self.should_commit_response_to_history(
|
||||
session, response_id
|
||||
):
|
||||
# Committed-turn models can receive the next utterance before the
|
||||
# speaker drains. Reserve the response's chronological slot now;
|
||||
# an ACK fills it later without moving it past newer user input.
|
||||
session.reserve_history_item(f"item_{response_id}")
|
||||
payload: dict[str, object] = {
|
||||
"type": "response.created",
|
||||
"session_id": session.session_id,
|
||||
@@ -202,22 +210,13 @@ class ModelChannel:
|
||||
session = self._ctx.session
|
||||
if self._ctx.stage_port.stage_count == 0:
|
||||
raise RuntimeError("duplex_data_plane_has_no_stage")
|
||||
payload_turn = payload_turn_id(payload)
|
||||
fence = DuplexFence(
|
||||
session.session_id,
|
||||
epoch=session.epoch,
|
||||
turn_id=(
|
||||
payload_turn
|
||||
if payload_turn is not None
|
||||
else (
|
||||
session.active_response_turn_id if session.active_response_turn_id is not None else session.turn_id
|
||||
)
|
||||
),
|
||||
)
|
||||
fence = helpers.append_fence(session, payload)
|
||||
lease_operation_id = f"append:{operation_id or uuid.uuid4().hex}"
|
||||
operation_started = False
|
||||
stage_id = 0
|
||||
request_id = self._ctx.manager.stage_request_id(fence, stage_id=stage_id)
|
||||
request_id = self._ctx.manager.stage_request_id(
|
||||
fence, stage_id=stage_id, resumable=session.capabilities.supports_core_resumable_request
|
||||
)
|
||||
try:
|
||||
session.begin_lease_operation(fence, lease_operation_id)
|
||||
operation_started = True
|
||||
@@ -226,10 +225,17 @@ class ModelChannel:
|
||||
request_context = self._ctx.manager.ensure_stage_request(session, stage_id=stage_id, fence=fence)
|
||||
if request_context is None:
|
||||
raise RuntimeError("duplex_data_plane_has_no_stage")
|
||||
append_plan = self._ctx.plugin.plan_append(
|
||||
prompt_payload: dict[str, object] = (
|
||||
{str(key): value for key, value in payload.items()} if isinstance(payload, Mapping) else {}
|
||||
)
|
||||
append_plan = await self._ctx.plugin.prepare_append_plan(
|
||||
request_id=request_id,
|
||||
fence=fence,
|
||||
session_config=dict(request_context.session_config),
|
||||
session_config=self._ctx.plugin.prepare_prompt_config(
|
||||
{**request_context.session_config, "conversation": list(session.history)},
|
||||
state=self._ctx.model_state,
|
||||
payload=prompt_payload,
|
||||
),
|
||||
runtime_config=dict(request_context.runtime_config),
|
||||
seq=reservation.update.seq,
|
||||
turn_seq=reservation.update.turn_seq,
|
||||
@@ -237,6 +243,10 @@ class ModelChannel:
|
||||
final=final,
|
||||
sampling_params=request_context.stage_sampling_params,
|
||||
)
|
||||
# Preparation may yield while cancellation or session teardown runs.
|
||||
# Never submit the worker's stale result to the model.
|
||||
if session.epoch != fence.epoch or session.state != DuplexSessionState.OPEN:
|
||||
raise DuplexFenceMismatchError(session.fence, fence)
|
||||
if not isinstance(append_plan, DuplexAppendPlan):
|
||||
raise TypeError("duplex plugin plan_append() must return DuplexAppendPlan")
|
||||
submission = DuplexStageSubmission(
|
||||
@@ -279,7 +289,7 @@ class ModelChannel:
|
||||
"seq": update.seq,
|
||||
"turn_id": update.turn_id,
|
||||
"turn_seq": update.turn_seq,
|
||||
"resumable": True,
|
||||
"resumable": session.capabilities.supports_core_resumable_request,
|
||||
},
|
||||
}
|
||||
],
|
||||
@@ -413,7 +423,7 @@ class ModelChannel:
|
||||
engine_output.finished = True
|
||||
engine_output = attach_duplex_output_decision(engine_output, item.decision)
|
||||
elif isinstance(output, OmniRequestOutput):
|
||||
engine_output = output
|
||||
engine_output = copy.copy(output)
|
||||
if finished:
|
||||
engine_output.finished = True
|
||||
else:
|
||||
@@ -425,6 +435,9 @@ class ModelChannel:
|
||||
stage_id=item.stage_id,
|
||||
final_output_type=final_output_type if isinstance(final_output_type, str) else "audio",
|
||||
)
|
||||
engine_output.stage_id = item.stage_id
|
||||
if self._ctx.plugin.projects_intermediate_outputs and item.stage_id < item.context.final_stage_id:
|
||||
engine_output.finished = False
|
||||
snapshot = self.stage_metrics_snapshot(item.stage_id, item.metrics, output)
|
||||
if snapshot is not None:
|
||||
existing = engine_output.metrics if isinstance(engine_output.metrics, dict) else {}
|
||||
@@ -461,6 +474,11 @@ class ModelChannel:
|
||||
await self._close_from_runtime(close_reason)
|
||||
return
|
||||
finished = self._data_plane_outputs_finished(drain_result)
|
||||
if finished and not session.capabilities.supports_core_resumable_request:
|
||||
if self._ctx.run.stream_request_id == item.request_id:
|
||||
self._ctx.run.stream_request_id = None
|
||||
await self._ctx.stage_port.cleanup([item.request_id])
|
||||
return
|
||||
if finished and emitted_response and not self._out.auto_responds():
|
||||
# A finished, emitted response releases the per-request projector
|
||||
# cursor on its way out and offers the model another
|
||||
@@ -759,8 +777,9 @@ class ModelChannel:
|
||||
response_id = session.begin_response(turn_id=model_turn_id)
|
||||
response_created = True
|
||||
self._out.emit(self.response_created_payload(response_id, epoch=session.epoch))
|
||||
stage_metrics = model_result.get("stage_metrics")
|
||||
response_stage_metrics = session.accumulate_response_stage_metrics(
|
||||
model_result.get("stage_metrics") if isinstance(model_result.get("stage_metrics"), Mapping) else None
|
||||
stage_metrics if isinstance(stage_metrics, Mapping) else None
|
||||
)
|
||||
if response_created:
|
||||
speak_payload = {
|
||||
@@ -805,6 +824,8 @@ class ModelChannel:
|
||||
mark_duration_ms,
|
||||
text_chars=mark_text_chars if mark_duration_ms is not None else None,
|
||||
audio_text_marks=audio_text_marks,
|
||||
text_requires_complete_audio=model_result.get("text_requires_complete_audio") is True,
|
||||
audio_complete=model_result.get("audio_complete") is True,
|
||||
)
|
||||
payload = {
|
||||
"type": "response.output_audio.delta",
|
||||
@@ -904,10 +925,10 @@ class ModelChannel:
|
||||
*,
|
||||
audio_offset_ms: int,
|
||||
text_offset_chars: int,
|
||||
) -> list[dict[str, int]] | None:
|
||||
) -> list[dict[str, object]] | None:
|
||||
if not audio_text_marks:
|
||||
return None
|
||||
normalized: list[dict[str, int]] = []
|
||||
normalized: list[dict[str, object]] = []
|
||||
for raw_mark in audio_text_marks:
|
||||
if not isinstance(raw_mark, dict):
|
||||
continue
|
||||
|
||||
@@ -71,7 +71,6 @@ from vllm_omni.engine.duplex.plugin import (
|
||||
DuplexModelPlugin,
|
||||
DuplexModelSessionState,
|
||||
PcmAppendReservation,
|
||||
payload_turn_id,
|
||||
)
|
||||
from vllm_omni.engine.duplex.realtime_events import (
|
||||
RealtimeProjectionState,
|
||||
@@ -251,7 +250,8 @@ class DuplexSessionRunner:
|
||||
if stage_id < context.final_stage_id:
|
||||
decision = self.model.decide_output(stage_id, output, context)
|
||||
consume = decision is not None or stage_id >= context.final_stage_id
|
||||
if not consume:
|
||||
project_intermediate = self.plugin.projects_intermediate_outputs and stage_id == 0
|
||||
if not consume and not project_intermediate:
|
||||
# Stage0 text without a direct decision feeds the TTS stage as before.
|
||||
# Its metrics still have to reach the client: before sessions moved
|
||||
# into the engine the orchestrator published them as a standalone
|
||||
@@ -274,7 +274,7 @@ class DuplexSessionRunner:
|
||||
decision=decision,
|
||||
)
|
||||
)
|
||||
return True
|
||||
return consume
|
||||
|
||||
def on_stage_failure(self, stage_id: int, exc: BaseException) -> None:
|
||||
"""A stage rejected this session's request: fail the active response now.
|
||||
@@ -434,12 +434,11 @@ class DuplexSessionRunner:
|
||||
worker.cancel()
|
||||
await asyncio.gather(worker, return_exceptions=True)
|
||||
|
||||
def spawn(self, coro: Awaitable[None], *, name: str) -> asyncio.Task[None]:
|
||||
def spawn(self, coro: Awaitable[None], *, name: str) -> None:
|
||||
task = asyncio.ensure_future(coro)
|
||||
task.set_name(name)
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._background_tasks.discard)
|
||||
return task
|
||||
|
||||
async def offload(self, fn: Callable[..., _OffloadT], *args: object, **kwargs: object) -> _OffloadT:
|
||||
loop = self._loop or asyncio.get_running_loop()
|
||||
@@ -539,14 +538,14 @@ class DuplexSessionRunner:
|
||||
elif isinstance(command, ClearInput):
|
||||
self._on_clear_input()
|
||||
elif isinstance(command, CancelResponse):
|
||||
resolved = resolve_cancel_response(projector, command)
|
||||
self._emit_events(resolved.events)
|
||||
for payload in resolved.payloads:
|
||||
control = resolve_cancel_response(projector, command)
|
||||
self._emit_events(control.events)
|
||||
for payload in control.payloads:
|
||||
await self._on_cancel(payload)
|
||||
elif isinstance(command, ClearOutputAudio):
|
||||
resolved = resolve_clear_output_audio(projector, command)
|
||||
self._emit_events(resolved.events)
|
||||
for payload in resolved.payloads:
|
||||
control = resolve_clear_output_audio(projector, command)
|
||||
self._emit_events(control.events)
|
||||
for payload in control.payloads:
|
||||
await self._on_cancel(payload)
|
||||
elif isinstance(command, CancelInput | BargeIn):
|
||||
await self._on_cancel(command.payload())
|
||||
@@ -564,7 +563,8 @@ class DuplexSessionRunner:
|
||||
return
|
||||
payload = command.payload()
|
||||
if command.event in _CANCEL_EVENTS:
|
||||
normalized = dict(payload.get("payload") or {})
|
||||
inner = payload.get("payload")
|
||||
normalized: dict[str, object] = dict(inner) if isinstance(inner, Mapping) else {}
|
||||
normalized.update(payload)
|
||||
normalized["type"] = command.event
|
||||
await self._on_cancel(normalized)
|
||||
@@ -577,19 +577,19 @@ class DuplexSessionRunner:
|
||||
elif isinstance(command, Heartbeat):
|
||||
self._on_heartbeat(command)
|
||||
elif isinstance(command, CreateItem):
|
||||
resolved = resolve_create_item(projector, command)
|
||||
self._emit_events(resolved.events)
|
||||
for payload in resolved.payloads:
|
||||
control = resolve_create_item(projector, command)
|
||||
self._emit_events(control.events)
|
||||
for payload in control.payloads:
|
||||
await self._run_internal_payload(payload)
|
||||
elif isinstance(command, DeleteItem):
|
||||
resolved = resolve_delete_item(projector, command)
|
||||
self._emit_events(resolved.events)
|
||||
for payload in resolved.payloads:
|
||||
control = resolve_delete_item(projector, command)
|
||||
self._emit_events(control.events)
|
||||
for payload in control.payloads:
|
||||
await self.control.on_turn_signal(payload)
|
||||
elif isinstance(command, TruncateItem):
|
||||
resolved = resolve_truncate_item(projector, command)
|
||||
self._emit_events(resolved.events)
|
||||
for payload in resolved.payloads:
|
||||
control = resolve_truncate_item(projector, command)
|
||||
self._emit_events(control.events)
|
||||
for payload in control.payloads:
|
||||
await self._run_internal_payload(payload)
|
||||
elif isinstance(command, CloseSession):
|
||||
await self._on_close_command(command.reason)
|
||||
@@ -776,7 +776,7 @@ class DuplexSessionRunner:
|
||||
sr_raw = event.get("sample_rate_hz") or event.get("sample_rate")
|
||||
sample_rate_hz = sr_raw if isinstance(sr_raw, int | float) else 16000
|
||||
try:
|
||||
audio, fmt, sample_rate_hz = await self.offload(
|
||||
converted: tuple[object, object, int | float | None] = await self.offload(
|
||||
convert_input_audio_with_rate,
|
||||
audio,
|
||||
fmt,
|
||||
@@ -785,13 +785,25 @@ class DuplexSessionRunner:
|
||||
except ValueError as exc:
|
||||
self._emit_error("bad_event", str(exc))
|
||||
return
|
||||
audio, fmt, converted_rate = converted
|
||||
if converted_rate is not None:
|
||||
sample_rate_hz = converted_rate
|
||||
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le"}:
|
||||
self._emit_error("bad_audio", "input_audio_buffer.append pcm16 audio could not be decoded")
|
||||
return
|
||||
event["audio"] = audio
|
||||
event["format"] = fmt
|
||||
event["sample_rate_hz"] = sample_rate_hz
|
||||
client_force_listen = bool(event.get("force_listen", False))
|
||||
vad_result = await self.control.run_turn_detection(event)
|
||||
if (
|
||||
vad_result is not None
|
||||
and not session.capabilities.supports_core_resumable_request
|
||||
and not client_force_listen
|
||||
):
|
||||
# VAD's force_listen hint controls native model decoding. Committed
|
||||
# turn models already buffer speech; it must not suppress barge-in.
|
||||
event.pop("force_listen", None)
|
||||
projector = self._require_projector()
|
||||
self._emit_events(note_input_append(projector, event, vad_result=vad_result))
|
||||
if self.run.closing or session.state != DuplexSessionState.OPEN:
|
||||
@@ -896,8 +908,11 @@ class DuplexSessionRunner:
|
||||
return
|
||||
if pcm_reservation.byte_count == 0:
|
||||
session.release_input_bytes(raw_audio_bytes)
|
||||
payload = pcm_reservation.payload
|
||||
await self._start_append(payload, final=False, pcm_reservation=pcm_reservation)
|
||||
append_payload = pcm_reservation.payload
|
||||
if append_payload is None:
|
||||
self._maybe_schedule_vad_commit(vad_result)
|
||||
return
|
||||
await self._start_append(append_payload, final=False, pcm_reservation=pcm_reservation)
|
||||
self._maybe_schedule_vad_commit(vad_result)
|
||||
|
||||
def _maybe_schedule_vad_commit(self, vad_result: TurnDetectionResult | None) -> None:
|
||||
@@ -968,10 +983,13 @@ class DuplexSessionRunner:
|
||||
|
||||
on_append_accepted = _reanchor_chain
|
||||
append_epoch = session.epoch
|
||||
append_turn_id = payload_turn_id(payload)
|
||||
if append_turn_id is None:
|
||||
append_turn_id = session.turn_id
|
||||
request_id = helpers.stage0_request_id(self.session, append_epoch)
|
||||
append_fence = helpers.append_fence(session, payload, epoch=append_epoch)
|
||||
append_turn_id = append_fence.turn_id
|
||||
request_id = self.ctx.manager.stage_request_id(
|
||||
append_fence,
|
||||
stage_id=0,
|
||||
resumable=session.capabilities.supports_core_resumable_request,
|
||||
)
|
||||
if final or precreate_response:
|
||||
session.bind_request(request_id)
|
||||
if precreate_response:
|
||||
@@ -979,6 +997,14 @@ class DuplexSessionRunner:
|
||||
if precreate_response and session.active_response_id is None:
|
||||
response_id = session.begin_response(turn_id=append_turn_id)
|
||||
self.emit(self.model.response_created_payload(response_id, epoch=append_epoch))
|
||||
if final and not session.capabilities.supports_core_resumable_request:
|
||||
logger.info(
|
||||
"Duplex committed turn session=%s request=%s response=%s audio_bytes=%s",
|
||||
session.session_id,
|
||||
request_id,
|
||||
session.active_response_id,
|
||||
helpers.audio_payload_size_bytes(payload),
|
||||
)
|
||||
attempt = AppendAttempt(
|
||||
ctx=self.ctx,
|
||||
out=self.out,
|
||||
@@ -1034,8 +1060,7 @@ class DuplexSessionRunner:
|
||||
try:
|
||||
return await predecessor
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
if current is not None and current.cancelling():
|
||||
if helpers.task_is_cancelling(asyncio.current_task()):
|
||||
raise
|
||||
return False
|
||||
except Exception:
|
||||
@@ -1063,8 +1088,7 @@ class DuplexSessionRunner:
|
||||
if not await pending_silence:
|
||||
return False
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
if current is not None and current.cancelling():
|
||||
if helpers.task_is_cancelling(asyncio.current_task()):
|
||||
raise
|
||||
return False
|
||||
except Exception:
|
||||
@@ -1523,6 +1547,10 @@ class DuplexSessionRunner:
|
||||
retained_committed_payload=committed_payload,
|
||||
)
|
||||
return
|
||||
if session.unanswered_user_items() and session.capabilities.supports_text_only_turn:
|
||||
session.reset_unanswered_user_items()
|
||||
await self._start_append({"type": "conversation"}, final=True, precreate_response=True)
|
||||
return
|
||||
if session.unanswered_user_items():
|
||||
# Conversation items are context, not a turn. A model-native model
|
||||
# decides to speak from the audio it hears, and there is no audio
|
||||
@@ -1601,13 +1629,16 @@ class DuplexSessionRunner:
|
||||
if deferred_payload is None:
|
||||
commit_reservation.commit()
|
||||
return False
|
||||
retained_payload: dict[str, object]
|
||||
if model_state.committed_audio_payload is not None:
|
||||
deferred_payload = overlap_policy.merge_audio_payloads(
|
||||
retained_payload = overlap_policy.merge_audio_payloads(
|
||||
model_state.committed_audio_payload,
|
||||
deferred_payload,
|
||||
)
|
||||
else:
|
||||
retained_payload = deferred_payload
|
||||
model_state.retain_committed_audio(
|
||||
deferred_payload,
|
||||
retained_payload,
|
||||
operation_id=commit_reservation.operation_id,
|
||||
reserved_bytes=commit_reservation.byte_count,
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user