[Frontend][Benchmark] Add duplex performance metrics to refactored full-duplex framework (#7714)

Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
ZacheryAUandamy-why-3459 authored and GitHub committed 2026-09-22 03:04:22 -07:00
1 parent 9ebef4b1d3
commit 0582844a41
21 files changed
+1449 -170

No files matched your search

+168 -21
View File
@@ -43,6 +43,14 @@ from vllm_omni.engine.duplex.session.lease import (
DuplexLeaseConfig,
DuplexLeaseState,
)
from vllm_omni.metrics.stats import (
DUPLEX_STAGE_TABLE_EXCLUDE,
OrchestratorAggregator,
StageRequestStats,
_one_row_per_stage,
_tpot_interval_weight,
)
from vllm_omni.metrics.utils import _as_int, _as_optional_int
if TYPE_CHECKING:
from vllm_omni.engine.duplex.plugin import DuplexModelSessionState
@@ -57,26 +65,6 @@ def _object_dict(**fields: object) -> dict[str, object]:
return dict(fields)
def _as_int(value: object, default: int = 0) -> int:
if isinstance(value, bool):
return default
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return default
def _as_optional_int(value: object) -> int | None:
if value is None or isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
return None
def _copy_mapping(value: object) -> dict[str, object] | None:
if not isinstance(value, dict):
return None
@@ -116,6 +104,22 @@ class ResponseState:
stage_metrics: dict[str, dict[str, object]] = field(default_factory=dict)
stage_metric_tpot_weighted_ms: dict[str, float] = field(default_factory=dict)
stage_metric_tpot_weight: dict[str, int] = field(default_factory=dict)
request_started_at_s_by_turn: dict[int, float] = field(default_factory=dict)
active_response_request_started_at_s: float | None = None
active_response_ttft_ms: float | None = None
active_response_ttfp_ms: float | None = None
RESPONSE_REQUEST_MEASUREMENT_ORIGIN: dict[str, str] = {
"ttft": (
"accepted native-append start to first non-empty text output; "
"pending turns keep the latest append before first output, so overlapping user speech shortens TTFT"
),
"ttfp": (
"accepted native-append start to first audio output; "
"pending turns keep the latest append before first output, so overlapping user speech shortens TTFP"
),
}
@dataclass
@@ -217,6 +221,14 @@ class DuplexEngineSession:
model_state: DuplexModelSessionState | None = field(default=None, repr=False)
projector: RealtimeProjectionState | None = field(default=None, repr=False)
created_monotonic: float = field(default_factory=time.monotonic)
#: Pipeline width for the per-response ``OrchestratorAggregator`` table.
num_stages: int = 1
#: Same serve ``log_stats`` gate as HTTP ``build_and_log_summary``.
log_stats: bool = False
_response_aggregator: OrchestratorAggregator | None = field(default=None, repr=False)
#: ``StageRequestStats`` that arrived before ``begin_response`` (stage 0
#: feeding TTS). Replayed onto the aggregator when the response opens.
_pending_stage_request_stats: list[tuple[int, StageRequestStats]] = field(default_factory=list, repr=False)
def __post_init__(self) -> None:
if self.accepted_fence is None:
@@ -743,6 +755,11 @@ class DuplexEngineSession:
def complete_model_turn(self, turn_id: int) -> None:
"""Advance the model-owned output identity after its terminal signal."""
completed_turn_id = int(turn_id)
self._response.request_started_at_s_by_turn = {
pending_turn_id: started_at_s
for pending_turn_id, started_at_s in self._response.request_started_at_s_by_turn.items()
if pending_turn_id > completed_turn_id
}
if completed_turn_id >= self.turn_id:
self.turn_id = completed_turn_id + 1
self.sync_fence()
@@ -827,10 +844,17 @@ class DuplexEngineSession:
return "".join(self._response.assistant_text_buffer)
def begin_response(self, *, turn_id: int | None = None) -> str:
if self._response_aggregator is not None:
self._log_response_aggregator()
self._activate_response_options()
response_id = f"resp-{self.session_id}-{self.epoch}-{uuid4().hex[:8]}"
self._response.active_response_id = response_id
self._response.active_response_turn_id = self.turn_id if turn_id is None else int(turn_id)
self._clear_response_request_timing()
self._response.active_response_request_started_at_s = self._response.request_started_at_s_by_turn.pop(
self._response.active_response_turn_id,
None,
)
self._response.active_response_input_commit_seq = self.input_commit_seq
self._response.active_response_awaits_input_commit = self.turn_state == DuplexTurnState.USER_SPEAKING
self._response.last_response_id = response_id
@@ -842,6 +866,7 @@ class DuplexEngineSession:
self._playback.current = DuplexPlaybackCursor()
self._playback.by_response[response_id] = self._playback.current
self.turn_state = DuplexTurnState.ASSISTANT_GENERATING
self._start_response_aggregator()
return response_id
def _clear_response_metrics(self) -> None:
@@ -849,6 +874,120 @@ class DuplexEngineSession:
self._response.stage_metric_tpot_weighted_ms.clear()
self._response.stage_metric_tpot_weight.clear()
def _response_aggregator_wall_start_ts(self) -> float:
now_wall_s = time.time()
started_at_s = self._response.active_response_request_started_at_s
if started_at_s is None:
return now_wall_s
elapsed_s = max(0.0, self._clock() - started_at_s)
return now_wall_s - elapsed_s
def _new_response_aggregator(self) -> OrchestratorAggregator:
num_stages = max(int(self.num_stages), 1)
return OrchestratorAggregator(
num_stages=num_stages,
log_stats=True,
wall_start_ts=self._response_aggregator_wall_start_ts(),
final_stage_id_for_e2e=num_stages - 1,
stage_table_exclude=DUPLEX_STAGE_TABLE_EXCLUDE,
)
def _start_response_aggregator(self) -> None:
if not self.log_stats:
self._pending_stage_request_stats.clear()
return
self._response_aggregator = self._new_response_aggregator()
pending, self._pending_stage_request_stats = self._pending_stage_request_stats, []
for stage_id, stats in pending:
self._record_stage_request_stats(stage_id, stats)
def _record_stage_request_stats(self, stage_id: int, metrics: StageRequestStats) -> None:
aggregator = self._response_aggregator
request_id = self.active_response_id
if aggregator is None or request_id is None:
return
if stage_id < 0 or stage_id >= aggregator.num_stages:
return
aggregator.on_stage_metrics(stage_id, request_id, metrics, metrics.final_output_type)
def observe_stage_request_stats(self, stage_id: int, metrics: StageRequestStats) -> None:
"""Feed one engine ``StageRequestStats`` into the response's logger table."""
if not self.log_stats:
return
event = copy.copy(metrics)
if event.stage_id is None:
event.stage_id = stage_id
if self.active_response_id is None:
self._pending_stage_request_stats.append((int(stage_id), event))
return
if self._response_aggregator is None:
self._start_response_aggregator()
self._record_stage_request_stats(int(stage_id), event)
def _log_response_aggregator(self) -> None:
aggregator = self._response_aggregator
self._response_aggregator = None
self._pending_stage_request_stats.clear()
if aggregator is None:
return
response_id = self._response.active_response_id
if response_id is not None and str(response_id) not in aggregator.e2e_done:
final_stage = aggregator.num_stages - 1 if aggregator.num_stages > 0 else 0
aggregator.on_finalize_request(final_stage, response_id, aggregator.wall_start_ts)
for rid, events in aggregator.stage_events.items():
aggregator.stage_events[rid] = _one_row_per_stage(events)
aggregator.build_and_log_summary()
def _clear_response_request_timing(self) -> None:
self._response.active_response_request_started_at_s = None
self._response.active_response_ttft_ms = None
self._response.active_response_ttfp_ms = None
def mark_model_turn_request_started(self, turn_id: int, started_at_s: float) -> None:
"""Record the native request start that can own one model turn.
Pending turns keep the latest accepted append. After ``begin_response``
the first accepted append wins; later appends must not rebind.
"""
turn_id = int(turn_id)
started_at_s = float(started_at_s)
if self.active_response_turn_id == turn_id:
if self._response.active_response_request_started_at_s is None:
self._response.active_response_request_started_at_s = started_at_s
return
self._response.request_started_at_s_by_turn[turn_id] = started_at_s
def mark_response_first_outputs(
self,
*,
observed_at_s: float,
has_text: bool,
has_audio: bool,
) -> dict[str, object]:
"""Return server-monotonic TTF metrics newly observed for the active response."""
started_at_s = self._response.active_response_request_started_at_s
if started_at_s is None:
return {}
elapsed_ms = max(0.0, (float(observed_at_s) - started_at_s) * 1000.0)
newly_observed = False
if has_text and self._response.active_response_ttft_ms is None:
self._response.active_response_ttft_ms = elapsed_ms
newly_observed = True
if has_audio and self._response.active_response_ttfp_ms is None:
self._response.active_response_ttfp_ms = elapsed_ms
newly_observed = True
if not newly_observed:
return {}
metrics: dict[str, object] = {
"source": "server_monotonic_request_start",
"measurement_origin": dict(RESPONSE_REQUEST_MEASUREMENT_ORIGIN),
}
if self._response.active_response_ttft_ms is not None:
metrics["ttft_ms"] = self._response.active_response_ttft_ms
if self._response.active_response_ttfp_ms is not None:
metrics["ttfp_ms"] = self._response.active_response_ttfp_ms
return metrics
def stash_stage_metrics(self, stage_metrics: Mapping[Any, Any] | None) -> None:
"""Hold a stage snapshot until a response exists to attribute it to.
@@ -945,7 +1084,7 @@ class DuplexEngineSession:
tpot_ms = raw_values.get("vllm_tpot_ms")
token_count = raw_values.get("num_tokens_out")
if isinstance(tpot_ms, int | float) and tpot_ms > 0:
weight = max(int(token_count) - 1, 1) if isinstance(token_count, int | float) else 1
weight = _tpot_interval_weight(token_count)
self._response.stage_metric_tpot_weighted_ms[stage_id] = (
self._response.stage_metric_tpot_weighted_ms.get(stage_id, 0.0) + float(tpot_ms) * weight
)
@@ -1160,6 +1299,7 @@ class DuplexEngineSession:
playback_commit_policy: str | None = None,
preserve_request: bool = False,
) -> dict[str, object] | None:
self._log_response_aggregator()
response_id = self._response.active_response_id
response_input_commit_seq = self._response.active_response_input_commit_seq
if response_input_commit_seq is None:
@@ -1205,6 +1345,7 @@ class DuplexEngineSession:
self._response.active_request_id = None
self._response.active_response_id = None
self._response.active_response_turn_id = None
self._clear_response_request_timing()
self._response.active_response_input_commit_seq = None
self._response.active_response_awaits_input_commit = False
# Keep draining bindings for other responses (older TTS may still play).
@@ -1534,6 +1675,7 @@ class DuplexEngineSession:
return self._playback.by_response.get(item_id.removeprefix("item_"))
def barge_in(self) -> int:
self._log_response_aggregator()
self.epoch += 1
self.sync_fence()
self._response.assistant_text_buffer.clear()
@@ -1541,6 +1683,8 @@ class DuplexEngineSession:
self._response.active_request_id = None
self._response.active_response_id = None
self._response.active_response_turn_id = None
self._response.request_started_at_s_by_turn.clear()
self._clear_response_request_timing()
self._response.active_response_input_commit_seq = None
self._response.active_response_awaits_input_commit = False
self._response.draining_response_by_request.clear()
@@ -1554,9 +1698,12 @@ class DuplexEngineSession:
self.state = DuplexSessionState.CLOSING
def close(self) -> None:
self._log_response_aggregator()
self.state = DuplexSessionState.CLOSED
self.turn_state = DuplexTurnState.IDLE
self._response.active_response_turn_id = None
self._response.request_started_at_s_by_turn.clear()
self._clear_response_request_timing()
self._response.active_response_input_commit_seq = None
self._response.active_response_awaits_input_commit = False
self._clear_response_metrics()