[Bugfix] Recover Omni benchmark TPOT from Stage 0 metrics (#6818)

Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
Gao Han authored and GitHub committed 2026-08-31 00:43:25 +08:00
1 parent f7f48929db
commit 71748388f8
2 files changed
+149

No files matched your search

+46
View File
@@ -119,6 +119,17 @@ def should_request_stage_metrics(args: Any) -> bool:
if backend in _IMAGE_STAGE_METRICS_BACKENDS:
return True
percentile_metrics = getattr(args, "percentile_metrics", ()) or ()
if isinstance(percentile_metrics, str):
percentile_metrics = percentile_metrics.split(",")
selected_metrics = {str(metric).strip().lower() for metric in percentile_metrics}
if backend == "openai-chat-omni" and selected_metrics.intersection({"tpot", "itl"}):
# Client receive timestamps normally provide these metrics. Under
# enough load, however, every text token can be coalesced into one
# event-loop read, leaving no positive interval. Request the engine's
# Stage 0 timings so that path still has an authoritative measurement.
return True
extra_body = getattr(args, "extra_body", None) or {}
modalities = extra_body.get("modalities") if isinstance(extra_body, dict) else None
return backend == "openai-chat-omni" and "image" in (modalities or [])
@@ -854,6 +865,33 @@ def _update_output_stage_metrics_from_payload(
output.stage_metrics.update(stage_snapshot)
def _apply_chat_stage0_token_timings(output: MixRequestFuncOutput) -> bool:
"""Apply native Stage 0 timings from a chat response snapshot."""
stage_metrics = output.stage_metrics
if not isinstance(stage_metrics, dict):
return False
stage0 = stage_metrics.get("0")
if not isinstance(stage0, dict):
return False
output_tokens = coerce_positive_int_scalar(stage0.get(defs.NUM_TOKENS_OUT))
expected_output_tokens = coerce_positive_int_scalar(output.output_tokens)
if output_tokens is None or expected_output_tokens is None or output_tokens != expected_output_tokens:
return False
return _apply_stage0_token_timings(
output,
[
{
"output_token_count": output_tokens,
"itls_ms": stage0.get(defs.VLLM_ITLS_MS),
"tpot_ms": stage0.get(defs.VLLM_TPOT_MS),
}
],
expected_output_tokens=expected_output_tokens,
)
def _image_metrics_from_stage_metrics(metrics: dict[str, Any] | None) -> tuple[int, float, int, float]:
if not isinstance(metrics, dict):
return 0, 0.0, 0, 0.0
@@ -1121,6 +1159,14 @@ async def async_request_openai_chat_omni_completions(
output.latency = timestamp - st
output.generated_text = generated_text
if output.output_tokens > 1 and not any(
isinstance(value, int | float)
and not isinstance(value, bool)
and np.isfinite(value)
and value > 0
for value in output.itl
):
_apply_chat_stage0_token_timings(output)
if output.itl:
# Align text_latency with ITL so TPOT formula and
# mean(ITL) are consistent. Do NOT infer output_tokens