[Bugfix] Recover Omni benchmark TPOT from Stage 0 metrics (#6818)
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
1 parent
f7f48929db
commit
71748388f8
2 files changed
+149
No files matched your search
@@ -119,6 +119,17 @@ def should_request_stage_metrics(args: Any) -> bool:
|
||||
if backend in _IMAGE_STAGE_METRICS_BACKENDS:
|
||||
return True
|
||||
|
||||
percentile_metrics = getattr(args, "percentile_metrics", ()) or ()
|
||||
if isinstance(percentile_metrics, str):
|
||||
percentile_metrics = percentile_metrics.split(",")
|
||||
selected_metrics = {str(metric).strip().lower() for metric in percentile_metrics}
|
||||
if backend == "openai-chat-omni" and selected_metrics.intersection({"tpot", "itl"}):
|
||||
# Client receive timestamps normally provide these metrics. Under
|
||||
# enough load, however, every text token can be coalesced into one
|
||||
# event-loop read, leaving no positive interval. Request the engine's
|
||||
# Stage 0 timings so that path still has an authoritative measurement.
|
||||
return True
|
||||
|
||||
extra_body = getattr(args, "extra_body", None) or {}
|
||||
modalities = extra_body.get("modalities") if isinstance(extra_body, dict) else None
|
||||
return backend == "openai-chat-omni" and "image" in (modalities or [])
|
||||
@@ -854,6 +865,33 @@ def _update_output_stage_metrics_from_payload(
|
||||
output.stage_metrics.update(stage_snapshot)
|
||||
|
||||
|
||||
def _apply_chat_stage0_token_timings(output: MixRequestFuncOutput) -> bool:
|
||||
"""Apply native Stage 0 timings from a chat response snapshot."""
|
||||
stage_metrics = output.stage_metrics
|
||||
if not isinstance(stage_metrics, dict):
|
||||
return False
|
||||
stage0 = stage_metrics.get("0")
|
||||
if not isinstance(stage0, dict):
|
||||
return False
|
||||
|
||||
output_tokens = coerce_positive_int_scalar(stage0.get(defs.NUM_TOKENS_OUT))
|
||||
expected_output_tokens = coerce_positive_int_scalar(output.output_tokens)
|
||||
if output_tokens is None or expected_output_tokens is None or output_tokens != expected_output_tokens:
|
||||
return False
|
||||
|
||||
return _apply_stage0_token_timings(
|
||||
output,
|
||||
[
|
||||
{
|
||||
"output_token_count": output_tokens,
|
||||
"itls_ms": stage0.get(defs.VLLM_ITLS_MS),
|
||||
"tpot_ms": stage0.get(defs.VLLM_TPOT_MS),
|
||||
}
|
||||
],
|
||||
expected_output_tokens=expected_output_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _image_metrics_from_stage_metrics(metrics: dict[str, Any] | None) -> tuple[int, float, int, float]:
|
||||
if not isinstance(metrics, dict):
|
||||
return 0, 0.0, 0, 0.0
|
||||
@@ -1121,6 +1159,14 @@ async def async_request_openai_chat_omni_completions(
|
||||
|
||||
output.latency = timestamp - st
|
||||
output.generated_text = generated_text
|
||||
if output.output_tokens > 1 and not any(
|
||||
isinstance(value, int | float)
|
||||
and not isinstance(value, bool)
|
||||
and np.isfinite(value)
|
||||
and value > 0
|
||||
for value in output.itl
|
||||
):
|
||||
_apply_chat_stage0_token_timings(output)
|
||||
if output.itl:
|
||||
# Align text_latency with ITL so TPOT formula and
|
||||
# mean(ITL) are consistent. Do NOT infer output_tokens
|
||||
|
||||
Reference in new issue
Block a user