[Bugfix] Avoid invalid TPOT metrics (#6696)

Signed-off-by: zhuhh97 <zhuhonghan@huawei.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
Honghan Zhuandamy-why-3459 authored and GitHub committed 2026-08-29 11:56:37 +00:00
1 parent e338f0ece3
commit 72a1ce488b
15 files changed
+399 -65

No files matched your search

+66 -4
View File
@@ -279,17 +279,79 @@ def test_unmeasured_tpot_stays_missing_after_tokenizer_fallback():
assert metrics.request_goodput == 0.0
def test_measured_zero_itl_is_not_treated_as_missing():
def test_zero_itl_does_not_create_zero_tpot():
output = _make_output(100, output_tokens=3)
output.itl = [0.0, 0.0]
output.duplex_session_metrics = {"mean_ttft_ms": 100.0}
metrics = _calculate_test_metrics([output], {"tpot": 1.0})
assert (metrics.num_tpot_samples, metrics.num_itl_samples) == (1, 2)
assert metrics.mean_tpot_ms == 0.0
assert (metrics.num_tpot_samples, metrics.num_itl_samples) == (0, 2)
assert math.isnan(metrics.mean_tpot_ms)
assert metrics.mean_itl_ms == 0.0
assert metrics.request_goodput == 1.0
assert metrics.request_goodput == 0.0
def test_stage_output_tokens_without_client_timing_omit_tpot(capsys):
output = _make_output(100, output_tokens=5)
output.itl = []
output.text_latency = 0.0
output.ttft = 0.1
metrics, _ = calculate_metrics(
input_requests=[],
outputs=[output],
dur_s=1.0,
tokenizer=None,
selected_percentiles=[50.0, 99.0],
goodput_config_dict={},
task_type=TaskType.GENERATION,
selected_percentile_metrics=["tpot"],
max_concurrency=None,
request_rate=float("inf"),
benchmark_duration=1.0,
)
printed = capsys.readouterr().out
assert metrics.num_tpot_samples == 0
assert math.isnan(metrics.mean_tpot_ms)
assert "Time per Output Token" not in printed
assert "Mean TPOT" not in printed
def test_consistent_client_latency_can_supply_tpot_fallback():
output = _make_output(100, output_tokens=10)
output.itl = []
output.ttft = 0.1
output.text_latency = 1.0
metrics = _calculate_test_metrics([output])
assert metrics.num_tpot_samples == 1
assert metrics.mean_tpot_ms == pytest.approx(100.0)
def test_single_token_responses_do_not_report_zero_tpot(capsys):
output = _make_output(100, output_tokens=1)
metrics, _ = calculate_metrics(
input_requests=[],
outputs=[output],
dur_s=1.0,
tokenizer=None,
selected_percentiles=[50.0],
goodput_config_dict={},
task_type=TaskType.GENERATION,
selected_percentile_metrics=["tpot"],
max_concurrency=None,
request_rate=float("inf"),
benchmark_duration=1.0,
)
printed = capsys.readouterr().out
assert metrics.num_tpot_samples == 0
assert math.isnan(metrics.mean_tpot_ms)
assert "Time per Output Token" not in printed
def test_duplex_goodput_does_not_pair_measurements_from_different_requests():
+73 -2
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Unit tests for patch.py
@@ -17,6 +17,7 @@ from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
from vllm_omni.benchmarks.patch.patch import (
MixRequestFuncOutput,
_apply_stage0_token_timings,
async_request_openai_chat_omni_completions,
async_request_openai_realtime_duplex,
)
@@ -102,7 +103,24 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
received_at_s=now + 0.03,
)
self.events.add(
{"type": "response.done", "response": {"id": response_id}},
{
"type": "response.done",
"response": {
"id": response_id,
"metadata": {
"vllm_omni": {
"stage_metrics": {
"0": {
"num_tokens_out": 3,
"vllm_ttft_ms": 20.0,
"vllm_tpot_ms": 10.0,
"vllm_itls_ms": [10.0, 10.0],
}
}
}
},
},
},
received_at_s=now + 0.04,
)
@@ -143,6 +161,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
assert client.configure_kwargs["native_duplex"] is False
assert client.configure_kwargs["extra_body"] == {
"ref_audio": "data:audio/wav;base64,AAAA",
"return_stage_metrics": True,
}
assert client.sent == [
event
@@ -167,6 +186,10 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
assert output.audio_ttfp > output.ttft
assert output.audio_rtf > 0
assert output.latency > 0
assert output.output_tokens == 12
assert output.itl == [0.01, 0.01] * 4
assert output.text_latency == pytest.approx(output.ttft + 0.08)
assert output.tpot_measured is True
assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
session_id = output.duplex_request_metrics[0]["session_id"]
@@ -198,6 +221,54 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
}
def test_stage0_token_timings_use_weighted_tpot_when_itls_are_incomplete():
output = MixRequestFuncOutput(ttft=0.1)
measured = _apply_stage0_token_timings(
output,
[
{"output_token_count": 3, "itls_ms": [], "tpot_ms": 10.0},
{"output_token_count": 2, "itls_ms": [], "tpot_ms": 40.0},
],
expected_output_tokens=5,
)
assert measured is True
assert output.itl == []
assert output.tpot_measured is True
assert (output.text_latency - output.ttft) / 4 == pytest.approx(0.02)
def test_stage0_zero_itls_fall_back_to_positive_tpot():
output = MixRequestFuncOutput(ttft=0.1)
measured = _apply_stage0_token_timings(
output,
[{"output_token_count": 3, "itls_ms": [0.0, 0.0], "tpot_ms": 25.0}],
expected_output_tokens=3,
)
assert measured is True
assert output.itl == []
assert output.tpot_measured is True
assert (output.text_latency - output.ttft) / 2 == pytest.approx(0.025)
def test_stage0_token_count_without_timing_is_not_measured():
output = MixRequestFuncOutput(ttft=0.1)
measured = _apply_stage0_token_timings(
output,
[{"output_token_count": 3, "itls_ms": [], "tpot_ms": None}],
expected_output_tokens=3,
)
assert measured is False
assert output.itl == []
assert output.text_latency == output.ttft
assert output.tpot_measured is False
def create_sse_chunk(data_dict):
"""Helper to create SSE formatted chunk"""
return f"data: {json.dumps(data_dict)}\n\n".encode()
+6 -2
View File
@@ -585,7 +585,7 @@ async def test_adapter_requires_and_forwards_exact_prepared_payload(tmp_path: Pa
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_metrics", "expected_itl", "expected_text_latency"),
("request_metrics", "expected_itl", "expected_text_latency", "expected_tpot_measured"),
[
(
[
@@ -594,6 +594,7 @@ async def test_adapter_requires_and_forwards_exact_prepared_payload(tmp_path: Pa
],
[0.01, 0.02, 0.03],
0.16,
True,
),
(
[
@@ -602,6 +603,7 @@ async def test_adapter_requires_and_forwards_exact_prepared_payload(tmp_path: Pa
],
[],
0.18,
True,
),
(
[
@@ -610,6 +612,7 @@ async def test_adapter_requires_and_forwards_exact_prepared_payload(tmp_path: Pa
],
[],
0.1,
False,
),
],
)
@@ -619,6 +622,7 @@ async def test_adapter_reports_exact_or_weighted_token_timing(
request_metrics: list[dict[str, object]],
expected_itl: list[float],
expected_text_latency: float,
expected_tpot_measured: bool,
):
case, options = _case(tmp_path), _session_options(tmp_path)
request = _request(case, options)
@@ -643,7 +647,7 @@ async def test_adapter_reports_exact_or_weighted_token_timing(
assert output.success
assert output.itl == expected_itl
assert output.text_latency == pytest.approx(expected_text_latency)
assert output.tpot_measured is True
assert output.tpot_measured is expected_tpot_measured
def test_batch_finalization_publishes_only_measured_results(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):