diff --git a/tests/benchmarks/metrics/test_metrics.py b/tests/benchmarks/metrics/test_metrics.py index f531a5026..788e1227d 100644 --- a/tests/benchmarks/metrics/test_metrics.py +++ b/tests/benchmarks/metrics/test_metrics.py @@ -63,5 +63,96 @@ def test_total_input_aggregated_from_output_prompt_len(): ) +# ============================================================================ +# TTFT suppression for pure-audio (TTS) benchmarks +# ============================================================================ + + +class _EmptyAwareTokenizer: + """Minimal tokenizer stub: token count == len(text), so '' -> 0 tokens. + + Mirrors production where a TTS speech endpoint returns empty generated_text, + making total_output == 0 (the real CI path uses a real tokenizer, not None). + """ + + def __call__(self, text, add_special_tokens=False): + class _R: + pass + + r = _R() + r.input_ids = [0] * len(text) + return r + + +def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput: + """Pure-TTS output: no text tokens, only audio. ttft is left unset (0.0).""" + output = MixRequestFuncOutput() + output.success = True + output.prompt_len = prompt_len + output.output_tokens = 0 + output.generated_text = "" + output.ttft = 0.0 + output.text_latency = 1.0 + output.latency = 1.0 + output.start_time = 0.0 + output.itl = [] + output.audio_ttfp = 0.05 + output.audio_rtf = 0.2 + output.audio_duration = 5.0 + output.audio_frames = 120000 + output.input_audio_duration = 0.0 + output.error = "" + return output + + +_TTS_PERCENTILE_METRICS = ["ttft", "e2el", "audio_rtf", "audio_ttfp", "audio_duration"] + + +def test_tts_benchmark_omits_ttft(capsys): + """Pure-TTS run (total_output == 0) must not print a Time to First Token section.""" + outputs = [_make_tts_output(100), _make_tts_output(120)] + + calculate_metrics( + input_requests=[], + outputs=outputs, + dur_s=10.0, + tokenizer=_EmptyAwareTokenizer(), + selected_percentiles=[99.0], + goodput_config_dict={}, + task_type=TaskType.GENERATION, + selected_percentile_metrics=_TTS_PERCENTILE_METRICS, + max_concurrency=None, + request_rate=float("inf"), + benchmark_duration=10.0, + ) + + out = capsys.readouterr().out + assert "Time to First Token" not in out, "TTS bench must not surface a meaningless TTFT" + assert "Time to First Packet" in out, "audio TTFP must still be reported" + assert "End-to-end Latency" in out, "e2el must still be reported" + + +def test_text_benchmark_still_reports_ttft(capsys): + """Regression guard: real text generation (total_output > 0) keeps TTFT.""" + outputs = [_make_output(100, output_tokens=10), _make_output(120, output_tokens=10)] + + calculate_metrics( + input_requests=[], + outputs=outputs, + dur_s=10.0, + tokenizer=_EmptyAwareTokenizer(), + selected_percentiles=[99.0], + goodput_config_dict={}, + task_type=TaskType.GENERATION, + selected_percentile_metrics=_TTS_PERCENTILE_METRICS, + max_concurrency=None, + request_rate=float("inf"), + benchmark_duration=10.0, + ) + + out = capsys.readouterr().out + assert "Time to First Token" in out, "text benchmarks must keep TTFT" + + if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) diff --git a/vllm_omni/benchmarks/metrics/metrics.py b/vllm_omni/benchmarks/metrics/metrics.py index f320fffe9..4b32662c5 100644 --- a/vllm_omni/benchmarks/metrics/metrics.py +++ b/vllm_omni/benchmarks/metrics/metrics.py @@ -68,10 +68,13 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput)) if task_type == TaskType.GENERATION: + # No text tokens generated (e.g. pure TTS speech endpoint): per-token + # latency metrics (ttft/tpot/itl) are undefined, so skip them. + has_text_output = metrics.total_output > 0 for metric in selected_percentile_metrics: if metric == "e2el": continue - if not metric.startswith("audio"): + if not metric.startswith("audio") and has_text_output: process_one_metric(metric, metrics) diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index 435236f1f..ea3b6b4c9 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -613,8 +613,9 @@ async def async_request_openai_audio_speech( continue timestamp = time.perf_counter() if output.audio_ttfp == 0.0: + # TTS speech endpoint emits no text tokens, so TTFT is + # not defined here; only audio TTFP is meaningful. output.audio_ttfp = timestamp - st - output.ttft = output.audio_ttfp total_pcm_bytes += len(chunk) if pcm_capture is not None: pcm_capture.extend(chunk) @@ -1017,6 +1018,11 @@ async def benchmark( # metric. if metric_attribute_name not in selected_percentile_metrics: return + # No text tokens generated (e.g. pure TTS speech endpoint): per-token + # latency metrics (ttft/tpot/itl) are undefined, so skip them. + is_text_token_metric = metric_attribute_name not in ("e2el", "audio_ttfp", "audio_rtf", "audio_duration") + if is_text_token_metric and getattr(metrics, "total_output", 0) == 0: + return is_audio_rtf = metric_attribute_name == "audio_rtf" is_audio_duration = metric_attribute_name == "audio_duration"