[Bugfix][TTS] Drop meaningless TTFT from speech-endpoint benchmarks (#3674)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>

tests/benchmarks/metrics/test_metrics.py

@@ -63,5 +63,96 @@ def test_total_input_aggregated_from_output_prompt_len():
     )
 
 
+# ============================================================================
+# TTFT suppression for pure-audio (TTS) benchmarks
+# ============================================================================
+
+
+class _EmptyAwareTokenizer:
+    """Minimal tokenizer stub: token count == len(text), so '' -> 0 tokens.
+
+    Mirrors production where a TTS speech endpoint returns empty generated_text,
+    making total_output == 0 (the real CI path uses a real tokenizer, not None).
+    """
+
+    def __call__(self, text, add_special_tokens=False):
+        class _R:
+            pass
+
+        r = _R()
+        r.input_ids = [0] * len(text)
+        return r
+
+
+def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput:
+    """Pure-TTS output: no text tokens, only audio. ttft is left unset (0.0)."""
+    output = MixRequestFuncOutput()
+    output.success = True
+    output.prompt_len = prompt_len
+    output.output_tokens = 0
+    output.generated_text = ""
+    output.ttft = 0.0
+    output.text_latency = 1.0
+    output.latency = 1.0
+    output.start_time = 0.0
+    output.itl = []
+    output.audio_ttfp = 0.05
+    output.audio_rtf = 0.2
+    output.audio_duration = 5.0
+    output.audio_frames = 120000
+    output.input_audio_duration = 0.0
+    output.error = ""
+    return output
+
+
+_TTS_PERCENTILE_METRICS = ["ttft", "e2el", "audio_rtf", "audio_ttfp", "audio_duration"]
+
+
+def test_tts_benchmark_omits_ttft(capsys):
+    """Pure-TTS run (total_output == 0) must not print a Time to First Token section."""
+    outputs = [_make_tts_output(100), _make_tts_output(120)]
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert "Time to First Token" not in out, "TTS bench must not surface a meaningless TTFT"
+    assert "Time to First Packet" in out, "audio TTFP must still be reported"
+    assert "End-to-end Latency" in out, "e2el must still be reported"
+
+
+def test_text_benchmark_still_reports_ttft(capsys):
+    """Regression guard: real text generation (total_output > 0) keeps TTFT."""
+    outputs = [_make_output(100, output_tokens=10), _make_output(120, output_tokens=10)]
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert "Time to First Token" in out, "text benchmarks must keep TTFT"
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v", "-s"])

vllm_omni/benchmarks/metrics/metrics.py

@@ -68,10 +68,13 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod
     print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput))
 
     if task_type == TaskType.GENERATION:
+        # No text tokens generated (e.g. pure TTS speech endpoint): per-token
+        # latency metrics (ttft/tpot/itl) are undefined, so skip them.
+        has_text_output = metrics.total_output > 0
         for metric in selected_percentile_metrics:
             if metric == "e2el":
                 continue
-            if not metric.startswith("audio"):
+            if not metric.startswith("audio") and has_text_output:
                 process_one_metric(metric, metrics)
 
 

vllm_omni/benchmarks/patch/patch.py

@@ -613,8 +613,9 @@ async def async_request_openai_audio_speech(
                         continue
                     timestamp = time.perf_counter()
                     if output.audio_ttfp == 0.0:
+                        # TTS speech endpoint emits no text tokens, so TTFT is
+                        # not defined here; only audio TTFP is meaningful.
                         output.audio_ttfp = timestamp - st
-                        output.ttft = output.audio_ttfp
                     total_pcm_bytes += len(chunk)
                     if pcm_capture is not None:
                         pcm_capture.extend(chunk)
@@ -1017,6 +1018,11 @@ async def benchmark(
         # metric.
         if metric_attribute_name not in selected_percentile_metrics:
             return
+        # No text tokens generated (e.g. pure TTS speech endpoint): per-token
+        # latency metrics (ttft/tpot/itl) are undefined, so skip them.
+        is_text_token_metric = metric_attribute_name not in ("e2el", "audio_ttfp", "audio_rtf", "audio_duration")
+        if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
+            return
         is_audio_rtf = metric_attribute_name == "audio_rtf"
         is_audio_duration = metric_attribute_name == "audio_duration"