[Bench] Add audio-streaming continuity metric for TTS (#3618)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

benchmarks/tts/README.md

@@ -40,7 +40,7 @@ vllm bench serve --omni \
     --num-prompts 20 --num-warmups 2 \
     --extra-body '{"task_type":"Base"}' \
     --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \
     --save-result --result-dir ./results
 ```
 
@@ -58,7 +58,7 @@ vllm bench serve --omni \
     --num-prompts 20 --num-warmups 2 \
     --extra-body '{"voice":"Vivian","language":"English","task_type":"CustomVoice"}' \
     --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \
     --save-result --result-dir ./results
 ```
 
@@ -76,10 +76,28 @@ vllm bench serve --omni \
     --num-prompts 20 --num-warmups 2 \
     --extra-body '{"task_type":"VoiceDesign","language":"English"}' \
     --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
+    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \
     --save-result --result-dir ./results
 ```
 
+#### Streaming continuity (`audio_underrun`)
+
+`audio_underrun` (seconds) is the per-request worst-case buffer deficit
+under a realtime-rate player simulation. The audio-speech backend records
+chunk arrival times during the SSE stream and surfaces:
+
+- `Mean / Median / P{50,99} AUDIO_UNDERRUN (s)` per the standard percentile
+  output - any positive value means at least one inter-chunk gap was longer
+  than the chunk's audio duration.
+- `Streaming continuity OK rate` - fraction of requests whose worst-case
+  underrun stayed under the threshold (default 100 ms, the commonly cited
+  "audible gap" budget). Override the threshold with
+  `VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S=<float>`.
+
+This captures the failure mode where `RTF_p50 < 1` (server keeps up in
+aggregate) but per-stream chunk arrival is bursty enough that listeners
+still hear gaps - common at high concurrency on streaming TTS pipelines.
+
 #### Add WER / SIM / UTMOS to any of the above
 
 Append `--seed-tts-wer-eval` (and optionally `SEED_TTS_EVAL_DEVICE=cuda:0`

benchmarks/tts/bench_tts.py

@@ -118,7 +118,7 @@ def build_bench_args(
         "--num-warmups",
         "2",
         "--percentile-metrics",
-        "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun",
     ]
 
     if resolved_dataset_path:

tests/benchmarks/metrics/test_metrics.py

@@ -63,6 +63,38 @@ def test_total_input_aggregated_from_output_prompt_len():
     )
 
 
+def test_audio_continuity_aggregation():
+    """Continuity rate and underrun percentile must aggregate from per-output fields."""
+    bad = _make_output(100)
+    bad.audio_underrun_s = 0.5
+    bad.audio_continuity_ok = False
+    good_a = _make_output(100)
+    good_a.audio_underrun_s = 0.02
+    good_a.audio_continuity_ok = True
+    good_b = _make_output(100)
+    good_b.audio_underrun_s = 0.0
+    good_b.audio_continuity_ok = True
+
+    metrics, _ = calculate_metrics(
+        input_requests=[],
+        outputs=[bad, good_a, good_b],
+        dur_s=10.0,
+        tokenizer=None,
+        selected_percentiles=[50.0, 99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=["audio_underrun"],
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    assert metrics.audio_continuity_ok_rate == pytest.approx(2 / 3, abs=1e-6)
+    # p99 of [0.5, 0.02, 0.0] is dominated by the 0.5 outlier.
+    p99 = dict(metrics.percentiles_audio_underrun_s).get(99.0)
+    assert p99 is not None and p99 > 0.4
+
+
 # ============================================================================
 # TTFT suppression for pure-audio (TTS) benchmarks
 # ============================================================================

tests/benchmarks/test_audio_continuity.py

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Unit tests for the audio-streaming continuity helper.
+
+The helper is intentionally vllm-free: it operates on a flat timeline of
+``(arrival_time_s, num_bytes)`` chunks plus PCM format params.  These tests
+exercise the underrun arithmetic against hand-crafted timelines so future
+refactors of the streaming backend cannot silently regress the metric.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from vllm_omni.benchmarks.audio_continuity import (
+    ContinuityStats,
+    compute_continuity_stats,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
+
+
+# PCM s16le mono at 24 kHz -> 48 000 bytes/sec, the Qwen3-TTS / VoxCPM2 default.
+_SR = 24_000
+_BPS = _SR * 2  # 48 000
+
+
+def test_empty_timeline_is_continuous() -> None:
+    stats = compute_continuity_stats([], [], sample_rate=_SR)
+    assert stats == ContinuityStats(
+        max_underrun_s=0.0,
+        underrun_event_count=0,
+        is_continuous=True,
+    )
+
+
+def test_single_chunk_has_no_underrun() -> None:
+    # One large chunk, no later arrivals to check against.
+    stats = compute_continuity_stats(
+        chunk_arrival_times_s=[0.05],
+        chunk_bytes=[_BPS],  # one second of audio in a single chunk
+        sample_rate=_SR,
+    )
+    assert stats.max_underrun_s == 0.0
+    assert stats.underrun_event_count == 0
+    assert stats.is_continuous is True
+
+
+def test_chunks_faster_than_realtime_have_no_underrun() -> None:
+    # 100ms of audio per chunk, but arrive every 50ms — buffer always builds.
+    stats = compute_continuity_stats(
+        chunk_arrival_times_s=[0.0, 0.05, 0.10, 0.15],
+        chunk_bytes=[_BPS // 10] * 4,
+        sample_rate=_SR,
+    )
+    assert stats.max_underrun_s == 0.0
+    assert stats.underrun_event_count == 0
+    assert stats.is_continuous is True
+
+
+def test_single_gap_above_threshold_breaks_continuity() -> None:
+    # First chunk carries 100ms of audio at t=0.  Next chunk arrives at t=0.5s,
+    # so the player ran dry from t=0.1 to t=0.5 -> 400ms underrun.
+    stats = compute_continuity_stats(
+        chunk_arrival_times_s=[0.0, 0.5],
+        chunk_bytes=[_BPS // 10, _BPS // 10],  # 100ms each
+        sample_rate=_SR,
+        threshold_s=0.1,
+    )
+    assert stats.max_underrun_s == pytest.approx(0.4, abs=1e-6)
+    assert stats.underrun_event_count == 1
+    assert stats.is_continuous is False
+
+
+def test_underrun_is_max_across_gaps_not_sum() -> None:
+    # Two gaps: a 200ms shortfall and a 50ms shortfall.  max_underrun_s
+    # must surface the 200ms one, not the sum.
+    stats = compute_continuity_stats(
+        chunk_arrival_times_s=[0.0, 0.3, 0.45],
+        chunk_bytes=[_BPS // 10, _BPS // 10, _BPS // 10],  # 100ms chunks
+        sample_rate=_SR,
+        threshold_s=0.1,
+    )
+    # At t=0.3 the player consumed 300ms but only 100ms received -> 200ms deficit
+    # At t=0.45 player consumed 450ms, received 200ms -> 250ms deficit.
+    assert stats.max_underrun_s == pytest.approx(0.25, abs=1e-6)
+    assert stats.underrun_event_count == 2
+    assert stats.is_continuous is False
+
+
+def test_continuity_threshold_boundary_inclusive() -> None:
+    # First chunk = 100ms audio at t=0.  Next arrives at t=0.2 -> 100ms deficit
+    # exactly. With threshold=0.1, continuity_ok stays True (boundary).
+    stats = compute_continuity_stats(
+        chunk_arrival_times_s=[0.0, 0.2],
+        chunk_bytes=[_BPS // 10, _BPS // 10],
+        sample_rate=_SR,
+        threshold_s=0.1,
+    )
+    assert stats.max_underrun_s == pytest.approx(0.1, abs=1e-6)
+    assert stats.is_continuous is True
+
+
+def test_mismatched_lengths_returns_zero_stats() -> None:
+    # Defensive: never crash on malformed timeline.
+    stats = compute_continuity_stats(
+        chunk_arrival_times_s=[0.0, 0.1],
+        chunk_bytes=[100],
+        sample_rate=_SR,
+    )
+    assert stats == ContinuityStats(0.0, 0, True)
+
+
+def test_threshold_is_configurable() -> None:
+    # 200ms underrun: violates 100ms threshold, OK against 500ms threshold.
+    arrivals = [0.0, 0.3]
+    bytes_list = [_BPS // 10, _BPS // 10]
+    strict = compute_continuity_stats(arrivals, bytes_list, sample_rate=_SR, threshold_s=0.1)
+    lenient = compute_continuity_stats(arrivals, bytes_list, sample_rate=_SR, threshold_s=0.5)
+    assert strict.is_continuous is False
+    assert lenient.is_continuous is True
+    assert strict.max_underrun_s == lenient.max_underrun_s

vllm_omni/benchmarks/audio_continuity.py

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Streaming-audio continuity metrics for the TTS bench.
+
+Captures the failure mode where ``RTF_p50 < 1`` (server keeps up with realtime
+in aggregate) but per-stream chunk arrival is bursty enough that listeners
+hear audible gaps.  Companion to ``audio_ttfp`` / ``audio_rtf``.
+
+The math is purposefully simple so the helper has no ``vllm`` dependency and
+can be unit-tested in isolation: given the wall-clock arrival timeline of
+audio chunks and the PCM format, simulate a player that consumes at realtime
+rate from the first chunk onward, and surface the worst-case deficit.
+
+A deficit > ``threshold_s`` at any point counts as an audible underrun.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ContinuityStats:
+    """Result of a single-request continuity analysis.
+
+    Attributes:
+        max_underrun_s: Worst-case wall-clock seconds the player was starved.
+        underrun_event_count: Inter-chunk intervals during which the buffer
+            went negative (one per gap, not per starved millisecond).
+        is_continuous: ``max_underrun_s <= threshold_s`` for the request.
+    """
+
+    max_underrun_s: float
+    underrun_event_count: int
+    is_continuous: bool
+
+
+def compute_continuity_stats(
+    chunk_arrival_times_s: list[float],
+    chunk_bytes: list[int],
+    sample_rate: int,
+    sample_width: int = 2,
+    channels: int = 1,
+    threshold_s: float = 0.1,
+) -> ContinuityStats:
+    """Compute the continuity stats for one streaming response.
+
+    Args:
+        chunk_arrival_times_s: Wall-clock seconds (since request start) at
+            which each non-empty audio chunk's *last byte* arrived. Must be
+            monotonically non-decreasing.
+        chunk_bytes: Byte count of each chunk in the same order.
+        sample_rate: PCM sample rate (Hz). 24 000 for Qwen3-TTS / VoxCPM2.
+        sample_width: Bytes per sample (2 = s16le).
+        channels: PCM channel count (1 = mono).
+        threshold_s: Underrun budget. The default 0.1 s matches the
+            commonly-cited "audible gap" threshold for streaming TTS.
+
+    Returns:
+        A :class:`ContinuityStats` summarising the worst-case deficit and
+        whether it stayed under the threshold.
+    """
+    n = len(chunk_arrival_times_s)
+    if n == 0 or n != len(chunk_bytes):
+        return ContinuityStats(0.0, 0, True)
+
+    bytes_per_s = sample_rate * sample_width * channels
+    if bytes_per_s <= 0:
+        return ContinuityStats(0.0, 0, True)
+
+    t0 = chunk_arrival_times_s[0]
+    received_before = 0
+    max_underrun_s = 0.0
+    event_count = 0
+    for i in range(n):
+        if i > 0:
+            played_bytes = (chunk_arrival_times_s[i] - t0) * bytes_per_s
+            deficit_bytes = played_bytes - received_before
+            if deficit_bytes > 0:
+                deficit_s = deficit_bytes / bytes_per_s
+                if deficit_s > max_underrun_s:
+                    max_underrun_s = deficit_s
+                event_count += 1
+        received_before += chunk_bytes[i]
+
+    return ContinuityStats(
+        max_underrun_s=max_underrun_s,
+        underrun_event_count=event_count,
+        is_continuous=max_underrun_s <= threshold_s,
+    )

vllm_omni/benchmarks/metrics/metrics.py

@@ -25,6 +25,12 @@ class MultiModalsBenchmarkMetrics(BenchmarkMetrics):
     median_audio_duration_s: float = 0.0
     std_audio_duration_s: float = 0.0
     percentiles_audio_duration_s: list[tuple[float, float]] = None
+    # Streaming-audio continuity (see vllm_omni/benchmarks/audio_continuity.py).
+    mean_audio_underrun_s: float = 0.0
+    median_audio_underrun_s: float = 0.0
+    std_audio_underrun_s: float = 0.0
+    percentiles_audio_underrun_s: list[tuple[float, float]] = None
+    audio_continuity_ok_rate: float = 1.0
 
 
 def print_metrics(
@@ -83,6 +89,12 @@ def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchma
     print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s))
     print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames))
     print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput))
+    print(
+        "{:<40} {:<10.2%}".format(
+            "Streaming continuity OK rate:",
+            metrics.audio_continuity_ok_rate,
+        )
+    )
     for metric in selected_percentile_metrics:
         if metric.startswith("audio"):
             process_one_metric(metric, metrics)
@@ -100,17 +112,18 @@ def process_one_metric(
         "audio_ttfp": "Time to First Packet",
         "audio_rtf": "Real Time Factor",
         "audio_duration": "Audio Duration",
+        "audio_underrun": "Streaming Audio Underrun",
     }
 
     header = metric_header_map.get(metric_attribute_name, metric_attribute_name)
     print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
 
     is_audio_rtf = metric_attribute_name == "audio_rtf"
-    is_audio_duration = metric_attribute_name == "audio_duration"
+    is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun")
 
     suffix = "_ms"
     unit_suffix = " (ms)"
-    if is_audio_duration:
+    if is_audio_duration_or_underrun:
         suffix = "_s"
         unit_suffix = " (s)"
     elif is_audio_rtf:
@@ -172,6 +185,8 @@ def calculate_metrics(
     audio_rtfs: list[float] = []
     audio_duration: list[float] = []
     audio_frames: list[int] = []
+    audio_underruns: list[float] = []
+    audio_continuity_ok: list[bool] = []
     input_audio_duration = 0.0
     for i in range(len(outputs)):
         if outputs[i].success:
@@ -205,6 +220,8 @@ def calculate_metrics(
             audio_rtfs.append(getattr(outputs[i], "audio_rtf", 0.0))
             audio_duration.append(getattr(outputs[i], "audio_duration", 0.0))
             audio_frames.append(getattr(outputs[i], "audio_frames", 0.0))
+            audio_underruns.append(getattr(outputs[i], "audio_underrun_s", 0.0))
+            audio_continuity_ok.append(bool(getattr(outputs[i], "audio_continuity_ok", True)))
             e2els.append(outputs[i].latency)
             input_audio_duration += outputs[i].input_audio_duration
             completed += 1
@@ -332,6 +349,11 @@ def calculate_metrics(
         std_audio_rtf=np.std(audio_rtfs or 0),
         median_audio_rtf=np.median(audio_rtfs or 0),
         percentiles_audio_rtf=[(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
+        mean_audio_underrun_s=np.mean(audio_underruns or 0),
+        std_audio_underrun_s=np.std(audio_underruns or 0),
+        median_audio_underrun_s=np.median(audio_underruns or 0),
+        percentiles_audio_underrun_s=[(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles],
+        audio_continuity_ok_rate=(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0,
         mean_tpot_ms=np.mean(tpots or 0) * 1000,
         std_tpot_ms=np.std(tpots or 0) * 1000,
         median_tpot_ms=np.median(tpots or 0) * 1000,

vllm_omni/benchmarks/patch/patch.py

@@ -36,6 +36,7 @@ from vllm.tokenizers import TokenizerLike
 
 logger = init_logger(__name__)
 
+from vllm_omni.benchmarks.audio_continuity import compute_continuity_stats
 from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest
 from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
 from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
@@ -46,6 +47,33 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
     SeedTTSTextDataset,
 )
 
+_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
+_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S = 0.1
+
+
+def _audio_continuity_threshold_s() -> float:
+    """Return the per-request underrun budget (s).
+
+    Read from ``VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S`` so users can
+    re-aim the SLO without rebuilding. Defaults to 100 ms - the standard
+    "audible gap" budget for streaming TTS.
+    """
+    raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV)
+    if not raw:
+        return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
+    try:
+        value = float(raw)
+    except ValueError:
+        logger.warning(
+            "Invalid %s=%r; using default %.3fs",
+            _AUDIO_CONTINUITY_THRESHOLD_ENV,
+            raw,
+            _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S,
+        )
+        return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S
+    return max(value, 0.0)
+
+
 get_samples_old = datasets.get_samples
 
 _DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni"
@@ -315,6 +343,16 @@ class MixRequestFuncOutput(RequestFuncOutput):
     audio_frames: int = 0
     audio_rtf: float = 0.0
     text_latency: float = 0.0
+    #: Worst-case streaming-audio underrun (wall-clock seconds the player
+    #: would have been starved). Populated by the audio-speech backend; ``0.0``
+    #: for backends that do not run continuity analysis.
+    audio_underrun_s: float = 0.0
+    #: Whether the request stayed under the continuity threshold (default
+    #: 100 ms). Mirrors ``audio_underrun_s <= threshold``.
+    audio_continuity_ok: bool = True
+    #: Number of inter-chunk intervals during which the player buffer went
+    #: negative.
+    audio_underrun_event_count: int = 0
     #: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
     #: resampled export after ``openai-chat-omni`` audio deltas.
     tts_output_pcm_bytes: bytes | None = None
@@ -605,6 +643,8 @@ async def async_request_openai_audio_speech(
     total_pcm_bytes = 0
     capture_wer_pcm = _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False)
     pcm_capture = bytearray() if capture_wer_pcm else None
+    chunk_arrival_times_s: list[float] = []
+    chunk_sizes: list[int] = []
     try:
         async with session.post(url=api_url, json=payload, headers=headers) as response:
             if response.status == 200:
@@ -617,6 +657,8 @@ async def async_request_openai_audio_speech(
                         # not defined here; only audio TTFP is meaningful.
                         output.audio_ttfp = timestamp - st
                     total_pcm_bytes += len(chunk)
+                    chunk_arrival_times_s.append(timestamp - st)
+                    chunk_sizes.append(len(chunk))
                     if pcm_capture is not None:
                         pcm_capture.extend(chunk)
 
@@ -631,6 +673,18 @@ async def async_request_openai_audio_speech(
                 else:
                     output.audio_rtf = 0
                     logger.warning("Audio duration is zero")
+
+                continuity = compute_continuity_stats(
+                    chunk_arrival_times_s=chunk_arrival_times_s,
+                    chunk_bytes=chunk_sizes,
+                    sample_rate=sample_rate,
+                    sample_width=sample_width,
+                    channels=channels,
+                    threshold_s=_audio_continuity_threshold_s(),
+                )
+                output.audio_underrun_s = continuity.max_underrun_s
+                output.audio_continuity_ok = continuity.is_continuous
+                output.audio_underrun_event_count = continuity.underrun_event_count
                 if pcm_capture is not None and pcm_capture:
                     output.tts_output_pcm_bytes = bytes(pcm_capture)
                 elif capture_wer_pcm:
@@ -1020,14 +1074,14 @@ async def benchmark(
             return
         # No text tokens generated (e.g. pure TTS speech endpoint): per-token
         # latency metrics (ttft/tpot/itl) are undefined, so skip them.
-        is_text_token_metric = metric_attribute_name not in ("e2el", "audio_ttfp", "audio_rtf", "audio_duration")
+        is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio"))
         if is_text_token_metric and getattr(metrics, "total_output", 0) == 0:
             return
         is_audio_rtf = metric_attribute_name == "audio_rtf"
-        is_audio_duration = metric_attribute_name == "audio_duration"
+        is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun")
 
         suffix = "_ms"
-        if is_audio_duration:
+        if is_audio_duration_or_underrun:
             suffix = "_s"
         elif is_audio_rtf:
             suffix = ""
@@ -1038,7 +1092,7 @@ async def benchmark(
         median_attr_name = f"median_{metric_attribute_name}{suffix}"
         median_value = getattr(metrics, median_attr_name, 0.0)
         result[median_attr_name] = median_value
-        for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}"):
+        for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}", None) or []:
             p_word = str(int(p)) if int(p) == p else str(p)
             result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value