From 117d9bd1223679d35a253467f4de7bddc1e01bdb Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Wed, 4 Feb 2026 14:07:46 +0800 Subject: [PATCH] [Bugfix] modify RTF use audio_e2e/audio_duration (#1157) Signed-off-by: wangyu31577 Co-authored-by: wangyu31577 --- docs/cli/bench/serve.md | 58 ++++++++++++------------- vllm_omni/benchmarks/metrics/metrics.py | 30 ++++++++++--- vllm_omni/benchmarks/patch/patch.py | 28 ++++++++---- 3 files changed, 73 insertions(+), 43 deletions(-) diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md index cc47bfc3c..f6e43be09 100644 --- a/docs/cli/bench/serve.md +++ b/docs/cli/bench/serve.md @@ -52,7 +52,7 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para - `--percentile-metrics` "Comma-separated list of selected metrics to report percentiles." "This argument specifies the metrics to report percentiles." - 'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf". ' + 'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". ' - `--save-result` Specify to save benchmark results to a json file @@ -220,48 +220,48 @@ If successful, you will see the following output: ============ Serving Benchmark Result ============ Successful requests: 2 Failed requests: 0 -Benchmark duration (s): 24.35 -Request throughput (req/s): 0.08 +Benchmark duration (s): 3.89 +Request throughput (req/s): 0.51 Peak concurrent requests: 2.00 ----------------End-to-end Latency---------------- -Mean E2EL (ms): 22576.23 -Median E2EL (ms): 22576.23 -P99 E2EL (ms): 24205.72 +Mean E2EL (ms): 3824.76 +Median E2EL (ms): 3824.76 +P99 E2EL (ms): 3888.54 ================== Text Result =================== Total input tokens: 30 -Total generated tokens: 8973 -Output token throughput (tok/s): 368.52 -Peak output token throughput (tok/s): 81.00 +Total generated tokens: 10101 +Output token throughput (tok/s): 2595.57 +Peak output token throughput (tok/s): 111.00 Peak concurrent requests: 2.00 -Total Token throughput (tok/s): 369.76 +Total Token throughput (tok/s): 2603.28 ---------------Time to First Token---------------- -Mean TTFT (ms): 125.16 -Median TTFT (ms): 125.16 -P99 TTFT (ms): 155.88 +Mean TTFT (ms): 117.15 +Median TTFT (ms): 117.15 +P99 TTFT (ms): 142.69 -----Time per Output Token (excl. 1st token)------ -Mean TPOT (ms): 5.01 -Median TPOT (ms): 5.01 -P99 TPOT (ms): 5.42 +Mean TPOT (ms): 0.73 +Median TPOT (ms): 0.73 +P99 TPOT (ms): 0.74 ---------------Inter-token Latency---------------- -Mean ITL (ms): 34.15 -Median ITL (ms): 0.01 -P99 ITL (ms): 376.19 +Mean ITL (ms): 16.47 +Median ITL (ms): 16.19 +P99 ITL (ms): 52.55 ================== Audio Result ================== -Total audio duration generated(s): 3.95 -Total audio frames generated: 94890 -Audio throughput(audio duration/s): 0.16 +Total audio duration generated(s): 15.79 +Total audio frames generated: 379050 +Audio throughput(audio duration/s): 4.06 ---------------Time to First Packet--------------- -Mean AUDIO_TTFP (ms): 11756.89 -Median AUDIO_TTFP (ms): 11756.89 -P99 AUDIO_TTFP (ms): 20854.25 +Mean AUDIO_TTFP (ms): 3701.37 +Median AUDIO_TTFP (ms): 3701.37 +P99 AUDIO_TTFP (ms): 3762.25 -----------------Real Time Factor----------------- -Mean AUDIO_RTF: 3.75 -Median AUDIO_RTF: 3.75 -P99 AUDIO_RTF: 7.39 +Mean AUDIO_RTF: 0.47 +Median AUDIO_RTF: 0.47 +P99 AUDIO_RTF: 0.48 ================================================== ``` Notes: -We use (audio generation time - first packet latency) / audio duration to calculate RTF. +We use audio generation time / audio duration to calculate RTF. diff --git a/vllm_omni/benchmarks/metrics/metrics.py b/vllm_omni/benchmarks/metrics/metrics.py index f404a12f8..5f36357b8 100644 --- a/vllm_omni/benchmarks/metrics/metrics.py +++ b/vllm_omni/benchmarks/metrics/metrics.py @@ -14,13 +14,17 @@ class MultiModalsBenchmarkMetrics(BenchmarkMetrics): median_audio_ttfp_ms: float = 0.0 std_audio_ttfp_ms: float = 0.0 percentiles_audio_ttfp_ms: list[tuple[float, float]] = None - total_audio_duration_ms: float = 0.0 + total_audio_duration_s: float = 0.0 total_audio_frames: int = 0 audio_throughput: float = 0.0 mean_audio_rtf: float = 0.0 median_audio_rtf: float = 0.0 std_audio_rtf: float = 0.0 percentiles_audio_rtf: list[tuple[float, float]] = None + mean_audio_duration_s: float = 0.0 + median_audio_duration_s: float = 0.0 + std_audio_duration_s: float = 0.0 + percentiles_audio_duration_s: list[tuple[float, float]] = None def print_metrics( @@ -73,7 +77,7 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics): print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="=")) - print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_ms)) + print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s)) print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames)) print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput)) for metric in selected_percentile_metrics: @@ -92,16 +96,23 @@ def process_one_metric( "e2el": "End-to-end Latency", "audio_ttfp": "Time to First Packet", "audio_rtf": "Real Time Factor", + "audio_duration": "Audio Duration", } header = metric_header_map.get(metric_attribute_name, metric_attribute_name) print("{s:{c}^{n}}".format(s=header, n=50, c="-")) is_audio_rtf = metric_attribute_name == "audio_rtf" + is_audio_duration = metric_attribute_name == "audio_duration" - suffix = "" if is_audio_rtf else "_ms" - unit_suffix = "" if is_audio_rtf else " (ms)" - + suffix = "_ms" + unit_suffix = " (ms)" + if is_audio_duration: + suffix = "_s" + unit_suffix = " (s)" + elif is_audio_rtf: + suffix = "" + unit_suffix = "" mean_attr_name = f"mean_{metric_attribute_name}{suffix}" mean_value = getattr(metrics, mean_attr_name, 0.0) print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}") @@ -212,6 +223,9 @@ def calculate_metrics( good_completed += 1 if completed == 0: + warnings.formatwarning = ( + lambda msg, category, filename, lineno, line=None: f"{filename}:{lineno}: {category.__name__}: {msg}\n" + ) warnings.warn( "All requests failed. This is likely due to a misconfiguration on the benchmark arguments.", stacklevel=2, @@ -296,7 +310,11 @@ def calculate_metrics( std_audio_ttfp_ms=np.std(audio_ttfps or 0) * 1000, median_audio_ttfp_ms=np.median(audio_ttfps or 0) * 1000, percentiles_audio_ttfp_ms=[(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles], - total_audio_duration_ms=sum(audio_duration), + mean_audio_duration_s=np.mean(audio_duration or 0), + std_audio_duration_s=np.std(audio_duration or 0), + median_audio_duration_s=np.median(audio_duration or 0), + percentiles_audio_duration_s=[(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles], + total_audio_duration_s=sum(audio_duration), total_audio_frames=sum(audio_frames), audio_throughput=sum(audio_duration) / dur_s, mean_audio_rtf=np.mean(audio_rtfs or 0), diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index cf2e5c195..775568a72 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -113,8 +113,8 @@ async def async_request_openai_chat_omni_completions( st = time.perf_counter() output.start_time = st most_recent_timestamp = st + timestamp = st audio_generate_time = 0.0 - audio_first_timestamp = st try: async with session.post(url=api_url, json=payload, headers=headers) as response: if response.status == 200: @@ -133,7 +133,6 @@ async def async_request_openai_chat_omni_completions( continue chunk = message.removeprefix("data: ") - if chunk != "[DONE]": timestamp = time.perf_counter() data = json.loads(chunk) @@ -148,11 +147,11 @@ async def async_request_openai_chat_omni_completions( else: output.itl.append(timestamp - most_recent_timestamp) generated_text += content or "" + most_recent_timestamp = timestamp elif modality == "audio": if output.audio_ttfp == 0.0: - audio_first_timestamp = timestamp output.audio_ttfp = timestamp - st - audio_generate_time = timestamp - audio_first_timestamp + audio_generate_time = timestamp - st if content != "": audio_bytes = base64.b64decode(content) seg = AudioSegment.from_file(io.BytesIO(audio_bytes)) @@ -164,8 +163,8 @@ async def async_request_openai_chat_omni_completions( elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") - most_recent_timestamp = timestamp + output.latency = timestamp - st output.generated_text = generated_text if generated_audio is not None: output.audio_duration = len(generated_audio) / 1000.0 @@ -181,9 +180,7 @@ async def async_request_openai_chat_omni_completions( else: output.audio_rtf = 0 logger.warning("Audio duration is zero") - output.success = True - output.latency = most_recent_timestamp - st else: output.error = response.reason or "" output.success = False @@ -473,6 +470,9 @@ async def benchmark( "request_goodput": metrics.request_goodput if goodput_config_dict else None, "output_throughput": metrics.output_throughput, "total_token_throughput": metrics.total_token_throughput, + "total_audio_duration_s": metrics.total_audio_duration_s, + "total_audio_frames": metrics.total_audio_frames, + "audio_throughput": metrics.audio_throughput, "input_lens": [output.prompt_len for output in outputs], "output_lens": actual_output_lens, "ttfts": [output.ttft for output in outputs], @@ -505,8 +505,20 @@ async def benchmark( if metric_attribute_name not in selected_percentile_metrics: return is_audio_rtf = metric_attribute_name == "audio_rtf" + is_audio_duration = metric_attribute_name == "audio_duration" - suffix = "" if is_audio_rtf else "_ms" + suffix = "_ms" + if is_audio_duration: + suffix = "_s" + elif is_audio_rtf: + suffix = "" + mean_attr_name = f"mean_{metric_attribute_name}{suffix}" + mean_value = getattr(metrics, mean_attr_name, 0.0) + result[mean_attr_name] = mean_value + + median_attr_name = f"median_{metric_attribute_name}{suffix}" + median_value = getattr(metrics, median_attr_name, 0.0) + result[median_attr_name] = median_value for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}"): p_word = str(int(p)) if int(p) == p else str(p) result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value