anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Bugfix] modify RTF use audio_e2e/audio_duration (#1157)

Browse Source 
Signed-off-by: wangyu31577 <wangyu31577@hundsun.com>
Co-authored-by: wangyu31577 <wangyu31577@hundsun.com>
This commit is contained in:
wangyuandwangyu31577 authored and GitHub committed 2026-02-04 14:07:46 +08:00
1 parent 78830562db
commit 117d9bd122
3 files changed
+73 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/cli/bench/serve.md
+29 -29
View File
@@ -52,7 +52,7 @@ You can use `vllm bench serve --omni --help=all` to get descriptions of all para
- `--percentile-metrics`
"Comma-separated list of selected metrics to report percentiles."
"This argument specifies the metrics to report percentiles."
'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf". '
'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
- `--save-result`
Specify to save benchmark results to a json file
@@ -220,48 +220,48 @@ If successful, you will see the following output:
============ Serving Benchmark Result ============
Successful requests: 2
Failed requests: 0
Benchmark duration (s): 24.35
Request throughput (req/s): 0.08
Benchmark duration (s): 3.89
Request throughput (req/s): 0.51
Peak concurrent requests: 2.00
----------------End-to-end Latency----------------
Mean E2EL (ms): 22576.23
Median E2EL (ms): 22576.23
P99 E2EL (ms): 24205.72
Mean E2EL (ms): 3824.76
Median E2EL (ms): 3824.76
P99 E2EL (ms): 3888.54
================== Text Result ===================
Total input tokens: 30
Total generated tokens: 8973
Output token throughput (tok/s): 368.52
Peak output token throughput (tok/s): 81.00
Total generated tokens: 10101
Output token throughput (tok/s): 2595.57
Peak output token throughput (tok/s): 111.00
Peak concurrent requests: 2.00
Total Token throughput (tok/s): 369.76
Total Token throughput (tok/s): 2603.28
---------------Time to First Token----------------
Mean TTFT (ms): 125.16
Median TTFT (ms): 125.16
P99 TTFT (ms): 155.88
Mean TTFT (ms): 117.15
Median TTFT (ms): 117.15
P99 TTFT (ms): 142.69
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms): 5.01
Median TPOT (ms): 5.01
P99 TPOT (ms): 5.42
Mean TPOT (ms): 0.73
Median TPOT (ms): 0.73
P99 TPOT (ms): 0.74
---------------Inter-token Latency----------------
Mean ITL (ms): 34.15
Median ITL (ms): 0.01
P99 ITL (ms): 376.19
Mean ITL (ms): 16.47
Median ITL (ms): 16.19
P99 ITL (ms): 52.55
================== Audio Result ==================
Total audio duration generated(s): 3.95
Total audio frames generated: 94890
Audio throughput(audio duration/s): 0.16
Total audio duration generated(s): 15.79
Total audio frames generated: 379050
Audio throughput(audio duration/s): 4.06
---------------Time to First Packet---------------
Mean AUDIO_TTFP (ms): 11756.89
Median AUDIO_TTFP (ms): 11756.89
P99 AUDIO_TTFP (ms): 20854.25
Mean AUDIO_TTFP (ms): 3701.37
Median AUDIO_TTFP (ms): 3701.37
P99 AUDIO_TTFP (ms): 3762.25
-----------------Real Time Factor-----------------
Mean AUDIO_RTF: 3.75
Median AUDIO_RTF: 3.75
P99 AUDIO_RTF: 7.39
Mean AUDIO_RTF: 0.47
Median AUDIO_RTF: 0.47
P99 AUDIO_RTF: 0.48
==================================================
```
Notes:
We use (audio generation time - first packet latency) / audio duration to calculate RTF.
We use audio generation time / audio duration to calculate RTF.
</details>
vllm_omni/benchmarks/metrics/metrics.py
+24 -6
View File
@@ -14,13 +14,17 @@ class MultiModalsBenchmarkMetrics(BenchmarkMetrics):
median_audio_ttfp_ms: float = 0.0
std_audio_ttfp_ms: float = 0.0
percentiles_audio_ttfp_ms: list[tuple[float, float]] = None
total_audio_duration_ms: float = 0.0
total_audio_duration_s: float = 0.0
total_audio_frames: int = 0
audio_throughput: float = 0.0
mean_audio_rtf: float = 0.0
median_audio_rtf: float = 0.0
std_audio_rtf: float = 0.0
percentiles_audio_rtf: list[tuple[float, float]] = None
mean_audio_duration_s: float = 0.0
median_audio_duration_s: float = 0.0
std_audio_duration_s: float = 0.0
percentiles_audio_duration_s: list[tuple[float, float]] = None
def print_metrics(
@@ -73,7 +77,7 @@ def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiMod
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_ms))
print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s))
print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames))
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput))
for metric in selected_percentile_metrics:
@@ -92,16 +96,23 @@ def process_one_metric(
"e2el": "End-to-end Latency",
"audio_ttfp": "Time to First Packet",
"audio_rtf": "Real Time Factor",
"audio_duration": "Audio Duration",
}
header = metric_header_map.get(metric_attribute_name, metric_attribute_name)
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
is_audio_rtf = metric_attribute_name == "audio_rtf"
is_audio_duration = metric_attribute_name == "audio_duration"
suffix = "" if is_audio_rtf else "_ms"
unit_suffix = "" if is_audio_rtf else " (ms)"
suffix = "_ms"
unit_suffix = " (ms)"
if is_audio_duration:
suffix = "_s"
unit_suffix = " (s)"
elif is_audio_rtf:
suffix = ""
unit_suffix = ""
mean_attr_name = f"mean_{metric_attribute_name}{suffix}"
mean_value = getattr(metrics, mean_attr_name, 0.0)
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
@@ -212,6 +223,9 @@ def calculate_metrics(
good_completed += 1
if completed == 0:
warnings.formatwarning = (
lambda msg, category, filename, lineno, line=None: f"{filename}:{lineno}: {category.__name__}: {msg}\n"
)
warnings.warn(
"All requests failed. This is likely due to a misconfiguration on the benchmark arguments.",
stacklevel=2,
@@ -296,7 +310,11 @@ def calculate_metrics(
std_audio_ttfp_ms=np.std(audio_ttfps or 0) * 1000,
median_audio_ttfp_ms=np.median(audio_ttfps or 0) * 1000,
percentiles_audio_ttfp_ms=[(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles],
total_audio_duration_ms=sum(audio_duration),
mean_audio_duration_s=np.mean(audio_duration or 0),
std_audio_duration_s=np.std(audio_duration or 0),
median_audio_duration_s=np.median(audio_duration or 0),
percentiles_audio_duration_s=[(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles],
total_audio_duration_s=sum(audio_duration),
total_audio_frames=sum(audio_frames),
audio_throughput=sum(audio_duration) / dur_s,
mean_audio_rtf=np.mean(audio_rtfs or 0),
vllm_omni/benchmarks/patch/patch.py
+20 -8
View File
@@ -113,8 +113,8 @@ async def async_request_openai_chat_omni_completions(
st = time.perf_counter()
output.start_time = st
most_recent_timestamp = st
timestamp = st
audio_generate_time = 0.0
audio_first_timestamp = st
try:
async with session.post(url=api_url, json=payload, headers=headers) as response:
if response.status == 200:
@@ -133,7 +133,6 @@ async def async_request_openai_chat_omni_completions(
continue
chunk = message.removeprefix("data: ")
if chunk != "[DONE]":
timestamp = time.perf_counter()
data = json.loads(chunk)
@@ -148,11 +147,11 @@ async def async_request_openai_chat_omni_completions(
else:
output.itl.append(timestamp - most_recent_timestamp)
generated_text += content or ""
most_recent_timestamp = timestamp
elif modality == "audio":
if output.audio_ttfp == 0.0:
audio_first_timestamp = timestamp
output.audio_ttfp = timestamp - st
audio_generate_time = timestamp - audio_first_timestamp
audio_generate_time = timestamp - st
if content != "":
audio_bytes = base64.b64decode(content)
seg = AudioSegment.from_file(io.BytesIO(audio_bytes))
@@ -164,8 +163,8 @@ async def async_request_openai_chat_omni_completions(
elif usage := data.get("usage"):
output.output_tokens = usage.get("completion_tokens")
most_recent_timestamp = timestamp
output.latency = timestamp - st
output.generated_text = generated_text
if generated_audio is not None:
output.audio_duration = len(generated_audio) / 1000.0
@@ -181,9 +180,7 @@ async def async_request_openai_chat_omni_completions(
else:
output.audio_rtf = 0
logger.warning("Audio duration is zero")
output.success = True
output.latency = most_recent_timestamp - st
else:
output.error = response.reason or ""
output.success = False
@@ -473,6 +470,9 @@ async def benchmark(
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
"output_throughput": metrics.output_throughput,
"total_token_throughput": metrics.total_token_throughput,
"total_audio_duration_s": metrics.total_audio_duration_s,
"total_audio_frames": metrics.total_audio_frames,
"audio_throughput": metrics.audio_throughput,
"input_lens": [output.prompt_len for output in outputs],
"output_lens": actual_output_lens,
"ttfts": [output.ttft for output in outputs],
@@ -505,8 +505,20 @@ async def benchmark(
if metric_attribute_name not in selected_percentile_metrics:
return
is_audio_rtf = metric_attribute_name == "audio_rtf"
is_audio_duration = metric_attribute_name == "audio_duration"
suffix = "" if is_audio_rtf else "_ms"
suffix = "_ms"
if is_audio_duration:
suffix = "_s"
elif is_audio_rtf:
suffix = ""
mean_attr_name = f"mean_{metric_attribute_name}{suffix}"
mean_value = getattr(metrics, mean_attr_name, 0.0)
result[mean_attr_name] = mean_value
median_attr_name = f"median_{metric_attribute_name}{suffix}"
median_value = getattr(metrics, median_attr_name, 0.0)
result[median_attr_name] = median_value
for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}"):
p_word = str(int(p)) if int(p) == p else str(p)
result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value