1291 lines
58 KiB
Python
1291 lines
58 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
import warnings
|
|
from collections import defaultdict
|
|
from collections.abc import Sequence
|
|
from dataclasses import field, make_dataclass
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
from vllm.benchmarks.datasets import SampleRequest
|
|
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
|
|
from vllm.benchmarks.serve import MILLISECONDS_TO_SECONDS_CONVERSION, TERM_PLOTLIB_AVAILABLE, BenchmarkMetrics, TaskType
|
|
from vllm.tokenizers import TokenizerLike
|
|
|
|
from vllm_omni.metrics import definitions as defs
|
|
|
|
_PERCENTILE_ROWS_TYPE = list[tuple[float, float]] | None
|
|
_STAGE_DURATION_MAP_TYPE = dict[str, float] | None
|
|
_FLOAT_LIST_TYPE = list[float]
|
|
_INT_LIST_TYPE = list[int]
|
|
|
|
_MULTIMODAL_BENCHMARK_FIELDS = [
|
|
("num_ttft_samples", int, field(default=0)),
|
|
("num_tpot_samples", int, field(default=0)),
|
|
("num_itl_samples", int, field(default=0)),
|
|
("num_audio_ttfp_samples", int, field(default=0)),
|
|
("num_audio_rtf_samples", int, field(default=0)),
|
|
(defs.MEAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_TTFP_MS, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_TTFP_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.TOTAL_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.TOTAL_AUDIO_FRAMES, int, field(default=0)),
|
|
(defs.AUDIO_THROUGHPUT, float, field(default=0.0)),
|
|
(defs.MEAN_AUDIO_RTF, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_RTF, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_RTF, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_RTF, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.MEAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_DURATION_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.MEAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_UNDERRUN_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.AUDIO_CONTINUITY_OK_RATE, float, field(default=1.0)),
|
|
(defs.TOTAL_IMAGES, int, field(default=0)),
|
|
(defs.IMAGE_THROUGHPUT, float, field(default=0.0)),
|
|
(defs.AVERAGE_PIXELS_PER_IMAGE, float, field(default=0.0)),
|
|
(defs.MEAN_DENOISE_STEP_LATENCY_MS, float, field(default=0.0)),
|
|
(defs.MEAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.MEDIAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.STD_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.PERCENTILES_IMAGE_GENERATION_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.TOTAL_VIDEO_DURATION_S, float, field(default=0.0)),
|
|
(defs.TOTAL_VIDEO_FRAMES, int, field(default=0)),
|
|
(defs.VIDEO_THROUGHPUT, float, field(default=0.0)),
|
|
(defs.MEAN_VIDEO_RTF, float, field(default=0.0)),
|
|
(defs.MEDIAN_VIDEO_RTF, float, field(default=0.0)),
|
|
(defs.STD_VIDEO_RTF, float, field(default=0.0)),
|
|
(defs.PERCENTILES_VIDEO_RTF, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.MEAN_VIDEO_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.MEDIAN_VIDEO_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.STD_VIDEO_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.PERCENTILES_VIDEO_GENERATION_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.MEAN_PEAK_MEMORY_MB, float, field(default=0.0)),
|
|
(defs.MEDIAN_PEAK_MEMORY_MB, float, field(default=0.0)),
|
|
(defs.STD_PEAK_MEMORY_MB, float, field(default=0.0)),
|
|
(defs.PERCENTILES_PEAK_MEMORY_MB, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.STAGE_DURATIONS_MEAN, _STAGE_DURATION_MAP_TYPE, field(default=None)),
|
|
(defs.STAGE_DURATIONS_P50, _STAGE_DURATION_MAP_TYPE, field(default=None)),
|
|
(defs.STAGE_DURATIONS_P99, _STAGE_DURATION_MAP_TYPE, field(default=None)),
|
|
]
|
|
|
|
# ``make_dataclass`` returns a runtime class that mypy treats as a variable, not
|
|
# a type. Annotate as Any so call sites can use these names in annotations.
|
|
MultiModalsBenchmarkMetrics: Any = make_dataclass(
|
|
"MultiModalsBenchmarkMetrics",
|
|
_MULTIMODAL_BENCHMARK_FIELDS,
|
|
bases=(BenchmarkMetrics,),
|
|
)
|
|
|
|
_STAGE_BENCHMARK_FIELDS = [
|
|
("stage_id", int, field(default=0)),
|
|
("stage_name", str, field(default="")),
|
|
("final_output_type", str, field(default="")),
|
|
(defs.TOTAL_OUTPUT, int, field(default=0)),
|
|
(defs.TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.VLLM_TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.VLLM_TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.VLLM_ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.AUDIO_TTFPS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.AUDIO_DURATIONS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.AUDIO_FRAMES, _INT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.MISSING_AUDIO_DURATION_COUNT, int, field(default=0)),
|
|
(defs.STAGE_GEN_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.POSTPROCESS_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
("output_unit_type", str, field(default="")),
|
|
(defs.OUTPUT_UNIT_COUNT, int, field(default=0)),
|
|
(defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.TIME_PER_OUTPUT_UNITS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.INTER_OUTPUT_LATENCIES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
]
|
|
|
|
StageBenchmarkMetrics: Any = make_dataclass(
|
|
"StageBenchmarkMetrics",
|
|
_STAGE_BENCHMARK_FIELDS,
|
|
namespace={"__doc__": "Aggregated metrics for one pipeline stage (for printing only)."},
|
|
)
|
|
|
|
|
|
def _percentile_rows_seconds(
|
|
values: list[float], selected_percentiles: list[float], *, to_ms: bool
|
|
) -> tuple[float, float, list[tuple[float, float]]]:
|
|
arr = np.asarray(values, dtype=float)
|
|
scale = 1000.0 if to_ms else 1.0
|
|
mean_v = float(np.mean(arr)) * scale
|
|
median_v = float(np.median(arr)) * scale
|
|
rows = [(float(p), float(np.percentile(arr, p)) * scale) for p in selected_percentiles]
|
|
return mean_v, median_v, rows
|
|
|
|
|
|
def _p_label(p: float) -> str:
|
|
pf = float(p)
|
|
if abs(pf - round(pf)) < 1e-9:
|
|
return str(int(round(pf)))
|
|
return str(pf)
|
|
|
|
|
|
_STREAMING_OUTPUT_UNIT_TYPES = frozenset(
|
|
{
|
|
"text",
|
|
"stream",
|
|
"audio",
|
|
}
|
|
)
|
|
|
|
_AGGREGATE_PERCENTILE_FIELD_NAMES = {
|
|
defs.AUDIO_TTFP: (
|
|
defs.MEAN_AUDIO_TTFP_MS,
|
|
defs.MEDIAN_AUDIO_TTFP_MS,
|
|
defs.PERCENTILES_AUDIO_TTFP_MS,
|
|
),
|
|
defs.AUDIO_RTF: (
|
|
defs.MEAN_AUDIO_RTF,
|
|
defs.MEDIAN_AUDIO_RTF,
|
|
defs.PERCENTILES_AUDIO_RTF,
|
|
),
|
|
defs.AUDIO_DURATION: (
|
|
defs.MEAN_AUDIO_DURATION_S,
|
|
defs.MEDIAN_AUDIO_DURATION_S,
|
|
defs.PERCENTILES_AUDIO_DURATION_S,
|
|
),
|
|
defs.AUDIO_UNDERRUN: (
|
|
defs.MEAN_AUDIO_UNDERRUN_S,
|
|
defs.MEDIAN_AUDIO_UNDERRUN_S,
|
|
defs.PERCENTILES_AUDIO_UNDERRUN_S,
|
|
),
|
|
}
|
|
|
|
|
|
def _wants_text_ttft(metrics: list[str]) -> bool:
|
|
return "ttft" in metrics
|
|
|
|
|
|
def _wants_text_tpot(metrics: list[str]) -> bool:
|
|
return "tpot" in metrics or "tpop" in metrics
|
|
|
|
|
|
def _wants_text_itl(metrics: list[str]) -> bool:
|
|
return "itl" in metrics
|
|
|
|
|
|
def _wants_stream_ttfc(metrics: list[str]) -> bool:
|
|
return "ttfc" in metrics
|
|
|
|
|
|
def _wants_stream_tpoc(metrics: list[str]) -> bool:
|
|
return "tpoc" in metrics or "tpop" in metrics
|
|
|
|
|
|
def _wants_stream_icl(metrics: list[str]) -> bool:
|
|
return "icl" in metrics
|
|
|
|
|
|
def _stage_modality_flags(
|
|
final_output_type: str,
|
|
output_unit_type: str,
|
|
) -> tuple[bool, bool, bool, bool, bool]:
|
|
is_text_stage = final_output_type == "text" or output_unit_type == "text"
|
|
is_audio_stage = final_output_type == "audio" or output_unit_type == "audio"
|
|
is_video_stage = final_output_type in {"video", "videos"} or output_unit_type == "video"
|
|
# Video diffusion may still report output_unit_type="image" when frames are
|
|
# stored in ``images``; prefer video when final_output_type says so.
|
|
is_image_stage = (not is_video_stage) and (final_output_type in {"image", "images"} or output_unit_type == "image")
|
|
is_internal_stream_stage = (
|
|
output_unit_type in _STREAMING_OUTPUT_UNIT_TYPES and not is_text_stage and not is_audio_stage
|
|
)
|
|
return (
|
|
is_text_stage,
|
|
is_audio_stage,
|
|
is_image_stage,
|
|
is_video_stage,
|
|
is_internal_stream_stage,
|
|
)
|
|
|
|
|
|
def has_metric_samples(metrics: object, metric_name: str) -> bool:
|
|
sample_count_attr = {
|
|
"ttft": "num_ttft_samples",
|
|
"tpot": "num_tpot_samples",
|
|
"itl": "num_itl_samples",
|
|
defs.AUDIO_TTFP: "num_audio_ttfp_samples",
|
|
defs.AUDIO_RTF: "num_audio_rtf_samples",
|
|
}.get(metric_name)
|
|
return sample_count_attr is None or getattr(metrics, sample_count_attr, 0) > 0
|
|
|
|
|
|
def print_metrics(
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
max_concurrency,
|
|
request_rate,
|
|
benchmark_duration,
|
|
goodput_config_dict,
|
|
metrics: MultiModalsBenchmarkMetrics,
|
|
*,
|
|
outputs: list[RequestFuncOutput] | None = None,
|
|
selected_percentiles: list[float] | None = None,
|
|
print_stage: bool = False,
|
|
):
|
|
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
|
print("{:<40} {:<10}".format("Failed requests:", metrics.failed))
|
|
if max_concurrency is not None:
|
|
print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency))
|
|
if request_rate != float("inf"):
|
|
print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate))
|
|
print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))
|
|
print("{:<40} {:<10.2f}".format("Request throughput (req/s):", metrics.request_throughput))
|
|
if goodput_config_dict:
|
|
print("{:<40} {:<10.2f}".format("Request goodput (req/s):", metrics.request_goodput))
|
|
if isinstance(metrics, MultiModalsBenchmarkMetrics):
|
|
print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests))
|
|
print_peak_memory_metrics(metrics)
|
|
print_stage_durations_metrics(selected_percentiles or [], metrics)
|
|
if task_type != TaskType.GENERATION or "e2el" in selected_percentile_metrics:
|
|
process_one_metric("e2el", metrics)
|
|
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
|
if task_type == TaskType.GENERATION:
|
|
if _has_audio_output(metrics):
|
|
print_audio_metrics(selected_percentile_metrics, metrics)
|
|
if _has_image_output(metrics):
|
|
print_image_metrics(selected_percentiles or [], metrics)
|
|
if _has_video_output(metrics):
|
|
print_video_metrics(selected_percentiles or [], metrics)
|
|
if print_stage and outputs and selected_percentiles is not None:
|
|
stage_metrics = _build_stage_metrics_from_outputs(outputs)
|
|
if stage_metrics:
|
|
print("\n{s:{c}^{n}}".format(s=" Stage Benchmark Result ", n=50, c="="))
|
|
for sm in stage_metrics:
|
|
print_stage_metrics(
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
selected_percentiles,
|
|
sm,
|
|
)
|
|
print("=" * 50)
|
|
|
|
|
|
def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
|
# Pure image/video runs have no user-facing text tokens; skip the whole
|
|
# Text Result section (token throughput / peak would be misleading).
|
|
if metrics.total_output <= 0 and (_has_image_output(metrics) or _has_video_output(metrics)):
|
|
return
|
|
|
|
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
|
|
if isinstance(metrics, MultiModalsBenchmarkMetrics):
|
|
print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))
|
|
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", metrics.output_throughput))
|
|
print("{:<40} {:<10.2f}".format("Peak output token throughput (tok/s):", metrics.max_output_tokens_per_s))
|
|
print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests))
|
|
print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput))
|
|
|
|
if task_type == TaskType.GENERATION:
|
|
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
|
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
|
has_text_output = metrics.total_output > 0
|
|
if has_text_output:
|
|
if _wants_text_ttft(selected_percentile_metrics):
|
|
process_one_metric("ttft", metrics)
|
|
if _wants_text_tpot(selected_percentile_metrics):
|
|
process_one_metric("tpot", metrics)
|
|
if _wants_text_itl(selected_percentile_metrics):
|
|
process_one_metric("itl", metrics)
|
|
|
|
|
|
def _has_audio_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
return bool(
|
|
getattr(metrics, defs.TOTAL_AUDIO_DURATION_S, 0.0) > 0 or getattr(metrics, defs.TOTAL_AUDIO_FRAMES, 0) > 0
|
|
)
|
|
|
|
|
|
def _has_image_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
return int(getattr(metrics, defs.TOTAL_IMAGES, 0) or 0) > 0
|
|
|
|
|
|
def _has_video_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
return (
|
|
float(getattr(metrics, defs.TOTAL_VIDEO_DURATION_S, 0.0) or 0.0) > 0.0
|
|
or int(getattr(metrics, defs.TOTAL_VIDEO_FRAMES, 0) or 0) > 0
|
|
or float(getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS, 0.0) or 0.0) > 0.0
|
|
)
|
|
|
|
|
|
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
|
print(
|
|
"{:<40} {:<10.2f}".format("Total audio duration generated(s):", getattr(metrics, defs.TOTAL_AUDIO_DURATION_S))
|
|
)
|
|
print("{:<40} {:<10}".format("Total audio frames generated:", getattr(metrics, defs.TOTAL_AUDIO_FRAMES)))
|
|
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", getattr(metrics, defs.AUDIO_THROUGHPUT)))
|
|
print(
|
|
"{:<40} {:<10.2%}".format(
|
|
"Streaming continuity OK rate:",
|
|
getattr(metrics, defs.AUDIO_CONTINUITY_OK_RATE),
|
|
)
|
|
)
|
|
for metric in selected_percentile_metrics:
|
|
if metric.startswith("audio"):
|
|
process_one_metric(metric, metrics)
|
|
|
|
|
|
def print_peak_memory_metrics(metrics: MultiModalsBenchmarkMetrics):
|
|
if getattr(metrics, defs.MEAN_PEAK_MEMORY_MB) <= 0:
|
|
return
|
|
print("{s:{c}^{n}}".format(s="Peak Memory", n=50, c="-"))
|
|
print("{:<40} {:<10.2f}".format("Mean PEAK_MEMORY_MB (MB):", getattr(metrics, defs.MEAN_PEAK_MEMORY_MB)))
|
|
print("{:<40} {:<10.2f}".format("Median PEAK_MEMORY_MB (MB):", getattr(metrics, defs.MEDIAN_PEAK_MEMORY_MB)))
|
|
for p, value in getattr(metrics, defs.PERCENTILES_PEAK_MEMORY_MB) or []:
|
|
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} PEAK_MEMORY_MB (MB):", value))
|
|
|
|
|
|
def aggregate_stage_durations(outputs: Sequence[RequestFuncOutput]) -> dict[str, dict[str, float]]:
|
|
"""Aggregate pipeline profiler timings into mean/p50/p99 maps.
|
|
|
|
``stage_N_gen_ms`` is omitted. That clock already belongs to
|
|
``stage_gen_time`` / image / video generation, not to this profiler block.
|
|
The maps are stored on ``MultiModalsBenchmarkMetrics`` and printed with the
|
|
other run-level metrics. ``--print-stage`` does not gate them.
|
|
"""
|
|
stage_duration_lists: dict[str, list[float]] = {}
|
|
for output in outputs:
|
|
if not getattr(output, "success", False):
|
|
continue
|
|
stage_durations = getattr(output, "stage_durations", None)
|
|
if not isinstance(stage_durations, dict):
|
|
continue
|
|
for stage, duration in stage_durations.items():
|
|
if _is_printed_stage_gen_key(str(stage)):
|
|
continue
|
|
if isinstance(duration, bool) or not isinstance(duration, (int, float)) or not np.isfinite(duration):
|
|
continue
|
|
stage_duration_lists.setdefault(str(stage), []).append(float(duration))
|
|
if not stage_duration_lists:
|
|
return {}
|
|
return {
|
|
defs.STAGE_DURATIONS_MEAN: {stage: float(np.mean(values)) for stage, values in stage_duration_lists.items()},
|
|
defs.STAGE_DURATIONS_P50: {
|
|
stage: float(np.percentile(values, 50)) for stage, values in stage_duration_lists.items()
|
|
},
|
|
defs.STAGE_DURATIONS_P99: {
|
|
stage: float(np.percentile(values, 99)) for stage, values in stage_duration_lists.items()
|
|
},
|
|
}
|
|
|
|
|
|
def _stage_duration_display_name(stage: str) -> str:
|
|
"""Strip pipeline-class prefix and prettify profiler keys for printing.
|
|
|
|
``Wan22I2VPipeline.diffuse`` -> ``Diffuse``
|
|
``Wan22I2VPipeline.text_encoder.forward`` -> ``Text Encoder Forward``
|
|
``queue_wait_ms`` -> ``Queue Wait``
|
|
"""
|
|
name = str(stage)
|
|
head, sep, tail = name.partition(".")
|
|
if sep and head.endswith("Pipeline"):
|
|
name = tail
|
|
if name.endswith("_ms"):
|
|
name = name[: -len("_ms")]
|
|
return name.replace("_", " ").replace(".", " ").title()
|
|
|
|
|
|
def _is_printed_stage_gen_key(stage: str) -> bool:
|
|
"""True for ``stage_0_gen_ms``, which duplicates ``--print-stage`` stage timing."""
|
|
name = str(stage)
|
|
prefix, suffix = "stage_", "_gen_ms"
|
|
if not name.startswith(prefix) or not name.endswith(suffix):
|
|
return False
|
|
return name[len(prefix) : -len(suffix)].isdigit()
|
|
|
|
|
|
def print_stage_durations_metrics(
|
|
selected_percentiles: list[float],
|
|
metrics: MultiModalsBenchmarkMetrics,
|
|
) -> None:
|
|
"""Print profiler timings already stored on ``metrics``.
|
|
|
|
Mean and median always print, matching :func:`print_image_metrics`. P99 prints
|
|
only when 99 is in ``selected_percentiles``. ``stage_N_gen_ms`` is not here;
|
|
that clock stays on stage_gen_time / image / video generation.
|
|
"""
|
|
mean = getattr(metrics, defs.STAGE_DURATIONS_MEAN, None) or {}
|
|
if not mean:
|
|
return
|
|
p50 = getattr(metrics, defs.STAGE_DURATIONS_P50, None) or {}
|
|
p99 = getattr(metrics, defs.STAGE_DURATIONS_P99, None) or {}
|
|
show_p99 = any(float(p) == 99.0 for p in selected_percentiles)
|
|
print("{s:{c}^{n}}".format(s=" Stage Durations ", n=50, c="-"))
|
|
for stage in mean:
|
|
label = _stage_duration_display_name(stage)
|
|
unit = " (ms)" if str(stage).endswith("_ms") else " (s)"
|
|
print("{:<40} {:<10.2f}".format(f"Mean {label}{unit}:", mean[stage]))
|
|
if stage in p50:
|
|
print("{:<40} {:<10.2f}".format(f"Median {label}{unit}:", p50[stage]))
|
|
if show_p99 and stage in p99:
|
|
print("{:<40} {:<10.2f}".format(f"P99 {label}{unit}:", p99[stage]))
|
|
|
|
|
|
def print_image_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics):
|
|
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Total images generated:", getattr(metrics, defs.TOTAL_IMAGES)))
|
|
print("{:<40} {:<10.2f}".format("Image throughput (img/s):", getattr(metrics, defs.IMAGE_THROUGHPUT)))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Average pixels per image:",
|
|
getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
|
)
|
|
)
|
|
if getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS) > 0:
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Mean denoise step latency (ms):",
|
|
getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
|
)
|
|
)
|
|
if getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS) > 0:
|
|
print("{s:{c}^{n}}".format(s=" Image Generation ", n=50, c="-"))
|
|
print("{:<40} {:<10.2f}".format("Mean IMAGE_GENERATION (ms):", getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS)))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Median IMAGE_GENERATION (ms):",
|
|
getattr(metrics, defs.MEDIAN_IMAGE_GENERATION_MS),
|
|
)
|
|
)
|
|
for p, value in getattr(metrics, defs.PERCENTILES_IMAGE_GENERATION_MS) or []:
|
|
if p in selected_percentiles:
|
|
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} IMAGE_GENERATION (ms):", value))
|
|
|
|
|
|
def print_video_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics):
|
|
print("{s:{c}^{n}}".format(s=" Video Result ", n=50, c="="))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Total video duration generated(s):",
|
|
getattr(metrics, defs.TOTAL_VIDEO_DURATION_S),
|
|
)
|
|
)
|
|
print("{:<40} {:<10}".format("Total video frames generated:", getattr(metrics, defs.TOTAL_VIDEO_FRAMES)))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Video throughput(video duration/s):",
|
|
getattr(metrics, defs.VIDEO_THROUGHPUT),
|
|
)
|
|
)
|
|
if getattr(metrics, defs.MEAN_VIDEO_RTF) > 0:
|
|
print("{s:{c}^{n}}".format(s=" Video RTF ", n=50, c="-"))
|
|
print("{:<40} {:<10.2f}".format("Mean VIDEO_RTF:", getattr(metrics, defs.MEAN_VIDEO_RTF)))
|
|
print("{:<40} {:<10.2f}".format("Median VIDEO_RTF:", getattr(metrics, defs.MEDIAN_VIDEO_RTF)))
|
|
for p, value in getattr(metrics, defs.PERCENTILES_VIDEO_RTF) or []:
|
|
if p in selected_percentiles:
|
|
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} VIDEO_RTF:", value))
|
|
if getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS) > 0:
|
|
print("{s:{c}^{n}}".format(s=" Video Generation ", n=50, c="-"))
|
|
print("{:<40} {:<10.2f}".format("Mean VIDEO_GENERATION (ms):", getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS)))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Median VIDEO_GENERATION (ms):",
|
|
getattr(metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
|
|
)
|
|
)
|
|
for p, value in getattr(metrics, defs.PERCENTILES_VIDEO_GENERATION_MS) or []:
|
|
if p in selected_percentiles:
|
|
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} VIDEO_GENERATION (ms):", value))
|
|
|
|
|
|
def process_one_metric(
|
|
metric_attribute_name: str,
|
|
metrics: MultiModalsBenchmarkMetrics,
|
|
):
|
|
if not has_metric_samples(metrics, metric_attribute_name):
|
|
return
|
|
|
|
metric_header_map = {
|
|
"ttft": "Time to First Token",
|
|
"tpot": "Time per Output Token (excl. 1st token)",
|
|
"itl": "Inter-token Latency",
|
|
"e2el": "End-to-end Latency",
|
|
defs.AUDIO_TTFP: "Time to First Packet",
|
|
defs.AUDIO_RTF: "Real Time Factor",
|
|
defs.AUDIO_DURATION: "Audio Duration",
|
|
defs.AUDIO_UNDERRUN: "Streaming Audio Underrun",
|
|
}
|
|
|
|
header = metric_header_map.get(metric_attribute_name, metric_attribute_name)
|
|
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
|
|
|
is_audio_rtf = metric_attribute_name == defs.AUDIO_RTF
|
|
is_audio_duration_or_underrun = metric_attribute_name in (defs.AUDIO_DURATION, defs.AUDIO_UNDERRUN)
|
|
|
|
suffix = "_ms"
|
|
unit_suffix = " (ms)"
|
|
if is_audio_duration_or_underrun:
|
|
suffix = "_s"
|
|
unit_suffix = " (s)"
|
|
elif is_audio_rtf:
|
|
suffix = ""
|
|
unit_suffix = ""
|
|
field_names = _AGGREGATE_PERCENTILE_FIELD_NAMES.get(metric_attribute_name)
|
|
mean_attr_name = field_names[0] if field_names else f"mean_{metric_attribute_name}{suffix}"
|
|
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
|
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
|
|
|
median_attr_name = field_names[1] if field_names else f"median_{metric_attribute_name}{suffix}"
|
|
median_value = getattr(metrics, median_attr_name, 0.0)
|
|
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
|
|
|
percentiles_attr_name = field_names[2] if field_names else f"percentiles_{metric_attribute_name}{suffix}"
|
|
percentiles = getattr(metrics, percentiles_attr_name, [])
|
|
|
|
for percentile, value in percentiles:
|
|
p_str = str(int(percentile)) if percentile.is_integer() else str(percentile)
|
|
label = f"P{p_str} {metric_attribute_name.upper()}{unit_suffix}:"
|
|
print(f"{label:<40} {value:<10.2f}")
|
|
|
|
|
|
def _print_percentile_metric(
|
|
header: str,
|
|
label: str,
|
|
values: list[float],
|
|
selected_percentiles: list[float],
|
|
*,
|
|
values_are_ms: bool = False,
|
|
to_ms: bool = True,
|
|
) -> None:
|
|
if not values:
|
|
return
|
|
values_s = [v / 1000.0 for v in values] if values_are_ms else values
|
|
mean_v, med_v, pcts = _percentile_rows_seconds(values_s, selected_percentiles, to_ms=to_ms)
|
|
unit_suffix = " (ms)" if to_ms else ""
|
|
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
|
print(f"{f'Mean {label}{unit_suffix}:':<40} {mean_v:<10.2f}")
|
|
print(f"{f'Median {label}{unit_suffix}:':<40} {med_v:<10.2f}")
|
|
for p, v in pcts:
|
|
p_str = _p_label(p)
|
|
print(f"{f'P{p_str} {label}{unit_suffix}:':<40} {v:<10.2f}")
|
|
|
|
|
|
def _print_stage_timing(sm: StageBenchmarkMetrics, selected_percentiles: list[float]) -> None:
|
|
_print_percentile_metric(
|
|
"Stage Timing",
|
|
"stage_gen_time",
|
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
postprocess_times_ms = getattr(sm, defs.POSTPROCESS_TIMES_MS)
|
|
if postprocess_times_ms and any(v > 0 for v in postprocess_times_ms):
|
|
_print_percentile_metric(
|
|
"Stage Postprocess",
|
|
"postprocess",
|
|
postprocess_times_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def _print_text_stage_metrics(
|
|
task_type: TaskType,
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Stage generated tokens:", getattr(sm, defs.TOTAL_OUTPUT)))
|
|
|
|
if task_type != TaskType.GENERATION:
|
|
return
|
|
if _wants_text_ttft(selected_percentile_metrics):
|
|
_print_percentile_metric(
|
|
"Serving Time to First Token",
|
|
"Serving TTFT",
|
|
getattr(sm, defs.TTFTS),
|
|
selected_percentiles,
|
|
)
|
|
_print_percentile_metric("Time to First Token", "TTFT", getattr(sm, defs.VLLM_TTFTS), selected_percentiles)
|
|
if _wants_text_tpot(selected_percentile_metrics):
|
|
_print_percentile_metric(
|
|
"Time per Output Token (excl. 1st token)",
|
|
"TPOT",
|
|
getattr(sm, defs.VLLM_TPOTS),
|
|
selected_percentiles,
|
|
)
|
|
if _wants_text_itl(selected_percentile_metrics):
|
|
_print_percentile_metric("Inter-token Latency", "ITL", getattr(sm, defs.VLLM_ITLS), selected_percentiles)
|
|
|
|
|
|
def _print_audio_stage_metrics(
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
total_duration = float(sum(getattr(sm, defs.AUDIO_DURATIONS)))
|
|
total_frames = int(sum(getattr(sm, defs.AUDIO_FRAMES)))
|
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
|
print("{:<40} {:<10.2f}".format("Stage audio duration generated(s):", total_duration))
|
|
print("{:<40} {:<10}".format("Stage audio frames generated:", total_frames))
|
|
if defs.AUDIO_TTFP in selected_percentile_metrics:
|
|
_print_percentile_metric(
|
|
"Serving Time to First Packet",
|
|
"Serving AUDIO_TTFP",
|
|
getattr(sm, defs.AUDIO_TTFPS),
|
|
selected_percentiles,
|
|
)
|
|
if defs.AUDIO_DURATION in selected_percentile_metrics:
|
|
audio_durations = getattr(sm, defs.AUDIO_DURATIONS)
|
|
if audio_durations:
|
|
_print_percentile_metric(
|
|
"Audio Duration",
|
|
"AUDIO_DURATION",
|
|
audio_durations,
|
|
selected_percentiles,
|
|
to_ms=False,
|
|
)
|
|
elif getattr(sm, defs.MISSING_AUDIO_DURATION_COUNT) > 0:
|
|
print("{:<40} {:<10}".format("AUDIO_DURATION skipped:", "missing stage-local audio duration/sample rate"))
|
|
|
|
|
|
def _print_image_stage_metrics(
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
image_num = getattr(sm, defs.OUTPUT_UNIT_COUNT) if getattr(sm, "output_unit_type") == "image" else 0
|
|
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Total images generated:", image_num))
|
|
_print_percentile_metric(
|
|
"Image Generation",
|
|
"IMAGE_GENERATION",
|
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def _print_video_stage_metrics(
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
print("{s:{c}^{n}}".format(s=" Video Result ", n=50, c="="))
|
|
_print_percentile_metric(
|
|
"Video Generation",
|
|
"VIDEO_GENERATION",
|
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def _print_internal_stream_stage_metrics(
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
print("{s:{c}^{n}}".format(s=" Internal Stream Result ", n=50, c="="))
|
|
serving_time_to_first_outputs_ms = getattr(sm, defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS)
|
|
if _wants_stream_ttfc(selected_percentile_metrics) and serving_time_to_first_outputs_ms:
|
|
_print_percentile_metric(
|
|
"Serving Time to First Chunk",
|
|
"Serving TTFC",
|
|
serving_time_to_first_outputs_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
time_per_output_units_ms = getattr(sm, defs.TIME_PER_OUTPUT_UNITS_MS)
|
|
if _wants_stream_tpoc(selected_percentile_metrics) and time_per_output_units_ms:
|
|
_print_percentile_metric(
|
|
"Time per Output Chunk (excl. 1st chunk)",
|
|
"TPOP",
|
|
time_per_output_units_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
inter_output_latencies_ms = getattr(sm, defs.INTER_OUTPUT_LATENCIES_MS)
|
|
if _wants_stream_icl(selected_percentile_metrics) and inter_output_latencies_ms:
|
|
_print_percentile_metric(
|
|
"Inter-chunk Latency",
|
|
"ICL",
|
|
inter_output_latencies_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def print_stage_metrics(
|
|
task_type: TaskType,
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
):
|
|
title = f" Stage {getattr(sm, 'stage_id')} ({getattr(sm, 'stage_name')}) "
|
|
(
|
|
is_text_stage,
|
|
is_audio_stage,
|
|
is_image_stage,
|
|
is_video_stage,
|
|
is_internal_stream_stage,
|
|
) = _stage_modality_flags(getattr(sm, "final_output_type"), getattr(sm, "output_unit_type"))
|
|
|
|
print("{s:{c}^{n}}".format(s=title, n=50, c="="))
|
|
if is_video_stage:
|
|
_print_video_stage_metrics(selected_percentiles, sm)
|
|
return
|
|
|
|
if is_image_stage:
|
|
_print_image_stage_metrics(selected_percentiles, sm)
|
|
return
|
|
|
|
_print_stage_timing(sm, selected_percentiles)
|
|
if is_text_stage:
|
|
_print_text_stage_metrics(task_type, selected_percentile_metrics, selected_percentiles, sm)
|
|
elif is_audio_stage:
|
|
_print_audio_stage_metrics(
|
|
selected_percentile_metrics,
|
|
selected_percentiles,
|
|
sm,
|
|
)
|
|
elif is_internal_stream_stage:
|
|
_print_internal_stream_stage_metrics(selected_percentile_metrics, selected_percentiles, sm)
|
|
|
|
|
|
def _build_stage_metrics_from_outputs(
|
|
outputs: list[RequestFuncOutput],
|
|
) -> list[StageBenchmarkMetrics]:
|
|
"""Aggregate per ``stage_id`` using ``stage_metrics`` snapshots from the client."""
|
|
buckets: dict[str, list[tuple[RequestFuncOutput, dict]]] = defaultdict(list)
|
|
for out in outputs:
|
|
if not getattr(out, "success", False):
|
|
continue
|
|
smap = getattr(out, "stage_metrics", None) or {}
|
|
if not isinstance(smap, dict) or not smap:
|
|
continue
|
|
for sid, info in smap.items():
|
|
if isinstance(info, dict):
|
|
buckets[str(sid)].append((out, info))
|
|
|
|
result: list[StageBenchmarkMetrics] = []
|
|
for sid in sorted(buckets.keys(), key=lambda x: int(x)):
|
|
rows = buckets[sid]
|
|
stage_id_int = int(sid)
|
|
stage_name = str((rows[0][1] or {}).get("stage_name") or f"stage_{stage_id_int}")
|
|
final_output_type = str((rows[0][1] or {}).get("final_output_type") or "unknown")
|
|
output_unit_types = [
|
|
str((info or {}).get("output_unit_type") or "") for _, info in rows if (info or {}).get("output_unit_type")
|
|
]
|
|
output_unit_type = output_unit_types[0] if output_unit_types else "other"
|
|
|
|
is_text_stage, is_audio_stage, _, _, is_internal_stream_stage = _stage_modality_flags(
|
|
final_output_type, output_unit_type
|
|
)
|
|
total_output = 0
|
|
|
|
ttfts: list[float] = []
|
|
tpots: list[float] = []
|
|
itls: list[float] = []
|
|
vllm_ttfts: list[float] = []
|
|
vllm_tpots: list[float] = []
|
|
vllm_itls: list[float] = []
|
|
stage_gen_times_ms = [float((info or {}).get(defs.STAGE_GEN_TIME_MS) or 0.0) for _, info in rows]
|
|
postprocess_times_ms = [float((info or {}).get(defs.POSTPROCESS_TIME_MS) or 0.0) for _, info in rows]
|
|
output_unit_count = sum(int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) for _, info in rows)
|
|
inter_output_latencies_ms: list[float] = []
|
|
serving_time_to_first_outputs_ms: list[float] = []
|
|
time_per_output_units_ms: list[float] = []
|
|
if is_text_stage or is_internal_stream_stage:
|
|
serving_time_to_first_outputs_ms = [
|
|
float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
for _, info in rows
|
|
if (info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) is not None
|
|
]
|
|
time_per_output_units_ms = [
|
|
float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
|
for _, info in rows
|
|
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1
|
|
and (info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) is not None
|
|
]
|
|
for _, info in rows:
|
|
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
|
if isinstance(values, list):
|
|
inter_output_latencies_ms.extend(float(v or 0.0) for v in values)
|
|
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
|
inter_output_latencies_ms.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0))
|
|
|
|
if is_text_stage:
|
|
for _, info in rows:
|
|
ttft_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
if ttft_ms > 0:
|
|
ttfts.append(ttft_ms / 1000.0)
|
|
tpot_ms = float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
|
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1 and tpot_ms > 0:
|
|
tpots.append(tpot_ms / 1000.0)
|
|
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
|
if isinstance(values, list):
|
|
itls.extend(float(v or 0.0) / 1000.0 for v in values)
|
|
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
|
itls.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0) / 1000.0)
|
|
vllm_ttft_ms = float((info or {}).get(defs.VLLM_TTFT_MS) or 0.0)
|
|
if vllm_ttft_ms > 0:
|
|
vllm_ttfts.append(vllm_ttft_ms / 1000.0)
|
|
vllm_tpot_ms = float((info or {}).get(defs.VLLM_TPOT_MS) or 0.0)
|
|
if vllm_tpot_ms > 0:
|
|
vllm_tpots.append(vllm_tpot_ms / 1000.0)
|
|
vllm_values = (info or {}).get(defs.VLLM_ITLS_MS)
|
|
if isinstance(vllm_values, list):
|
|
vllm_itls.extend(float(v or 0.0) / 1000.0 for v in vllm_values)
|
|
elif (info or {}).get(defs.VLLM_ITL_MS) is not None:
|
|
vllm_itls.append(float((info or {}).get(defs.VLLM_ITL_MS) or 0.0) / 1000.0)
|
|
total_output = sum(int((info or {}).get(defs.NUM_TOKENS_OUT) or 0) for _, info in rows)
|
|
|
|
audio_ttfps: list[float] = []
|
|
audio_durations: list[float] = []
|
|
audio_frames: list[int] = []
|
|
missing_audio_duration_count = 0
|
|
if is_audio_stage:
|
|
for _, info in rows:
|
|
audio_ttfp_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
if audio_ttfp_ms > 0:
|
|
audio_ttfps.append(audio_ttfp_ms / 1000.0)
|
|
|
|
frame_count = int((info or {}).get(defs.AUDIO_FRAMES) or 0)
|
|
if frame_count <= 0 and (info or {}).get("output_unit_type") == "audio":
|
|
frame_count = int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0)
|
|
audio_frames.append(frame_count)
|
|
|
|
duration_s = float((info or {}).get(f"{defs.AUDIO_DURATION}_s") or 0.0)
|
|
if duration_s > 0:
|
|
audio_durations.append(duration_s)
|
|
else:
|
|
missing_audio_duration_count += 1
|
|
|
|
result.append(
|
|
StageBenchmarkMetrics(
|
|
**{
|
|
"stage_id": stage_id_int,
|
|
"stage_name": stage_name,
|
|
"final_output_type": final_output_type,
|
|
defs.TOTAL_OUTPUT: total_output,
|
|
defs.TTFTS: ttfts,
|
|
defs.TPOTS: tpots,
|
|
defs.ITLS: itls,
|
|
defs.VLLM_TTFTS: vllm_ttfts,
|
|
defs.VLLM_TPOTS: vllm_tpots,
|
|
defs.VLLM_ITLS: vllm_itls,
|
|
defs.AUDIO_TTFPS: audio_ttfps,
|
|
defs.AUDIO_DURATIONS: audio_durations,
|
|
defs.AUDIO_FRAMES: audio_frames,
|
|
defs.MISSING_AUDIO_DURATION_COUNT: missing_audio_duration_count,
|
|
defs.STAGE_GEN_TIMES_MS: stage_gen_times_ms,
|
|
defs.POSTPROCESS_TIMES_MS: postprocess_times_ms,
|
|
"output_unit_type": output_unit_type,
|
|
defs.OUTPUT_UNIT_COUNT: output_unit_count,
|
|
defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS: serving_time_to_first_outputs_ms,
|
|
defs.TIME_PER_OUTPUT_UNITS_MS: time_per_output_units_ms,
|
|
defs.INTER_OUTPUT_LATENCIES_MS: inter_output_latencies_ms,
|
|
}
|
|
)
|
|
)
|
|
return result
|
|
|
|
|
|
def calculate_metrics(
|
|
input_requests: list[SampleRequest],
|
|
outputs: list[RequestFuncOutput],
|
|
dur_s: float,
|
|
tokenizer: TokenizerLike | None,
|
|
selected_percentiles: list[float],
|
|
goodput_config_dict: dict[str, float],
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
max_concurrency,
|
|
request_rate,
|
|
benchmark_duration,
|
|
print_stage: bool = False,
|
|
) -> tuple[MultiModalsBenchmarkMetrics, list[int]]:
|
|
"""Calculate the metrics for the benchmark.
|
|
|
|
Args:
|
|
input_requests: The input requests.
|
|
outputs: The outputs of the requests.
|
|
dur_s: The duration of the benchmark.
|
|
tokenizer: The tokenizer to use.
|
|
selected_percentiles: The percentiles to select.
|
|
goodput_config_dict: The goodput configuration.
|
|
|
|
Returns:
|
|
A tuple of the benchmark metrics and the actual output lengths.
|
|
"""
|
|
actual_output_lens: list[int] = []
|
|
total_input = 0
|
|
completed = 0
|
|
good_completed = 0
|
|
itls: list[float] = []
|
|
tpots: list[float] = []
|
|
all_tpots: list[float | None] = []
|
|
ttfts: list[float] = []
|
|
goodput_ttfts: list[float | None] = []
|
|
e2els: list[float] = []
|
|
audio_ttfps: list[float] = []
|
|
goodput_audio_ttfps: list[float | None] = []
|
|
audio_rtfs: list[float] = []
|
|
audio_duration: list[float] = []
|
|
audio_frames: list[int] = []
|
|
image_generation_times_ms: list[float] = []
|
|
denoise_step_latencies_ms: list[float] = []
|
|
total_images = 0
|
|
total_image_pixels = 0
|
|
video_durations: list[float] = []
|
|
video_rtfs: list[float] = []
|
|
total_video_frames = 0
|
|
video_generation_times_ms: list[float] = []
|
|
peak_memories_mb: list[float] = []
|
|
audio_underruns: list[float] = []
|
|
audio_continuity_ok: list[bool] = []
|
|
input_audio_duration = 0.0
|
|
for i in range(len(outputs)):
|
|
if outputs[i].success:
|
|
output_len = outputs[i].output_tokens
|
|
|
|
if not output_len:
|
|
# Pure image/video (no generated text) must stay at 0 tokens.
|
|
# Do not invent output_len=1 via the tokenizer-is-None placeholder.
|
|
generated_text = getattr(outputs[i], "generated_text", None) or ""
|
|
has_image = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0) > 0
|
|
has_video = (
|
|
float(getattr(outputs[i], defs.VIDEO_DURATION, 0.0) or 0.0) > 0.0
|
|
or int(getattr(outputs[i], defs.VIDEO_FRAMES, 0) or 0) > 0
|
|
or float(getattr(outputs[i], defs.VIDEO_GENERATION_TIME_MS, 0.0) or 0.0) > 0.0
|
|
)
|
|
if not generated_text and (has_image or has_video):
|
|
output_len = 0
|
|
elif tokenizer is None:
|
|
output_len = 1
|
|
else:
|
|
# We use the tokenizer to count the number of output tokens
|
|
# for some serving backends instead of looking at
|
|
# len(outputs[i].itl) since multiple output tokens may be
|
|
# bundled together
|
|
# Note : this may inflate the output token count slightly
|
|
output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids)
|
|
actual_output_lens.append(output_len)
|
|
total_input += outputs[i].prompt_len
|
|
tpot: float | None = None
|
|
if output_len > 1:
|
|
if outputs[i].itl:
|
|
# Use mean(ITL) directly so per-request TPOT == mean(ITL).
|
|
# The ITL list records one entry per SSE chunk; server may
|
|
# bundle multiple tokens per chunk, so len(itl)+1 != output_len.
|
|
# Using mean(itl) keeps TPOT and ITL on the same footing.
|
|
measured_tpot = sum(outputs[i].itl) / len(outputs[i].itl)
|
|
if np.isfinite(measured_tpot) and measured_tpot > 0:
|
|
tpot = measured_tpot
|
|
elif getattr(outputs[i], "tpot_measured", True):
|
|
text_latency = getattr(outputs[i], "text_latency", None)
|
|
ttft = getattr(outputs[i], "ttft", None)
|
|
if (
|
|
isinstance(text_latency, (int, float))
|
|
and isinstance(ttft, (int, float))
|
|
and np.isfinite(text_latency)
|
|
and np.isfinite(ttft)
|
|
and text_latency > ttft > 0
|
|
):
|
|
tpot = (text_latency - ttft) / (output_len - 1)
|
|
if tpot is not None:
|
|
tpots.append(tpot)
|
|
# TPOT is not measurable for a single-token response or when the
|
|
# client did not observe a positive decode interval. In
|
|
# particular, stage-level output token counts alone must not turn
|
|
# an unset text_latency into a negative TPOT sample.
|
|
all_tpots.append(tpot)
|
|
itls += outputs[i].itl
|
|
session_metrics = getattr(outputs[i], "duplex_session_metrics", None)
|
|
ttft_measured = not isinstance(session_metrics, dict) or session_metrics.get("ttft_ms") is not None
|
|
audio_ttfp_measured = not isinstance(session_metrics, dict) or session_metrics.get("ttfp_ms") is not None
|
|
audio_rtf_measured = not isinstance(session_metrics, dict) or session_metrics.get("rtf") is not None
|
|
if ttft_measured:
|
|
ttfts.append(outputs[i].ttft)
|
|
if audio_ttfp_measured:
|
|
audio_ttfps.append(getattr(outputs[i], defs.AUDIO_TTFP, 0.0))
|
|
if audio_rtf_measured:
|
|
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
|
goodput_ttfts.append(outputs[i].ttft if ttft_measured else None)
|
|
goodput_audio_ttfps.append(getattr(outputs[i], defs.AUDIO_TTFP, 0.0) if audio_ttfp_measured else None)
|
|
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
|
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0))
|
|
image_count = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0)
|
|
total_images += image_count
|
|
image_generation_time_ms = float(getattr(outputs[i], defs.IMAGE_GENERATION_TIME_MS, 0.0) or 0.0)
|
|
if image_generation_time_ms > 0:
|
|
image_generation_times_ms.append(image_generation_time_ms)
|
|
total_image_pixels += int(getattr(outputs[i], defs.IMAGE_PIXELS, 0) or 0)
|
|
denoise_step_latency_ms = float(getattr(outputs[i], defs.DENOISE_STEP_LATENCY_MS, 0.0) or 0.0)
|
|
if denoise_step_latency_ms > 0:
|
|
denoise_step_latencies_ms.append(denoise_step_latency_ms)
|
|
video_duration = float(getattr(outputs[i], defs.VIDEO_DURATION, 0.0) or 0.0)
|
|
if video_duration > 0:
|
|
video_durations.append(video_duration)
|
|
total_video_frames += int(getattr(outputs[i], defs.VIDEO_FRAMES, 0) or 0)
|
|
video_rtf = float(getattr(outputs[i], defs.VIDEO_RTF, 0.0) or 0.0)
|
|
if video_rtf > 0:
|
|
video_rtfs.append(video_rtf)
|
|
video_generation_time_ms = float(getattr(outputs[i], defs.VIDEO_GENERATION_TIME_MS, 0.0) or 0.0)
|
|
if video_generation_time_ms > 0:
|
|
video_generation_times_ms.append(video_generation_time_ms)
|
|
peak_memory_mb = float(getattr(outputs[i], defs.PEAK_MEMORY_MB, 0.0) or 0.0)
|
|
if peak_memory_mb > 0:
|
|
peak_memories_mb.append(peak_memory_mb)
|
|
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
|
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
|
e2els.append(outputs[i].latency)
|
|
input_audio_duration += outputs[i].input_audio_duration
|
|
completed += 1
|
|
else:
|
|
actual_output_lens.append(0)
|
|
|
|
if goodput_config_dict:
|
|
valid_metrics: list[Sequence[float | None]] = []
|
|
slo_values = []
|
|
|
|
if "ttft" in goodput_config_dict:
|
|
valid_metrics.append(goodput_ttfts)
|
|
slo_values.append(goodput_config_dict["ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
if "audio_ttft" in goodput_config_dict:
|
|
valid_metrics.append(goodput_audio_ttfps)
|
|
slo_values.append(goodput_config_dict["audio_ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
if "tpot" in goodput_config_dict:
|
|
valid_metrics.append(all_tpots)
|
|
slo_values.append(goodput_config_dict["tpot"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
if "e2el" in goodput_config_dict:
|
|
valid_metrics.append(e2els)
|
|
slo_values.append(goodput_config_dict["e2el"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
|
|
for req_metric in zip(*valid_metrics):
|
|
is_good_req = all(r is not None and s >= r for s, r in zip(slo_values, req_metric))
|
|
if is_good_req:
|
|
good_completed += 1
|
|
|
|
if completed == 0:
|
|
|
|
def _formatwarning(
|
|
message: Warning | str,
|
|
category: type[Warning],
|
|
filename: str,
|
|
lineno: int,
|
|
line: str | None = None,
|
|
) -> str:
|
|
return f"{filename}:{lineno}: {category.__name__}: {message}\n"
|
|
|
|
warnings.formatwarning = _formatwarning
|
|
warnings.warn(
|
|
"All requests failed. This is likely due to a misconfiguration on the benchmark arguments.",
|
|
stacklevel=2,
|
|
)
|
|
|
|
# Calculate max output tokens per second metric
|
|
max_output_tokens_per_s = 0.0
|
|
max_concurrent_requests = 0
|
|
token_timeline_available = True
|
|
|
|
# Find the time range across all successful requests
|
|
successful_outputs = [output for output in outputs if output.success]
|
|
failed_outputs = [output for output in outputs if not output.success]
|
|
if successful_outputs:
|
|
min_start_time = min(output.start_time for output in successful_outputs)
|
|
max_end_time = max(output.start_time + output.latency for output in successful_outputs)
|
|
|
|
# Create second buckets (ceiling to ensure we capture all time)
|
|
duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1
|
|
tokens_per_second = np.zeros(duration_seconds)
|
|
concurrent_requests_per_second = np.zeros(duration_seconds)
|
|
|
|
for i, output in enumerate(successful_outputs):
|
|
session_metrics = getattr(output, "duplex_session_metrics", None)
|
|
if isinstance(session_metrics, dict):
|
|
# Duplex TTFT/ITL samples are response-local. Without each
|
|
# response.created offset they cannot form a session-global
|
|
# token timeline, so do not publish a misleading peak rate.
|
|
token_timeline_available = False
|
|
else:
|
|
# Calculate token generation timestamp using
|
|
# start_time, ttft, and itl. Skip requests with no text tokens
|
|
# (empty itl, zero output_tokens, and no generated_text) so pure
|
|
# image/video runs do not seed a fake peak of 1 tok/s from
|
|
# start_time+ttft alone.
|
|
output_token_count = int(getattr(output, "output_tokens", 0) or 0)
|
|
has_generated_text = bool(getattr(output, "generated_text", None) or "")
|
|
if output_token_count > 0 or output.itl or has_generated_text:
|
|
token_times = [output.start_time + output.ttft]
|
|
current_time = token_times[0]
|
|
for itl_value in output.itl:
|
|
current_time += itl_value
|
|
token_times.append(current_time)
|
|
|
|
# Add tokens to second buckets
|
|
for token_time in token_times:
|
|
second_bucket = int(token_time - min_start_time)
|
|
if 0 <= second_bucket < duration_seconds:
|
|
tokens_per_second[second_bucket] += 1
|
|
|
|
# Track concurrent requests for each second this request was active
|
|
request_start_second = int(output.start_time - min_start_time)
|
|
request_end_second = int((output.start_time + output.latency) - min_start_time)
|
|
|
|
for second in range(request_start_second, request_end_second + 1):
|
|
concurrent_requests_per_second[second] += 1
|
|
|
|
# Find the maximum tokens per second and corresponding
|
|
# concurrent requests
|
|
if len(tokens_per_second) > 0:
|
|
if token_timeline_available:
|
|
max_output_tokens_per_s = float(np.max(tokens_per_second))
|
|
else:
|
|
max_output_tokens_per_s = float("nan")
|
|
max_concurrent_requests = int(np.max(concurrent_requests_per_second))
|
|
|
|
if TERM_PLOTLIB_AVAILABLE:
|
|
import termplotlib as tpl
|
|
|
|
fig = tpl.figure()
|
|
if token_timeline_available:
|
|
fig.plot(
|
|
np.arange(len(tokens_per_second)),
|
|
tokens_per_second,
|
|
title="Output tokens per second",
|
|
)
|
|
fig.plot(
|
|
np.arange(len(concurrent_requests_per_second)),
|
|
concurrent_requests_per_second,
|
|
title="Concurrent requests per second",
|
|
)
|
|
fig.show()
|
|
else:
|
|
print("tip: install termplotlib and gnuplot to plot the metrics")
|
|
|
|
duplex_metrics_present = any(
|
|
isinstance(getattr(output, "duplex_session_metrics", None), dict) for output in outputs
|
|
)
|
|
missing_duplex_value = float("nan") if duplex_metrics_present else 0
|
|
stage_duration_summaries = aggregate_stage_durations(outputs)
|
|
metrics = MultiModalsBenchmarkMetrics(
|
|
completed=completed,
|
|
failed=len(failed_outputs),
|
|
total_input=total_input,
|
|
total_output=sum(actual_output_lens),
|
|
request_throughput=completed / dur_s,
|
|
request_goodput=good_completed / dur_s,
|
|
output_throughput=sum(actual_output_lens) / dur_s,
|
|
total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
|
|
mean_ttft_ms=np.mean(ttfts or missing_duplex_value) * 1000,
|
|
std_ttft_ms=np.std(ttfts or missing_duplex_value) * 1000,
|
|
median_ttft_ms=np.median(ttfts or missing_duplex_value) * 1000,
|
|
percentiles_ttft_ms=[(p, np.percentile(ttfts or missing_duplex_value, p) * 1000) for p in selected_percentiles],
|
|
mean_tpot_ms=np.mean(tpots or float("nan")) * 1000,
|
|
std_tpot_ms=np.std(tpots or float("nan")) * 1000,
|
|
median_tpot_ms=np.median(tpots or float("nan")) * 1000,
|
|
percentiles_tpot_ms=[(p, np.percentile(tpots or float("nan"), p) * 1000) for p in selected_percentiles],
|
|
mean_itl_ms=np.mean(itls or missing_duplex_value) * 1000,
|
|
std_itl_ms=np.std(itls or missing_duplex_value) * 1000,
|
|
median_itl_ms=np.median(itls or missing_duplex_value) * 1000,
|
|
percentiles_itl_ms=[(p, np.percentile(itls or missing_duplex_value, p) * 1000) for p in selected_percentiles],
|
|
mean_e2el_ms=np.mean(e2els or 0) * 1000,
|
|
std_e2el_ms=np.std(e2els or 0) * 1000,
|
|
median_e2el_ms=np.median(e2els or 0) * 1000,
|
|
percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles],
|
|
max_output_tokens_per_s=max_output_tokens_per_s,
|
|
max_concurrent_requests=max_concurrent_requests,
|
|
rtfx=input_audio_duration / dur_s,
|
|
**{
|
|
"num_ttft_samples": len(ttfts),
|
|
"num_tpot_samples": len(tpots),
|
|
"num_itl_samples": len(itls),
|
|
"num_audio_ttfp_samples": len(audio_ttfps),
|
|
"num_audio_rtf_samples": len(audio_rtfs),
|
|
defs.MEAN_AUDIO_TTFP_MS: np.mean(audio_ttfps or missing_duplex_value) * 1000,
|
|
defs.STD_AUDIO_TTFP_MS: np.std(audio_ttfps or missing_duplex_value) * 1000,
|
|
defs.MEDIAN_AUDIO_TTFP_MS: np.median(audio_ttfps or missing_duplex_value) * 1000,
|
|
defs.PERCENTILES_AUDIO_TTFP_MS: [
|
|
(p, np.percentile(audio_ttfps or missing_duplex_value, p) * 1000) for p in selected_percentiles
|
|
],
|
|
defs.MEAN_AUDIO_DURATION_S: np.mean(audio_duration or 0),
|
|
defs.STD_AUDIO_DURATION_S: np.std(audio_duration or 0),
|
|
defs.MEDIAN_AUDIO_DURATION_S: np.median(audio_duration or 0),
|
|
defs.PERCENTILES_AUDIO_DURATION_S: [
|
|
(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.TOTAL_AUDIO_DURATION_S: sum(audio_duration),
|
|
defs.TOTAL_AUDIO_FRAMES: sum(audio_frames),
|
|
defs.AUDIO_THROUGHPUT: sum(audio_duration) / dur_s,
|
|
defs.MEAN_AUDIO_RTF: np.mean(audio_rtfs or missing_duplex_value),
|
|
defs.STD_AUDIO_RTF: np.std(audio_rtfs or missing_duplex_value),
|
|
defs.MEDIAN_AUDIO_RTF: np.median(audio_rtfs or missing_duplex_value),
|
|
defs.PERCENTILES_AUDIO_RTF: [
|
|
(p, np.percentile(audio_rtfs or missing_duplex_value, p)) for p in selected_percentiles
|
|
],
|
|
defs.TOTAL_IMAGES: total_images,
|
|
defs.IMAGE_THROUGHPUT: total_images / dur_s,
|
|
defs.AVERAGE_PIXELS_PER_IMAGE: (total_image_pixels / total_images) if total_images > 0 else 0.0,
|
|
defs.MEAN_DENOISE_STEP_LATENCY_MS: np.mean(denoise_step_latencies_ms or 0),
|
|
defs.MEAN_IMAGE_GENERATION_MS: np.mean(image_generation_times_ms or 0),
|
|
defs.STD_IMAGE_GENERATION_MS: np.std(image_generation_times_ms or 0),
|
|
defs.MEDIAN_IMAGE_GENERATION_MS: np.median(image_generation_times_ms or 0),
|
|
defs.PERCENTILES_IMAGE_GENERATION_MS: [
|
|
(p, np.percentile(image_generation_times_ms or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.TOTAL_VIDEO_DURATION_S: sum(video_durations),
|
|
defs.TOTAL_VIDEO_FRAMES: total_video_frames,
|
|
defs.VIDEO_THROUGHPUT: sum(video_durations) / dur_s,
|
|
defs.MEAN_VIDEO_RTF: np.mean(video_rtfs or 0),
|
|
defs.STD_VIDEO_RTF: np.std(video_rtfs or 0),
|
|
defs.MEDIAN_VIDEO_RTF: np.median(video_rtfs or 0),
|
|
defs.PERCENTILES_VIDEO_RTF: [(p, np.percentile(video_rtfs or 0, p)) for p in selected_percentiles],
|
|
defs.MEAN_VIDEO_GENERATION_MS: np.mean(video_generation_times_ms or 0),
|
|
defs.STD_VIDEO_GENERATION_MS: np.std(video_generation_times_ms or 0),
|
|
defs.MEDIAN_VIDEO_GENERATION_MS: np.median(video_generation_times_ms or 0),
|
|
defs.PERCENTILES_VIDEO_GENERATION_MS: [
|
|
(p, np.percentile(video_generation_times_ms or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.MEAN_PEAK_MEMORY_MB: np.mean(peak_memories_mb or 0),
|
|
defs.STD_PEAK_MEMORY_MB: np.std(peak_memories_mb or 0),
|
|
defs.MEDIAN_PEAK_MEMORY_MB: np.median(peak_memories_mb or 0),
|
|
defs.PERCENTILES_PEAK_MEMORY_MB: [
|
|
(p, np.percentile(peak_memories_mb or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.MEAN_AUDIO_UNDERRUN_S: np.mean(audio_underruns or 0),
|
|
defs.STD_AUDIO_UNDERRUN_S: np.std(audio_underruns or 0),
|
|
defs.MEDIAN_AUDIO_UNDERRUN_S: np.median(audio_underruns or 0),
|
|
defs.PERCENTILES_AUDIO_UNDERRUN_S: [
|
|
(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.AUDIO_CONTINUITY_OK_RATE: (
|
|
(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0
|
|
),
|
|
defs.STAGE_DURATIONS_MEAN: stage_duration_summaries.get(defs.STAGE_DURATIONS_MEAN) or {},
|
|
defs.STAGE_DURATIONS_P50: stage_duration_summaries.get(defs.STAGE_DURATIONS_P50) or {},
|
|
defs.STAGE_DURATIONS_P99: stage_duration_summaries.get(defs.STAGE_DURATIONS_P99) or {},
|
|
},
|
|
)
|
|
print_metrics(
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
max_concurrency,
|
|
request_rate,
|
|
benchmark_duration,
|
|
goodput_config_dict,
|
|
metrics,
|
|
outputs=outputs,
|
|
selected_percentiles=selected_percentiles,
|
|
print_stage=print_stage,
|
|
)
|
|
return metrics, actual_output_lens
|