966 lines
42 KiB
Python
966 lines
42 KiB
Python
import warnings
|
|
from collections import defaultdict
|
|
from dataclasses import field, make_dataclass
|
|
|
|
import numpy as np
|
|
from vllm.benchmarks.datasets import SampleRequest
|
|
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
|
|
from vllm.benchmarks.serve import MILLISECONDS_TO_SECONDS_CONVERSION, TERM_PLOTLIB_AVAILABLE, BenchmarkMetrics, TaskType
|
|
from vllm.tokenizers import TokenizerLike
|
|
|
|
from vllm_omni.metrics import definitions as defs
|
|
|
|
_PERCENTILE_ROWS_TYPE = list[tuple[float, float]] | None
|
|
_FLOAT_LIST_TYPE = list[float]
|
|
_INT_LIST_TYPE = list[int]
|
|
|
|
_MULTIMODAL_BENCHMARK_FIELDS = [
|
|
(defs.MEAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_TTFP_MS, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_TTFP_MS, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_TTFP_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.TOTAL_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.TOTAL_AUDIO_FRAMES, int, field(default=0)),
|
|
(defs.AUDIO_THROUGHPUT, float, field(default=0.0)),
|
|
(defs.MEAN_AUDIO_RTF, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_RTF, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_RTF, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_RTF, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.MEAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_DURATION_S, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_DURATION_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.MEAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
|
(defs.MEDIAN_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
|
(defs.STD_AUDIO_UNDERRUN_S, float, field(default=0.0)),
|
|
(defs.PERCENTILES_AUDIO_UNDERRUN_S, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
(defs.AUDIO_CONTINUITY_OK_RATE, float, field(default=1.0)),
|
|
(defs.TOTAL_IMAGES, int, field(default=0)),
|
|
(defs.IMAGE_THROUGHPUT, float, field(default=0.0)),
|
|
(defs.AVERAGE_PIXELS_PER_IMAGE, float, field(default=0.0)),
|
|
(defs.MEAN_DENOISE_STEP_LATENCY_MS, float, field(default=0.0)),
|
|
(defs.MEAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.MEDIAN_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.STD_IMAGE_GENERATION_MS, float, field(default=0.0)),
|
|
(defs.PERCENTILES_IMAGE_GENERATION_MS, _PERCENTILE_ROWS_TYPE, field(default=None)),
|
|
]
|
|
|
|
MultiModalsBenchmarkMetrics = make_dataclass(
|
|
"MultiModalsBenchmarkMetrics",
|
|
_MULTIMODAL_BENCHMARK_FIELDS,
|
|
bases=(BenchmarkMetrics,),
|
|
)
|
|
|
|
_STAGE_BENCHMARK_FIELDS = [
|
|
("stage_id", int, field(default=0)),
|
|
("stage_name", str, field(default="")),
|
|
("final_output_type", str, field(default="")),
|
|
(defs.TOTAL_OUTPUT, int, field(default=0)),
|
|
(defs.TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.VLLM_TTFTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.VLLM_TPOTS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.VLLM_ITLS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.AUDIO_TTFPS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.AUDIO_DURATIONS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.AUDIO_FRAMES, _INT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.MISSING_AUDIO_DURATION_COUNT, int, field(default=0)),
|
|
(defs.STAGE_GEN_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.POSTPROCESS_TIMES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
("output_unit_type", str, field(default="")),
|
|
(defs.OUTPUT_UNIT_COUNT, int, field(default=0)),
|
|
(defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.TIME_PER_OUTPUT_UNITS_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
(defs.INTER_OUTPUT_LATENCIES_MS, _FLOAT_LIST_TYPE, field(default_factory=list)),
|
|
]
|
|
|
|
StageBenchmarkMetrics = make_dataclass(
|
|
"StageBenchmarkMetrics",
|
|
_STAGE_BENCHMARK_FIELDS,
|
|
namespace={"__doc__": "Aggregated metrics for one pipeline stage (for printing only)."},
|
|
)
|
|
|
|
|
|
def _percentile_rows_seconds(
|
|
values: list[float], selected_percentiles: list[float], *, to_ms: bool
|
|
) -> tuple[float, float, list[tuple[float, float]]]:
|
|
arr = np.asarray(values, dtype=float)
|
|
scale = 1000.0 if to_ms else 1.0
|
|
mean_v = float(np.mean(arr)) * scale
|
|
median_v = float(np.median(arr)) * scale
|
|
rows = [(float(p), float(np.percentile(arr, p)) * scale) for p in selected_percentiles]
|
|
return mean_v, median_v, rows
|
|
|
|
|
|
def _p_label(p: float) -> str:
|
|
pf = float(p)
|
|
if abs(pf - round(pf)) < 1e-9:
|
|
return str(int(round(pf)))
|
|
return str(pf)
|
|
|
|
|
|
_STREAMING_OUTPUT_UNIT_TYPES = frozenset(
|
|
{
|
|
"text",
|
|
"stream",
|
|
"audio",
|
|
}
|
|
)
|
|
|
|
_AGGREGATE_PERCENTILE_FIELD_NAMES = {
|
|
defs.AUDIO_TTFP: (
|
|
defs.MEAN_AUDIO_TTFP_MS,
|
|
defs.MEDIAN_AUDIO_TTFP_MS,
|
|
defs.PERCENTILES_AUDIO_TTFP_MS,
|
|
),
|
|
defs.AUDIO_RTF: (
|
|
defs.MEAN_AUDIO_RTF,
|
|
defs.MEDIAN_AUDIO_RTF,
|
|
defs.PERCENTILES_AUDIO_RTF,
|
|
),
|
|
defs.AUDIO_DURATION: (
|
|
defs.MEAN_AUDIO_DURATION_S,
|
|
defs.MEDIAN_AUDIO_DURATION_S,
|
|
defs.PERCENTILES_AUDIO_DURATION_S,
|
|
),
|
|
defs.AUDIO_UNDERRUN: (
|
|
defs.MEAN_AUDIO_UNDERRUN_S,
|
|
defs.MEDIAN_AUDIO_UNDERRUN_S,
|
|
defs.PERCENTILES_AUDIO_UNDERRUN_S,
|
|
),
|
|
}
|
|
|
|
|
|
def _wants_text_ttft(metrics: list[str]) -> bool:
|
|
return "ttft" in metrics
|
|
|
|
|
|
def _wants_text_tpot(metrics: list[str]) -> bool:
|
|
return "tpot" in metrics or "tpop" in metrics
|
|
|
|
|
|
def _wants_text_itl(metrics: list[str]) -> bool:
|
|
return "itl" in metrics
|
|
|
|
|
|
def _wants_stream_ttfc(metrics: list[str]) -> bool:
|
|
return "ttfc" in metrics
|
|
|
|
|
|
def _wants_stream_tpoc(metrics: list[str]) -> bool:
|
|
return "tpoc" in metrics or "tpop" in metrics
|
|
|
|
|
|
def _wants_stream_icl(metrics: list[str]) -> bool:
|
|
return "icl" in metrics
|
|
|
|
|
|
def _stage_modality_flags(
|
|
final_output_type: str,
|
|
output_unit_type: str,
|
|
) -> tuple[bool, bool, bool, bool, bool]:
|
|
is_text_stage = final_output_type == "text" or output_unit_type == "text"
|
|
is_audio_stage = final_output_type == "audio" or output_unit_type == "audio"
|
|
is_image_stage = final_output_type in {"image", "images"} or output_unit_type == "image"
|
|
is_video_stage = final_output_type in {"video", "videos"} or output_unit_type == "video"
|
|
is_internal_stream_stage = (
|
|
output_unit_type in _STREAMING_OUTPUT_UNIT_TYPES and not is_text_stage and not is_audio_stage
|
|
)
|
|
return (
|
|
is_text_stage,
|
|
is_audio_stage,
|
|
is_image_stage,
|
|
is_video_stage,
|
|
is_internal_stream_stage,
|
|
)
|
|
|
|
|
|
def print_metrics(
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
max_concurrency,
|
|
request_rate,
|
|
benchmark_duration,
|
|
goodput_config_dict,
|
|
metrics: MultiModalsBenchmarkMetrics,
|
|
*,
|
|
outputs: list[RequestFuncOutput] | None = None,
|
|
selected_percentiles: list[float] | None = None,
|
|
print_stage: bool = False,
|
|
):
|
|
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
|
|
print("{:<40} {:<10}".format("Failed requests:", metrics.failed))
|
|
if max_concurrency is not None:
|
|
print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency))
|
|
if request_rate != float("inf"):
|
|
print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate))
|
|
print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))
|
|
print("{:<40} {:<10.2f}".format("Request throughput (req/s):", metrics.request_throughput))
|
|
if goodput_config_dict:
|
|
print("{:<40} {:<10.2f}".format("Request goodput (req/s):", metrics.request_goodput))
|
|
if isinstance(metrics, MultiModalsBenchmarkMetrics):
|
|
print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests))
|
|
if task_type != TaskType.GENERATION or "e2el" in selected_percentile_metrics:
|
|
process_one_metric("e2el", metrics)
|
|
print_text_metrics(task_type, selected_percentile_metrics, metrics)
|
|
if task_type == TaskType.GENERATION:
|
|
if _has_audio_output(metrics):
|
|
print_audio_metrics(selected_percentile_metrics, metrics)
|
|
if _has_image_output(metrics):
|
|
print_image_metrics(selected_percentiles or [], metrics)
|
|
if print_stage and outputs and selected_percentiles is not None:
|
|
print("\n{s:{c}^{n}}".format(s=" Stage Benchmark Result ", n=50, c="="))
|
|
for sm in _build_stage_metrics_from_outputs(outputs):
|
|
print_stage_metrics(
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
selected_percentiles,
|
|
sm,
|
|
)
|
|
print("=" * 50)
|
|
|
|
|
|
def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
|
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
|
|
if isinstance(metrics, MultiModalsBenchmarkMetrics):
|
|
print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))
|
|
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", metrics.output_throughput))
|
|
print("{:<40} {:<10.2f}".format("Peak output token throughput (tok/s):", metrics.max_output_tokens_per_s))
|
|
print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests))
|
|
print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput))
|
|
|
|
if task_type == TaskType.GENERATION:
|
|
# No text tokens generated (e.g. pure TTS speech endpoint): per-token
|
|
# latency metrics (ttft/tpot/itl) are undefined, so skip them.
|
|
has_text_output = metrics.total_output > 0
|
|
if has_text_output:
|
|
if _wants_text_ttft(selected_percentile_metrics):
|
|
process_one_metric("ttft", metrics)
|
|
if _wants_text_tpot(selected_percentile_metrics):
|
|
process_one_metric("tpot", metrics)
|
|
if _wants_text_itl(selected_percentile_metrics):
|
|
process_one_metric("itl", metrics)
|
|
|
|
|
|
def _has_audio_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
return bool(
|
|
getattr(metrics, defs.TOTAL_AUDIO_DURATION_S, 0.0) > 0 or getattr(metrics, defs.TOTAL_AUDIO_FRAMES, 0) > 0
|
|
)
|
|
|
|
|
|
def _has_image_output(metrics: MultiModalsBenchmarkMetrics) -> bool:
|
|
return int(getattr(metrics, defs.TOTAL_IMAGES, 0) or 0) > 0
|
|
|
|
|
|
def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics):
|
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
|
print(
|
|
"{:<40} {:<10.2f}".format("Total audio duration generated(s):", getattr(metrics, defs.TOTAL_AUDIO_DURATION_S))
|
|
)
|
|
print("{:<40} {:<10}".format("Total audio frames generated:", getattr(metrics, defs.TOTAL_AUDIO_FRAMES)))
|
|
print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", getattr(metrics, defs.AUDIO_THROUGHPUT)))
|
|
print(
|
|
"{:<40} {:<10.2%}".format(
|
|
"Streaming continuity OK rate:",
|
|
getattr(metrics, defs.AUDIO_CONTINUITY_OK_RATE),
|
|
)
|
|
)
|
|
for metric in selected_percentile_metrics:
|
|
if metric.startswith("audio"):
|
|
process_one_metric(metric, metrics)
|
|
|
|
|
|
def print_image_metrics(selected_percentiles: list[float], metrics: MultiModalsBenchmarkMetrics):
|
|
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Total images generated:", getattr(metrics, defs.TOTAL_IMAGES)))
|
|
print("{:<40} {:<10.2f}".format("Image throughput (img/s):", getattr(metrics, defs.IMAGE_THROUGHPUT)))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Average pixels per image:",
|
|
getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
|
)
|
|
)
|
|
if getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS) > 0:
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Mean denoise step latency (ms):",
|
|
getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
|
)
|
|
)
|
|
if getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS) > 0:
|
|
print("-----------------Image Generation-----------------")
|
|
print("{:<40} {:<10.2f}".format("Mean IMAGE_GENERATION (ms):", getattr(metrics, defs.MEAN_IMAGE_GENERATION_MS)))
|
|
print(
|
|
"{:<40} {:<10.2f}".format(
|
|
"Median IMAGE_GENERATION (ms):",
|
|
getattr(metrics, defs.MEDIAN_IMAGE_GENERATION_MS),
|
|
)
|
|
)
|
|
for p, value in getattr(metrics, defs.PERCENTILES_IMAGE_GENERATION_MS) or []:
|
|
if p in selected_percentiles:
|
|
print("{:<40} {:<10.2f}".format(f"P{_p_label(p)} IMAGE_GENERATION (ms):", value))
|
|
|
|
|
|
def process_one_metric(
|
|
metric_attribute_name: str,
|
|
metrics: MultiModalsBenchmarkMetrics,
|
|
):
|
|
metric_header_map = {
|
|
"ttft": "Time to First Token",
|
|
"tpot": "Time per Output Token (excl. 1st token)",
|
|
"itl": "Inter-token Latency",
|
|
"e2el": "End-to-end Latency",
|
|
defs.AUDIO_TTFP: "Time to First Packet",
|
|
defs.AUDIO_RTF: "Real Time Factor",
|
|
defs.AUDIO_DURATION: "Audio Duration",
|
|
defs.AUDIO_UNDERRUN: "Streaming Audio Underrun",
|
|
}
|
|
|
|
header = metric_header_map.get(metric_attribute_name, metric_attribute_name)
|
|
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
|
|
|
is_audio_rtf = metric_attribute_name == defs.AUDIO_RTF
|
|
is_audio_duration_or_underrun = metric_attribute_name in (defs.AUDIO_DURATION, defs.AUDIO_UNDERRUN)
|
|
|
|
suffix = "_ms"
|
|
unit_suffix = " (ms)"
|
|
if is_audio_duration_or_underrun:
|
|
suffix = "_s"
|
|
unit_suffix = " (s)"
|
|
elif is_audio_rtf:
|
|
suffix = ""
|
|
unit_suffix = ""
|
|
field_names = _AGGREGATE_PERCENTILE_FIELD_NAMES.get(metric_attribute_name)
|
|
mean_attr_name = field_names[0] if field_names else f"mean_{metric_attribute_name}{suffix}"
|
|
mean_value = getattr(metrics, mean_attr_name, 0.0)
|
|
print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}")
|
|
|
|
median_attr_name = field_names[1] if field_names else f"median_{metric_attribute_name}{suffix}"
|
|
median_value = getattr(metrics, median_attr_name, 0.0)
|
|
print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}")
|
|
|
|
percentiles_attr_name = field_names[2] if field_names else f"percentiles_{metric_attribute_name}{suffix}"
|
|
percentiles = getattr(metrics, percentiles_attr_name, [])
|
|
|
|
for percentile, value in percentiles:
|
|
p_str = str(int(percentile)) if percentile.is_integer() else str(percentile)
|
|
label = f"P{p_str} {metric_attribute_name.upper()}{unit_suffix}:"
|
|
print(f"{label:<40} {value:<10.2f}")
|
|
|
|
|
|
def _print_percentile_metric(
|
|
header: str,
|
|
label: str,
|
|
values: list[float],
|
|
selected_percentiles: list[float],
|
|
*,
|
|
values_are_ms: bool = False,
|
|
to_ms: bool = True,
|
|
) -> None:
|
|
if not values:
|
|
return
|
|
values_s = [v / 1000.0 for v in values] if values_are_ms else values
|
|
mean_v, med_v, pcts = _percentile_rows_seconds(values_s, selected_percentiles, to_ms=to_ms)
|
|
unit_suffix = " (ms)" if to_ms else ""
|
|
print("{s:{c}^{n}}".format(s=header, n=50, c="-"))
|
|
print(f"{f'Mean {label}{unit_suffix}:':<40} {mean_v:<10.2f}")
|
|
print(f"{f'Median {label}{unit_suffix}:':<40} {med_v:<10.2f}")
|
|
for p, v in pcts:
|
|
p_str = _p_label(p)
|
|
print(f"{f'P{p_str} {label}{unit_suffix}:':<40} {v:<10.2f}")
|
|
|
|
|
|
def _print_stage_timing(sm: StageBenchmarkMetrics, selected_percentiles: list[float]) -> None:
|
|
_print_percentile_metric(
|
|
"Stage Timing",
|
|
"stage_gen_time",
|
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
postprocess_times_ms = getattr(sm, defs.POSTPROCESS_TIMES_MS)
|
|
if postprocess_times_ms and any(v > 0 for v in postprocess_times_ms):
|
|
_print_percentile_metric(
|
|
"Stage Postprocess",
|
|
"postprocess",
|
|
postprocess_times_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def _print_text_stage_metrics(
|
|
task_type: TaskType,
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Stage generated tokens:", getattr(sm, defs.TOTAL_OUTPUT)))
|
|
|
|
if task_type != TaskType.GENERATION:
|
|
return
|
|
if _wants_text_ttft(selected_percentile_metrics):
|
|
_print_percentile_metric(
|
|
"Serving Time to First Token",
|
|
"Serving TTFT",
|
|
getattr(sm, defs.TTFTS),
|
|
selected_percentiles,
|
|
)
|
|
_print_percentile_metric("Time to First Token", "TTFT", getattr(sm, defs.VLLM_TTFTS), selected_percentiles)
|
|
if _wants_text_tpot(selected_percentile_metrics):
|
|
_print_percentile_metric(
|
|
"Time per Output Token (excl. 1st token)",
|
|
"TPOT",
|
|
getattr(sm, defs.VLLM_TPOTS),
|
|
selected_percentiles,
|
|
)
|
|
if _wants_text_itl(selected_percentile_metrics):
|
|
_print_percentile_metric("Inter-token Latency", "ITL", getattr(sm, defs.VLLM_ITLS), selected_percentiles)
|
|
|
|
|
|
def _print_audio_stage_metrics(
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
total_duration = float(sum(getattr(sm, defs.AUDIO_DURATIONS)))
|
|
total_frames = int(sum(getattr(sm, defs.AUDIO_FRAMES)))
|
|
print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="="))
|
|
print("{:<40} {:<10.2f}".format("Stage audio duration generated(s):", total_duration))
|
|
print("{:<40} {:<10}".format("Stage audio frames generated:", total_frames))
|
|
if defs.AUDIO_TTFP in selected_percentile_metrics:
|
|
_print_percentile_metric(
|
|
"Serving Time to First Packet",
|
|
"Serving AUDIO_TTFP",
|
|
getattr(sm, defs.AUDIO_TTFPS),
|
|
selected_percentiles,
|
|
)
|
|
if defs.AUDIO_DURATION in selected_percentile_metrics:
|
|
audio_durations = getattr(sm, defs.AUDIO_DURATIONS)
|
|
if audio_durations:
|
|
_print_percentile_metric(
|
|
"Audio Duration",
|
|
"AUDIO_DURATION",
|
|
audio_durations,
|
|
selected_percentiles,
|
|
to_ms=False,
|
|
)
|
|
elif getattr(sm, defs.MISSING_AUDIO_DURATION_COUNT) > 0:
|
|
print("{:<40} {:<10}".format("AUDIO_DURATION skipped:", "missing stage-local audio duration/sample rate"))
|
|
|
|
|
|
def _print_image_stage_metrics(
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
image_num = getattr(sm, defs.OUTPUT_UNIT_COUNT) if getattr(sm, "output_unit_type") == "image" else 0
|
|
print("{s:{c}^{n}}".format(s=" Image Result ", n=50, c="="))
|
|
print("{:<40} {:<10}".format("Total images generated:", image_num))
|
|
_print_percentile_metric(
|
|
"Image Generation",
|
|
"IMAGE_GENERATION",
|
|
getattr(sm, defs.STAGE_GEN_TIMES_MS),
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def _print_internal_stream_stage_metrics(
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
) -> None:
|
|
print("{s:{c}^{n}}".format(s=" Internal Stream Result ", n=50, c="="))
|
|
serving_time_to_first_outputs_ms = getattr(sm, defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS)
|
|
if _wants_stream_ttfc(selected_percentile_metrics) and serving_time_to_first_outputs_ms:
|
|
_print_percentile_metric(
|
|
"Serving Time to First Chunk",
|
|
"Serving TTFC",
|
|
serving_time_to_first_outputs_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
time_per_output_units_ms = getattr(sm, defs.TIME_PER_OUTPUT_UNITS_MS)
|
|
if _wants_stream_tpoc(selected_percentile_metrics) and time_per_output_units_ms:
|
|
_print_percentile_metric(
|
|
"Time per Output Chunk (excl. 1st chunk)",
|
|
"TPOP",
|
|
time_per_output_units_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
inter_output_latencies_ms = getattr(sm, defs.INTER_OUTPUT_LATENCIES_MS)
|
|
if _wants_stream_icl(selected_percentile_metrics) and inter_output_latencies_ms:
|
|
_print_percentile_metric(
|
|
"Inter-chunk Latency",
|
|
"ICL",
|
|
inter_output_latencies_ms,
|
|
selected_percentiles,
|
|
values_are_ms=True,
|
|
to_ms=True,
|
|
)
|
|
|
|
|
|
def print_stage_metrics(
|
|
task_type: TaskType,
|
|
selected_percentile_metrics: list[str],
|
|
selected_percentiles: list[float],
|
|
sm: StageBenchmarkMetrics,
|
|
):
|
|
title = f" Stage {getattr(sm, 'stage_id')} ({getattr(sm, 'stage_name')}) "
|
|
(
|
|
is_text_stage,
|
|
is_audio_stage,
|
|
is_image_stage,
|
|
is_video_stage,
|
|
is_internal_stream_stage,
|
|
) = _stage_modality_flags(getattr(sm, "final_output_type"), getattr(sm, "output_unit_type"))
|
|
|
|
print("{s:{c}^{n}}".format(s=title, n=50, c="="))
|
|
if is_image_stage:
|
|
_print_image_stage_metrics(selected_percentiles, sm)
|
|
return
|
|
|
|
if is_video_stage:
|
|
return
|
|
|
|
_print_stage_timing(sm, selected_percentiles)
|
|
if is_text_stage:
|
|
_print_text_stage_metrics(task_type, selected_percentile_metrics, selected_percentiles, sm)
|
|
elif is_audio_stage:
|
|
_print_audio_stage_metrics(
|
|
selected_percentile_metrics,
|
|
selected_percentiles,
|
|
sm,
|
|
)
|
|
elif is_internal_stream_stage:
|
|
_print_internal_stream_stage_metrics(selected_percentile_metrics, selected_percentiles, sm)
|
|
|
|
|
|
def _build_stage_metrics_from_outputs(
|
|
outputs: list[RequestFuncOutput],
|
|
) -> list[StageBenchmarkMetrics]:
|
|
"""Aggregate per ``stage_id`` using ``stage_metrics`` snapshots from the client."""
|
|
buckets: dict[str, list[tuple[RequestFuncOutput, dict]]] = defaultdict(list)
|
|
for out in outputs:
|
|
if not getattr(out, "success", False):
|
|
continue
|
|
smap = getattr(out, "stage_metrics", None) or {}
|
|
if not isinstance(smap, dict) or not smap:
|
|
continue
|
|
for sid, info in smap.items():
|
|
if isinstance(info, dict):
|
|
buckets[str(sid)].append((out, info))
|
|
|
|
result: list[StageBenchmarkMetrics] = []
|
|
for sid in sorted(buckets.keys(), key=lambda x: int(x)):
|
|
rows = buckets[sid]
|
|
stage_id_int = int(sid)
|
|
stage_name = str((rows[0][1] or {}).get("stage_name") or f"stage_{stage_id_int}")
|
|
final_output_type = str((rows[0][1] or {}).get("final_output_type") or "unknown")
|
|
output_unit_types = [
|
|
str((info or {}).get("output_unit_type") or "") for _, info in rows if (info or {}).get("output_unit_type")
|
|
]
|
|
output_unit_type = output_unit_types[0] if output_unit_types else "other"
|
|
|
|
is_text_stage, is_audio_stage, _, _, is_internal_stream_stage = _stage_modality_flags(
|
|
final_output_type, output_unit_type
|
|
)
|
|
total_output = 0
|
|
|
|
ttfts: list[float] = []
|
|
tpots: list[float] = []
|
|
itls: list[float] = []
|
|
vllm_ttfts: list[float] = []
|
|
vllm_tpots: list[float] = []
|
|
vllm_itls: list[float] = []
|
|
stage_gen_times_ms = [float((info or {}).get(defs.STAGE_GEN_TIME_MS) or 0.0) for _, info in rows]
|
|
postprocess_times_ms = [float((info or {}).get(defs.POSTPROCESS_TIME_MS) or 0.0) for _, info in rows]
|
|
output_unit_count = sum(int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) for _, info in rows)
|
|
inter_output_latencies_ms: list[float] = []
|
|
serving_time_to_first_outputs_ms: list[float] = []
|
|
time_per_output_units_ms: list[float] = []
|
|
if is_text_stage or is_internal_stream_stage:
|
|
serving_time_to_first_outputs_ms = [
|
|
float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
for _, info in rows
|
|
if (info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) is not None
|
|
]
|
|
time_per_output_units_ms = [
|
|
float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
|
for _, info in rows
|
|
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1
|
|
and (info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) is not None
|
|
]
|
|
for _, info in rows:
|
|
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
|
if isinstance(values, list):
|
|
inter_output_latencies_ms.extend(float(v or 0.0) for v in values)
|
|
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
|
inter_output_latencies_ms.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0))
|
|
|
|
if is_text_stage:
|
|
for _, info in rows:
|
|
ttft_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
if ttft_ms > 0:
|
|
ttfts.append(ttft_ms / 1000.0)
|
|
tpot_ms = float((info or {}).get(defs.TIME_PER_OUTPUT_UNIT_MS) or 0.0)
|
|
if int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0) > 1 and tpot_ms > 0:
|
|
tpots.append(tpot_ms / 1000.0)
|
|
values = (info or {}).get(defs.INTER_OUTPUT_LATENCIES_MS)
|
|
if isinstance(values, list):
|
|
itls.extend(float(v or 0.0) / 1000.0 for v in values)
|
|
elif (info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) is not None:
|
|
itls.append(float((info or {}).get(defs.INTER_OUTPUT_LATENCY_MS) or 0.0) / 1000.0)
|
|
vllm_ttft_ms = float((info or {}).get(defs.VLLM_TTFT_MS) or 0.0)
|
|
if vllm_ttft_ms > 0:
|
|
vllm_ttfts.append(vllm_ttft_ms / 1000.0)
|
|
vllm_tpot_ms = float((info or {}).get(defs.VLLM_TPOT_MS) or 0.0)
|
|
if vllm_tpot_ms > 0:
|
|
vllm_tpots.append(vllm_tpot_ms / 1000.0)
|
|
vllm_values = (info or {}).get(defs.VLLM_ITLS_MS)
|
|
if isinstance(vllm_values, list):
|
|
vllm_itls.extend(float(v or 0.0) / 1000.0 for v in vllm_values)
|
|
elif (info or {}).get(defs.VLLM_ITL_MS) is not None:
|
|
vllm_itls.append(float((info or {}).get(defs.VLLM_ITL_MS) or 0.0) / 1000.0)
|
|
total_output = sum(int((info or {}).get(defs.NUM_TOKENS_OUT) or 0) for _, info in rows)
|
|
|
|
audio_ttfps: list[float] = []
|
|
audio_durations: list[float] = []
|
|
audio_frames: list[int] = []
|
|
missing_audio_duration_count = 0
|
|
if is_audio_stage:
|
|
for _, info in rows:
|
|
audio_ttfp_ms = float((info or {}).get(defs.SERVING_TIME_TO_FIRST_OUTPUT_MS) or 0.0)
|
|
if audio_ttfp_ms > 0:
|
|
audio_ttfps.append(audio_ttfp_ms / 1000.0)
|
|
|
|
frame_count = int((info or {}).get(defs.AUDIO_FRAMES) or 0)
|
|
if frame_count <= 0 and (info or {}).get("output_unit_type") == "audio":
|
|
frame_count = int((info or {}).get(defs.OUTPUT_UNIT_COUNT) or 0)
|
|
audio_frames.append(frame_count)
|
|
|
|
duration_s = float((info or {}).get(f"{defs.AUDIO_DURATION}_s") or 0.0)
|
|
if duration_s > 0:
|
|
audio_durations.append(duration_s)
|
|
else:
|
|
missing_audio_duration_count += 1
|
|
|
|
result.append(
|
|
StageBenchmarkMetrics(
|
|
**{
|
|
"stage_id": stage_id_int,
|
|
"stage_name": stage_name,
|
|
"final_output_type": final_output_type,
|
|
defs.TOTAL_OUTPUT: total_output,
|
|
defs.TTFTS: ttfts,
|
|
defs.TPOTS: tpots,
|
|
defs.ITLS: itls,
|
|
defs.VLLM_TTFTS: vllm_ttfts,
|
|
defs.VLLM_TPOTS: vllm_tpots,
|
|
defs.VLLM_ITLS: vllm_itls,
|
|
defs.AUDIO_TTFPS: audio_ttfps,
|
|
defs.AUDIO_DURATIONS: audio_durations,
|
|
defs.AUDIO_FRAMES: audio_frames,
|
|
defs.MISSING_AUDIO_DURATION_COUNT: missing_audio_duration_count,
|
|
defs.STAGE_GEN_TIMES_MS: stage_gen_times_ms,
|
|
defs.POSTPROCESS_TIMES_MS: postprocess_times_ms,
|
|
"output_unit_type": output_unit_type,
|
|
defs.OUTPUT_UNIT_COUNT: output_unit_count,
|
|
defs.SERVING_TIME_TO_FIRST_OUTPUTS_MS: serving_time_to_first_outputs_ms,
|
|
defs.TIME_PER_OUTPUT_UNITS_MS: time_per_output_units_ms,
|
|
defs.INTER_OUTPUT_LATENCIES_MS: inter_output_latencies_ms,
|
|
}
|
|
)
|
|
)
|
|
return result
|
|
|
|
|
|
def calculate_metrics(
|
|
input_requests: list[SampleRequest],
|
|
outputs: list[RequestFuncOutput],
|
|
dur_s: float,
|
|
tokenizer: TokenizerLike | None,
|
|
selected_percentiles: list[float],
|
|
goodput_config_dict: dict[str, float],
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
max_concurrency,
|
|
request_rate,
|
|
benchmark_duration,
|
|
print_stage: bool = False,
|
|
) -> tuple[BenchmarkMetrics, list[int]]:
|
|
"""Calculate the metrics for the benchmark.
|
|
|
|
Args:
|
|
input_requests: The input requests.
|
|
outputs: The outputs of the requests.
|
|
dur_s: The duration of the benchmark.
|
|
tokenizer: The tokenizer to use.
|
|
selected_percentiles: The percentiles to select.
|
|
goodput_config_dict: The goodput configuration.
|
|
|
|
Returns:
|
|
A tuple of the benchmark metrics and the actual output lengths.
|
|
"""
|
|
actual_output_lens: list[int] = []
|
|
total_input = 0
|
|
completed = 0
|
|
good_completed = 0
|
|
itls: list[float] = []
|
|
tpots: list[float] = []
|
|
all_tpots: list[float] = []
|
|
ttfts: list[float] = []
|
|
e2els: list[float] = []
|
|
audio_ttfps: list[float] = []
|
|
audio_rtfs: list[float] = []
|
|
audio_duration: list[float] = []
|
|
audio_frames: list[int] = []
|
|
image_generation_times_ms: list[float] = []
|
|
denoise_step_latencies_ms: list[float] = []
|
|
total_images = 0
|
|
total_image_pixels = 0
|
|
audio_underruns: list[float] = []
|
|
audio_continuity_ok: list[bool] = []
|
|
input_audio_duration = 0.0
|
|
for i in range(len(outputs)):
|
|
if outputs[i].success:
|
|
output_len = outputs[i].output_tokens
|
|
|
|
if not output_len:
|
|
if tokenizer is None:
|
|
output_len = 1
|
|
else:
|
|
# We use the tokenizer to count the number of output tokens
|
|
# for some serving backends instead of looking at
|
|
# len(outputs[i].itl) since multiple output tokens may be
|
|
# bundled together
|
|
# Note : this may inflate the output token count slightly
|
|
output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids)
|
|
actual_output_lens.append(output_len)
|
|
total_input += outputs[i].prompt_len
|
|
tpot = 0
|
|
if output_len > 1:
|
|
if outputs[i].itl:
|
|
# Use mean(ITL) directly so per-request TPOT == mean(ITL).
|
|
# The ITL list records one entry per SSE chunk; server may
|
|
# bundle multiple tokens per chunk, so len(itl)+1 != output_len.
|
|
# Using mean(itl) keeps TPOT and ITL on the same footing.
|
|
tpot = sum(outputs[i].itl) / len(outputs[i].itl)
|
|
else:
|
|
try:
|
|
latency_minus_ttft = outputs[i].text_latency - outputs[i].ttft
|
|
except Exception:
|
|
latency_minus_ttft = outputs[i].latency - outputs[i].ttft
|
|
tpot = latency_minus_ttft / (output_len - 1)
|
|
tpots.append(tpot)
|
|
# Note: if output_len <= 1, we regard tpot as 0 for goodput
|
|
all_tpots.append(tpot)
|
|
itls += outputs[i].itl
|
|
ttfts.append(outputs[i].ttft)
|
|
audio_ttfps.append(getattr(outputs[i], defs.AUDIO_TTFP, 0.0))
|
|
audio_rtfs.append(getattr(outputs[i], defs.AUDIO_RTF, 0.0))
|
|
audio_duration.append(getattr(outputs[i], defs.AUDIO_DURATION, 0.0))
|
|
audio_frames.append(getattr(outputs[i], defs.AUDIO_FRAMES, 0.0))
|
|
image_count = int(getattr(outputs[i], defs.IMAGE_COUNT, 0) or 0)
|
|
total_images += image_count
|
|
image_generation_time_ms = float(getattr(outputs[i], defs.IMAGE_GENERATION_TIME_MS, 0.0) or 0.0)
|
|
if image_generation_time_ms > 0:
|
|
image_generation_times_ms.append(image_generation_time_ms)
|
|
total_image_pixels += int(getattr(outputs[i], defs.IMAGE_PIXELS, 0) or 0)
|
|
denoise_step_latency_ms = float(getattr(outputs[i], defs.DENOISE_STEP_LATENCY_MS, 0.0) or 0.0)
|
|
if denoise_step_latency_ms > 0:
|
|
denoise_step_latencies_ms.append(denoise_step_latency_ms)
|
|
audio_underruns.append(getattr(outputs[i], f"{defs.AUDIO_UNDERRUN}_s", 0.0))
|
|
audio_continuity_ok.append(bool(getattr(outputs[i], defs.AUDIO_CONTINUITY_OK, True)))
|
|
e2els.append(outputs[i].latency)
|
|
input_audio_duration += outputs[i].input_audio_duration
|
|
completed += 1
|
|
else:
|
|
actual_output_lens.append(0)
|
|
|
|
if goodput_config_dict:
|
|
valid_metrics = []
|
|
slo_values = []
|
|
|
|
if "ttft" in goodput_config_dict:
|
|
valid_metrics.append(ttfts)
|
|
slo_values.append(goodput_config_dict["ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
if "audio_ttft" in goodput_config_dict:
|
|
valid_metrics.append(audio_ttfps)
|
|
slo_values.append(goodput_config_dict["audio_ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
if "tpot" in goodput_config_dict:
|
|
valid_metrics.append(all_tpots)
|
|
slo_values.append(goodput_config_dict["tpot"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
if "e2el" in goodput_config_dict:
|
|
valid_metrics.append(e2els)
|
|
slo_values.append(goodput_config_dict["e2el"] / MILLISECONDS_TO_SECONDS_CONVERSION)
|
|
|
|
for req_metric in zip(*valid_metrics):
|
|
is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])
|
|
if is_good_req:
|
|
good_completed += 1
|
|
|
|
if completed == 0:
|
|
warnings.formatwarning = lambda msg, category, filename, lineno, line=None: (
|
|
f"{filename}:{lineno}: {category.__name__}: {msg}\n"
|
|
)
|
|
warnings.warn(
|
|
"All requests failed. This is likely due to a misconfiguration on the benchmark arguments.",
|
|
stacklevel=2,
|
|
)
|
|
|
|
# Calculate max output tokens per second metric
|
|
max_output_tokens_per_s = 0.0
|
|
max_concurrent_requests = 0
|
|
|
|
# Find the time range across all successful requests
|
|
successful_outputs = [output for output in outputs if output.success]
|
|
failed_outputs = [output for output in outputs if not output.success]
|
|
if successful_outputs:
|
|
min_start_time = min(output.start_time for output in successful_outputs)
|
|
max_end_time = max(output.start_time + output.latency for output in successful_outputs)
|
|
|
|
# Create second buckets (ceiling to ensure we capture all time)
|
|
duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1
|
|
tokens_per_second = np.zeros(duration_seconds)
|
|
concurrent_requests_per_second = np.zeros(duration_seconds)
|
|
|
|
for i, output in enumerate(successful_outputs):
|
|
# Calculate token generation timestamp using
|
|
# start_time, ttft, and itl
|
|
token_times = [output.start_time + output.ttft]
|
|
current_time = token_times[0]
|
|
for itl_value in output.itl:
|
|
current_time += itl_value
|
|
token_times.append(current_time)
|
|
|
|
# Add tokens to second buckets
|
|
for token_time in token_times:
|
|
second_bucket = int(token_time - min_start_time)
|
|
if 0 <= second_bucket < duration_seconds:
|
|
tokens_per_second[second_bucket] += 1
|
|
|
|
# Track concurrent requests for each second this request was active
|
|
request_start_second = int(output.start_time - min_start_time)
|
|
request_end_second = int((output.start_time + output.latency) - min_start_time)
|
|
|
|
for second in range(request_start_second, request_end_second + 1):
|
|
concurrent_requests_per_second[second] += 1
|
|
|
|
# Find the maximum tokens per second and corresponding
|
|
# concurrent requests
|
|
if len(tokens_per_second) > 0:
|
|
max_output_tokens_per_s = float(np.max(tokens_per_second))
|
|
max_concurrent_requests = int(np.max(concurrent_requests_per_second))
|
|
|
|
if TERM_PLOTLIB_AVAILABLE:
|
|
import termplotlib as tpl
|
|
|
|
fig = tpl.figure()
|
|
fig.plot(
|
|
np.arange(len(tokens_per_second)),
|
|
tokens_per_second,
|
|
title="Output tokens per second",
|
|
)
|
|
fig.plot(
|
|
np.arange(len(concurrent_requests_per_second)),
|
|
concurrent_requests_per_second,
|
|
title="Concurrent requests per second",
|
|
)
|
|
fig.show()
|
|
else:
|
|
print("tip: install termplotlib and gnuplot to plot the metrics")
|
|
|
|
metrics = MultiModalsBenchmarkMetrics(
|
|
completed=completed,
|
|
failed=len(failed_outputs),
|
|
total_input=total_input,
|
|
total_output=sum(actual_output_lens),
|
|
request_throughput=completed / dur_s,
|
|
request_goodput=good_completed / dur_s,
|
|
output_throughput=sum(actual_output_lens) / dur_s,
|
|
total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
|
|
mean_ttft_ms=np.mean(ttfts or 0) * 1000, # ttfts is empty if streaming is not supported by the endpoint
|
|
std_ttft_ms=np.std(ttfts or 0) * 1000,
|
|
median_ttft_ms=np.median(ttfts or 0) * 1000,
|
|
percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles],
|
|
mean_tpot_ms=np.mean(tpots or 0) * 1000,
|
|
std_tpot_ms=np.std(tpots or 0) * 1000,
|
|
median_tpot_ms=np.median(tpots or 0) * 1000,
|
|
percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000) for p in selected_percentiles],
|
|
mean_itl_ms=np.mean(itls or 0) * 1000,
|
|
std_itl_ms=np.std(itls or 0) * 1000,
|
|
median_itl_ms=np.median(itls or 0) * 1000,
|
|
percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000) for p in selected_percentiles],
|
|
mean_e2el_ms=np.mean(e2els or 0) * 1000,
|
|
std_e2el_ms=np.std(e2els or 0) * 1000,
|
|
median_e2el_ms=np.median(e2els or 0) * 1000,
|
|
percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles],
|
|
max_output_tokens_per_s=max_output_tokens_per_s,
|
|
max_concurrent_requests=max_concurrent_requests,
|
|
rtfx=input_audio_duration / dur_s,
|
|
**{
|
|
defs.MEAN_AUDIO_TTFP_MS: np.mean(audio_ttfps or 0) * 1000,
|
|
defs.STD_AUDIO_TTFP_MS: np.std(audio_ttfps or 0) * 1000,
|
|
defs.MEDIAN_AUDIO_TTFP_MS: np.median(audio_ttfps or 0) * 1000,
|
|
defs.PERCENTILES_AUDIO_TTFP_MS: [
|
|
(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles
|
|
],
|
|
defs.MEAN_AUDIO_DURATION_S: np.mean(audio_duration or 0),
|
|
defs.STD_AUDIO_DURATION_S: np.std(audio_duration or 0),
|
|
defs.MEDIAN_AUDIO_DURATION_S: np.median(audio_duration or 0),
|
|
defs.PERCENTILES_AUDIO_DURATION_S: [
|
|
(p, np.percentile(audio_duration or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.TOTAL_AUDIO_DURATION_S: sum(audio_duration),
|
|
defs.TOTAL_AUDIO_FRAMES: sum(audio_frames),
|
|
defs.AUDIO_THROUGHPUT: sum(audio_duration) / dur_s,
|
|
defs.MEAN_AUDIO_RTF: np.mean(audio_rtfs or 0),
|
|
defs.STD_AUDIO_RTF: np.std(audio_rtfs or 0),
|
|
defs.MEDIAN_AUDIO_RTF: np.median(audio_rtfs or 0),
|
|
defs.PERCENTILES_AUDIO_RTF: [(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles],
|
|
defs.TOTAL_IMAGES: total_images,
|
|
defs.IMAGE_THROUGHPUT: total_images / dur_s,
|
|
defs.AVERAGE_PIXELS_PER_IMAGE: (total_image_pixels / total_images) if total_images > 0 else 0.0,
|
|
defs.MEAN_DENOISE_STEP_LATENCY_MS: np.mean(denoise_step_latencies_ms or 0),
|
|
defs.MEAN_IMAGE_GENERATION_MS: np.mean(image_generation_times_ms or 0),
|
|
defs.STD_IMAGE_GENERATION_MS: np.std(image_generation_times_ms or 0),
|
|
defs.MEDIAN_IMAGE_GENERATION_MS: np.median(image_generation_times_ms or 0),
|
|
defs.PERCENTILES_IMAGE_GENERATION_MS: [
|
|
(p, np.percentile(image_generation_times_ms or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.MEAN_AUDIO_UNDERRUN_S: np.mean(audio_underruns or 0),
|
|
defs.STD_AUDIO_UNDERRUN_S: np.std(audio_underruns or 0),
|
|
defs.MEDIAN_AUDIO_UNDERRUN_S: np.median(audio_underruns or 0),
|
|
defs.PERCENTILES_AUDIO_UNDERRUN_S: [
|
|
(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles
|
|
],
|
|
defs.AUDIO_CONTINUITY_OK_RATE: (
|
|
(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0
|
|
),
|
|
},
|
|
)
|
|
print_metrics(
|
|
task_type,
|
|
selected_percentile_metrics,
|
|
max_concurrency,
|
|
request_rate,
|
|
benchmark_duration,
|
|
goodput_config_dict,
|
|
metrics,
|
|
outputs=outputs,
|
|
selected_percentiles=selected_percentiles,
|
|
print_stage=print_stage,
|
|
)
|
|
return metrics, actual_output_lens
|