68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
"""
|
|
Unit tests for metrics.py
|
|
"""
|
|
|
|
import pytest
|
|
from vllm.benchmarks.serve import TaskType
|
|
|
|
from vllm_omni.benchmarks.metrics.metrics import calculate_metrics
|
|
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput
|
|
|
|
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
|
|
|
|
|
def _make_output(prompt_len: int, output_tokens: int = 10) -> MixRequestFuncOutput:
|
|
"""Build a minimal successful MixRequestFuncOutput for metrics aggregation."""
|
|
output = MixRequestFuncOutput()
|
|
output.success = True
|
|
output.prompt_len = prompt_len
|
|
output.output_tokens = output_tokens
|
|
output.generated_text = "x" * output_tokens
|
|
output.ttft = 0.1
|
|
output.text_latency = 1.0
|
|
output.latency = 1.0
|
|
output.start_time = 0.0
|
|
output.itl = [0.1] * max(output_tokens - 1, 0)
|
|
output.audio_ttfp = 0.0
|
|
output.audio_rtf = 0.0
|
|
output.audio_duration = 0.0
|
|
output.audio_frames = 0
|
|
output.input_audio_duration = 0.0
|
|
output.error = ""
|
|
return output
|
|
|
|
|
|
# ============================================================================
|
|
# total_input Tests
|
|
# ============================================================================
|
|
|
|
|
|
def test_total_input_aggregated_from_output_prompt_len():
|
|
"""Test that total_input sums outputs[i].prompt_len, not input_requests[i].prompt_len."""
|
|
outputs = [_make_output(4992), _make_output(3000)]
|
|
|
|
metrics, _ = calculate_metrics(
|
|
input_requests=[],
|
|
outputs=outputs,
|
|
dur_s=10.0,
|
|
tokenizer=None,
|
|
selected_percentiles=[99.0],
|
|
goodput_config_dict={},
|
|
task_type=TaskType.GENERATION,
|
|
selected_percentile_metrics=[],
|
|
max_concurrency=None,
|
|
request_rate=float("inf"),
|
|
benchmark_duration=10.0,
|
|
)
|
|
|
|
assert metrics.total_input == 7992, (
|
|
"total_input should aggregate from outputs[i].prompt_len to reflect the true multimodal input token count"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v", "-s"])
|