From 1053f7b0bdf0a1ac18c09c536d08eefc32bc2995 Mon Sep 17 00:00:00 2001
From: psv666 <150513104+psv666@users.noreply.github.com>
Date: Wed, 16 Sep 2026 23:39:01 +0800
Subject: [PATCH] [Bugfix] Preserve benchmark metric sample counts in results
 (#7624)

Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
---
 .../benchmarks/patch/test_benchmark_result.py | 74 +++++++++++++++++++
 tests/dfx/perf/tests/test_runner_metadata.py  | 16 ++--
 vllm_omni/benchmarks/patch/patch.py           |  8 ++
 3 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 tests/benchmarks/patch/test_benchmark_result.py

diff --git a/tests/benchmarks/patch/test_benchmark_result.py b/tests/benchmarks/patch/test_benchmark_result.py
new file mode 100644
index 000000000..1e9619fe3
--- /dev/null
+++ b/tests/benchmarks/patch/test_benchmark_result.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Verify metric sample counts survive benchmark JSON serialization."""
+
+import asyncio
+import json
+import time
+
+import pytest
+from vllm.benchmarks.datasets import SampleRequest
+from vllm.benchmarks.serve import TaskType
+
+from vllm_omni.benchmarks.patch import patch
+
+pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
+
+
+@pytest.mark.parametrize("selected_metrics", [["ttft", "tpot", "itl"], []])
+@pytest.mark.parametrize("measured", [True, False])
+def test_benchmark_result_preserves_tpot_sample_count(monkeypatch, measured, selected_metrics):
+    async def request_func(**kwargs):
+        return patch.MixRequestFuncOutput(
+            success=True,
+            prompt_len=2,
+            output_tokens=3,
+            generated_text="hello world!",
+            start_time=time.perf_counter(),
+            ttft=0.01,
+            latency=0.03,
+            text_latency=0.03,
+            itl=[0.01, 0.01] if measured else [0.0, 0.0],
+        )
+
+    monkeypatch.setitem(patch.ASYNC_REQUEST_FUNCS, "test-result", request_func)
+    result = asyncio.run(
+        patch.benchmark(
+            task_type=TaskType.GENERATION,
+            endpoint_type="test-result",
+            api_url="http://unused/v1/chat/completions",
+            base_url="http://unused",
+            model_id="test-model",
+            model_name="test-model",
+            tokenizer=None,
+            input_requests=[SampleRequest(prompt="hi", prompt_len=2, expected_output_len=3)],
+            logprobs=None,
+            request_rate=float("inf"),
+            burstiness=1.0,
+            disable_tqdm=True,
+            num_warmups=0,
+            profile=False,
+            selected_percentile_metrics=selected_metrics,
+            selected_percentiles=[50.0],
+            ignore_eos=False,
+            goodput_config_dict={},
+            max_concurrency=1,
+            lora_modules=None,
+            extra_headers=None,
+            extra_body=None,
+            ready_check_timeout_sec=0,
+        )
+    )
+    # DFX reads this dictionary from the benchmark's JSON result file.
+    saved_result = json.loads(json.dumps(result))
+    assert saved_result["completed"] == 1
+    assert saved_result.get("num_tpot_samples") == int(measured)
+    assert saved_result["num_ttft_samples"] == 1
+    assert saved_result["num_itl_samples"] == 2
+    assert "num_audio_ttfp_samples" in saved_result
+    assert "num_audio_rtf_samples" in saved_result
+    if measured and "tpot" in selected_metrics:
+        assert saved_result["mean_tpot_ms"] == pytest.approx(10.0)
+    else:
+        assert "mean_tpot_ms" not in saved_result
diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py
index df99f090d..a0f966121 100644
--- a/tests/dfx/perf/tests/test_runner_metadata.py
+++ b/tests/dfx/perf/tests/test_runner_metadata.py
@@ -570,6 +570,7 @@ def test_omni_tpot_baseline_accepts_measured_finite_sample():
 @pytest.mark.parametrize(
     ("num_tpot_samples", "mean_tpot_ms", "match"),
     [
+        (None, 10.0, "no measurable TPOT samples"),
         (0, float("nan"), "no measurable TPOT samples"),
         (1, float("nan"), "mean_tpot_ms is not finite"),
     ],
@@ -577,14 +578,17 @@ def test_omni_tpot_baseline_accepts_measured_finite_sample():
 def test_omni_tpot_baseline_rejects_missing_or_nonfinite_sample(num_tpot_samples, mean_tpot_ms, match):
     from tests.dfx.perf.scripts.run_benchmark import assert_result
 
+    result = {
+        "completed": 1,
+        "Hardware": "H100",
+        "mean_tpot_ms": mean_tpot_ms,
+    }
+    if num_tpot_samples is not None:
+        result["num_tpot_samples"] = num_tpot_samples
+
     with pytest.raises(AssertionError, match=match):
         assert_result(
-            {
-                "completed": 1,
-                "Hardware": "H100",
-                "num_tpot_samples": num_tpot_samples,
-                "mean_tpot_ms": mean_tpot_ms,
-            },
+            result,
             {"baseline": {"H100": {"mean_tpot_ms": 20.0}}},
             1,
         )
diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py
index bee910c90..5a5e74b0d 100644
--- a/vllm_omni/benchmarks/patch/patch.py
+++ b/vllm_omni/benchmarks/patch/patch.py
@@ -3078,6 +3078,14 @@ async def benchmark(
             "max_concurrent_requests": metrics.max_concurrent_requests,
             "rtfx": metrics.rtfx,
         }
+        for sample_count in (
+            "num_ttft_samples",
+            "num_tpot_samples",
+            "num_itl_samples",
+            "num_audio_ttfp_samples",
+            "num_audio_rtf_samples",
+        ):
+            result[sample_count] = getattr(metrics, sample_count)
     else:
         result = {
             "duration": benchmark_duration,