[CI/Build] Resolve hardware-nested perf baselines per concurrency sweep (#5845)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/dfx/perf/scripts/diffusion_result_template.json

@@ -21,10 +21,12 @@
       "num-input-images": 0,
       "enable-negative-prompt": false,
       "baseline": {
-        "throughput_qps": 0,
-        "latency_mean": 0,
-        "peak_memory_mb_max": 0,
-        "peak_memory_mb_mean": 0
+        "H100": {
+          "throughput_qps": 0,
+          "latency_mean": 0,
+          "peak_memory_mb_max": 0,
+          "peak_memory_mb_mean": 0
+        }
       }
     },
     "result": {

tests/dfx/perf/scripts/result_omni_template.json

@@ -46,9 +46,11 @@
   "median_audio_duration_s": 0,
   "p99_audio_duration_s": 0,
   "baseline": {
-    "mean_ttft_ms": 0,
-    "mean_audio_ttfp_ms": 0,
-    "mean_audio_rtf": 0
+    "H100": {
+      "mean_ttft_ms": 0,
+      "mean_audio_ttfp_ms": 0,
+      "mean_audio_rtf": 0
+    }
   },
   "random_input_len": 0,
   "random_output_len": 0

tests/dfx/perf/scripts/run_benchmark.py

@@ -237,7 +237,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
         break
 
     # QPS / request-rate sweep
-    for qps, num_prompt in zip(qps_list, num_prompt_list):
+    for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
         args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
         result = run_benchmark(
             args=args,
@@ -246,6 +246,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
             dataset_name=dataset_name,
             num_prompt=num_prompt,
             baseline_config=params.get("baseline"),
+            sweep_index=sweep_index,
             random_input_len=params.get("random_input_len"),
             random_output_len=params.get("random_output_len"),
             resource_label=resource_label,
@@ -253,7 +254,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(result, num_prompt)
 
     # concurrency test
-    for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
+    for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
         args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
         result = run_benchmark(
             args=args,
@@ -262,6 +263,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
             dataset_name=dataset_name,
             num_prompt=num_prompt,
             baseline_config=params.get("baseline"),
+            sweep_index=sweep_index,
             random_input_len=params.get("random_input_len"),
             random_output_len=params.get("random_output_len"),
             resource_label=resource_label,

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -23,7 +23,6 @@ of all runs from cases in that file). Bulk load without ``--test-config-file`` u
 the same per-file aggregation; ``-m`` only selects which cases run.
 """
 
-import copy
 import json
 import os
 import socket
@@ -45,6 +44,7 @@ from tests.dfx.conftest import (
     get_runtime_resource_label,
     hardware_json_value,
     is_diffusion_perf_config,
+    resolve_baseline_for_sweep,
     resolve_pytest_marks,
     resource_label_for_filename,
 )
@@ -766,8 +766,8 @@ def run_benchmark(
     completed = metrics.get("completed_requests", metrics.get("completed", 0))
     failed = metrics.get("failed_requests", metrics.get("failed", 0))
 
-    # Persist the JSON baseline as-is (no hardware / sweep resolution).
-    baseline = copy.deepcopy(params.get("baseline") or {})
+    # Persist sweep-resolved baseline from params (already narrowed in _build_run_params).
+    baseline = params.get("baseline") or {}
     metrics["baseline"] = baseline
 
     record: dict[str, Any] = {
@@ -862,6 +862,7 @@ def _build_run_params(
     params: dict[str, Any],
     *,
     num_prompts: int,
+    sweep_index: int | None = None,
     request_rate: Any | None = None,
     max_concurrency: Any | None = None,
 ) -> dict[str, Any]:
@@ -873,6 +874,12 @@ def _build_run_params(
         run_params["request-rate"] = request_rate
     if max_concurrency is not None:
         run_params["max-concurrency"] = max_concurrency
+    if "baseline" in params:
+        # Keep all hardware buckets; pick the metric value for this sweep step only.
+        run_params["baseline"] = resolve_baseline_for_sweep(
+            params.get("baseline"),
+            sweep_index=sweep_index,
+        )
     return run_params
 
 
@@ -895,19 +902,20 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
 
     sweep_runs: list[dict[str, Any]] = []
 
-    for request_rate, num_prompts in zip(request_rate_list, num_prompt_list):
+    for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
         sweep_runs.append(
             {
                 "params": _build_run_params(
                     params,
                     request_rate=request_rate,
                     num_prompts=num_prompts,
+                    sweep_index=sweep_index,
                 ),
                 "num_prompts": num_prompts,
             }
         )
 
-    for max_concurrency, num_prompts in zip(max_concurrency_list, num_prompt_list):
+    for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
         sweep_runs.append(
             {
                 "params": _build_run_params(
@@ -915,6 +923,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
                     max_concurrency=max_concurrency,
                     num_prompts=num_prompts,
                     request_rate="inf",
+                    sweep_index=sweep_index,
                 ),
                 "num_prompts": num_prompts,
             }
@@ -927,6 +936,7 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
                 "params": _build_run_params(
                     params,
                     num_prompts=default_num_prompts,
+                    sweep_index=0,
                 ),
                 "num_prompts": default_num_prompts,
             }