[CI][Perf] Add JSON mark support and paired parametrize for L4 perf, so as to further support performance‑case selection for NPU‑CI. (#5084)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>

tests/dfx/perf/scripts/run_benchmark.py

@@ -7,21 +7,24 @@ from typing import Any
 import pytest
 
 from tests.dfx.conftest import (
-    create_benchmark_indices,
+    create_paired_omni_benchmark_pytest_params,
     create_test_parameter_mapping,
-    create_unique_server_params,
     get_benchmark_params_for_server,
-    load_configs,
+    get_runtime_resource_label,
+    is_diffusion_perf_config,
+    load_benchmark_configs,
     resolve_baseline_value,
     run_benchmark,
 )
 from tests.helpers.runtime import OmniServer
 
-pytestmark = [pytest.mark.full_model]
-
 # Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
 # (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
 # defined in the same module.
+#
+# Optional JSON field ``mark`` is applied as pytest marks via
+# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
+# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
 
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -40,22 +43,22 @@ def _get_config_file_from_argv() -> str | None:
 
 
 _PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests"
-_DEFAULT_CONFIG_FILE = str(_PERF_TESTS_DIR / "test_qwen_omni.json")
 
 CONFIG_FILE_PATH = _get_config_file_from_argv()
 if CONFIG_FILE_PATH is None:
+    _all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR)
+    BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)]
     print(
-        "No --test-config-file in argv, using default: tests/dfx/perf/tests/test_qwen_omni.json "
-        "(override with e.g. --test-config-file tests/dfx/perf/tests/test_tts.json)"
+        f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from "
+        f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; "
+        f"use -m to filter, e.g. -m tts)"
     )
-    CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
-
-BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
-
+else:
+    BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH)
 
 DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
+paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
 
 _omni_server_lock = threading.Lock()
 
@@ -91,17 +94,11 @@ def omni_server(request):
         print("OmniServer stopped")
 
 
-benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
-
-
 @pytest.fixture
-def benchmark_params(request, omni_server):
-    """Benchmark parameters fixture with proper parametrization"""
+def benchmark_params(request):
+    """Benchmark parameters fixture; paired with ``omni_server`` via parametrization."""
     test_name, param_index = request.param
 
-    if test_name != omni_server.test_name:
-        pytest.skip(f"Skipping parameter for {test_name} - current server is {omni_server.test_name}")
-
     all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
 
     if not all_params:
@@ -153,8 +150,11 @@ def assert_result(
 
 
 @pytest.mark.benchmark
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
+@pytest.mark.parametrize(
+    "omni_server,benchmark_params",
+    paired_benchmark_params,
+    indirect=["omni_server", "benchmark_params"],
+)
 def test_performance_benchmark(omni_server, benchmark_params, request):
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
@@ -168,6 +168,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
     print(f"Benchmark parameters: {benchmark_params}")
 
     assert_baseline = request.config.getoption("--assert-baseline", default=False)
+    resource_label = get_runtime_resource_label()
 
     def to_list(value, default=None):
         if value is None:
@@ -239,6 +240,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
             max_concurrency=None,
             random_input_len=params.get("random_input_len"),
             random_output_len=params.get("random_output_len"),
+            resource_label=resource_label,
         )
         assert_result(
             result,
@@ -264,6 +266,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
             max_concurrency=concurrency,
             random_input_len=params.get("random_input_len"),
             random_output_len=params.get("random_output_len"),
+            resource_label=resource_label,
         )
         assert_result(
             result,