[CI][Perf] Add JSON mark support and paired parametrize for L4 perf, so as to further support performance‑case selection for NPU‑CI. (#5084)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>

tests/dfx/perf/tests/test_runner_metadata.py

@@ -86,3 +86,202 @@ def test_enabled_false_entry_is_skipped():
     # Only the enabled=True entry should appear
     assert len(params) == 1
     assert params[0].get("task") == "voice_clone"
+
+
+def test_resolve_pytest_marks_hardware_dict_with_extra():
+    from tests.dfx.conftest import resolve_pytest_marks
+
+    marks = resolve_pytest_marks(
+        [
+            {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}},
+            "full_model",
+            "diffusion",
+            "local_model",
+        ]
+    )
+    names = {m.name for m in marks}
+    assert "H100" in names
+    assert "cuda" in names
+    assert "gpu" in names
+    assert "distributed_cuda" in names
+    assert "full_model" in names
+    assert "diffusion" in names
+    assert "local_model" in names
+
+
+def test_resolve_pytest_marks_rejects_legacy_object_format():
+    from tests.dfx.conftest import resolve_pytest_marks
+
+    with pytest.raises(ValueError, match="mark must be a list"):
+        resolve_pytest_marks(
+            {
+                "hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1},
+                "marks": ["full_model"],
+            }
+        )
+
+
+def test_extract_mark_resource_label():
+    from tests.dfx.conftest import extract_mark_resource_label
+
+    assert extract_mark_resource_label(None) == "na"
+    assert (
+        extract_mark_resource_label(
+            [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "full_model",
+            ]
+        )
+        == "H100"
+    )
+    assert (
+        extract_mark_resource_label(
+            [
+                {
+                    "hardware_marks": {
+                        "res": {"cuda": "H100", "rocm": "MI325"},
+                        "num_cards": 2,
+                    }
+                },
+                "diffusion",
+            ]
+        )
+        == "H100-MI325"
+    )
+
+
+def test_resource_label_for_filename():
+    from tests.dfx.conftest import resource_label_for_filename
+
+    assert resource_label_for_filename("H100") == ""
+    assert resource_label_for_filename("L4") == "L4"
+    assert resource_label_for_filename("910B") == "910B"
+    assert resource_label_for_filename("na") == "na"
+
+
+def test_hardware_json_value():
+    from tests.dfx.conftest import hardware_json_value
+
+    assert hardware_json_value("H100") == "H100"
+    assert hardware_json_value("na") == ""
+    assert hardware_json_value(None) == ""
+
+
+def test_extract_configs_resource_label(monkeypatch):
+    from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
+
+    monkeypatch.setattr(
+        "tests.dfx.conftest._read_runtime_device_name",
+        lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
+    )
+    get_runtime_resource_label(refresh=True)
+    assert extract_configs_resource_label([]) == "H100"
+    get_runtime_resource_label(refresh=True)
+    monkeypatch.setattr(
+        "tests.dfx.conftest._read_runtime_device_name",
+        lambda *, device_id=0: "Ascend910B2",
+    )
+    assert get_runtime_resource_label(refresh=True) == "910B"
+
+
+def test_load_benchmark_configs_from_dir(tmp_path):
+    from tests.dfx.conftest import load_benchmark_configs
+
+    (tmp_path / "a.json").write_text(
+        json.dumps([{"test_name": "test_a", "server_params": {"model": "m/a"}, "benchmark_params": []}]),
+        encoding="utf-8",
+    )
+    (tmp_path / "b.json").write_text(
+        json.dumps([{"test_name": "test_b", "server_params": {"model": "m/b"}, "benchmark_params": []}]),
+        encoding="utf-8",
+    )
+    configs = load_benchmark_configs(config_dir=tmp_path)
+    assert [c["test_name"] for c in configs] == ["test_a", "test_b"]
+
+
+def test_create_unique_server_pytest_params_applies_marks(tmp_path):
+    from tests.dfx.conftest import create_unique_server_pytest_params
+
+    configs = [
+        {
+            "test_name": "test_with_mark",
+            "mark": [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "full_model",
+            ],
+            "server_params": {"model": "some/model"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}],
+        },
+        {
+            "test_name": "test_without_mark",
+            "server_params": {"model": "other/model"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}],
+        },
+    ]
+    params = create_unique_server_pytest_params(configs, tmp_path)
+    by_id = {p.id: p for p in params}
+    assert len(by_id["test_with_mark"].values) == 1
+    assert isinstance(by_id["test_with_mark"].values[0], tuple)
+    assert any(m.name == "H100" for m in by_id["test_with_mark"].marks)
+    assert not any(m.name == "H100" for m in by_id["test_without_mark"].marks)
+
+
+def test_is_diffusion_perf_config():
+    from tests.dfx.conftest import is_diffusion_perf_config
+
+    assert not is_diffusion_perf_config(
+        {"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]}
+    )
+    assert is_diffusion_perf_config(
+        {
+            "test_name": "diff_a",
+            "server_type": "vllm-omni",
+            "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
+        }
+    )
+
+
+def test_benchmark_param_id_suffix_from_task_eval_phase():
+    from tests.dfx.conftest import _unique_benchmark_param_id_suffixes
+
+    params = [
+        {"task": "default_voice", "eval_phase": "latency"},
+        {"task": "default_voice", "eval_phase": "throughput"},
+    ]
+    assert _unique_benchmark_param_id_suffixes(params) == [
+        "default_voice_latency",
+        "default_voice_throughput",
+    ]
+
+
+def test_create_paired_omni_benchmark_pytest_params(tmp_path):
+    from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params
+
+    configs = [
+        {
+            "test_name": "test_omni",
+            "mark": [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "omni",
+            ],
+            "server_params": {"model": "m/omni"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}, {"name": "p1", "num_prompts": 2}],
+        },
+        {
+            "test_name": "test_tts",
+            "mark": [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "tts",
+            ],
+            "server_params": {"model": "m/tts"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}],
+        },
+    ]
+    params = create_paired_omni_benchmark_pytest_params(configs, tmp_path)
+    by_id = {p.id: p for p in params}
+    assert set(by_id) == {"test_omni-p0", "test_omni-p1", "test_tts-p0"}
+    omni_row, bench_row = by_id["test_tts-p0"].values
+    assert omni_row[0] == "test_tts"
+    assert bench_row == ("test_tts", 0)
+    assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
+    assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)