[CI][Perf] Add JSON mark support and paired parametrize for L4 perf, so as to further support performance‑case selection for NPU‑CI. (#5084)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>

tests/dfx/conftest.py

@@ -11,11 +11,180 @@ from typing import Any
 import pytest
 
 from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw
+from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServerParams
 from tests.helpers.stage_config import modify_stage_config
 from vllm_omni.platforms import current_omni_platform
 
 
+def _named_pytest_marks(names: list[str]) -> list[pytest.MarkDecorator]:
+    marks: list[pytest.MarkDecorator] = []
+    for name in names:
+        name = name.strip()
+        if not name:
+            raise ValueError("mark name must be a non-empty string")
+        marks.append(getattr(pytest.mark, name))
+    return marks
+
+
+def _hardware_marks_from_dict(hw: Any) -> list[pytest.MarkDecorator]:
+    if not isinstance(hw, dict):
+        raise ValueError(f"mark.hardware_marks must be a dict, got {type(hw).__name__}")
+    res = hw.get("res")
+    if not isinstance(res, dict):
+        raise ValueError(f"mark.hardware_marks.res must be a dict, got {type(res).__name__}")
+    num_cards = hw.get("num_cards", 1)
+    return list(hardware_marks(res=res, num_cards=num_cards))
+
+
+def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]:
+    """Convert a JSON ``mark`` field into pytest mark decorators.
+
+    Supported form (per test-case object in perf/stability JSON)::
+
+        "mark": [
+            {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}},
+            "full_model",
+            "diffusion"
+        ]
+
+    Exactly one ``hardware_marks`` object is required when ``mark`` is present.
+    ``hardware_marks`` delegates to :func:`tests.helpers.mark.hardware_marks`
+    (same shape as ``@hardware_test``). Additional array entries are registered
+    pytest marker names (strings).
+    """
+    if mark_field is None:
+        return []
+
+    if isinstance(mark_field, list):
+        marks: list[pytest.MarkDecorator] = []
+        hw_seen = False
+        for item in mark_field:
+            if isinstance(item, dict) and "hardware_marks" in item:
+                if hw_seen:
+                    raise ValueError("mark array must contain at most one hardware_marks object")
+                hw_seen = True
+                marks.extend(_hardware_marks_from_dict(item["hardware_marks"]))
+                unknown_keys = set(item) - {"hardware_marks"}
+                if unknown_keys:
+                    raise ValueError(
+                        f"mark hardware_marks object only allows hardware_marks; unknown keys: {sorted(unknown_keys)}"
+                    )
+            elif isinstance(item, str):
+                item = item.strip()
+                if not item:
+                    raise ValueError("mark name must be a non-empty string")
+                marks.extend(_named_pytest_marks([item]))
+            else:
+                raise ValueError(
+                    f"mark array entries must be hardware_marks objects or marker name strings; got {type(item).__name__}"
+                )
+        if not hw_seen:
+            raise ValueError("mark array must contain a hardware_marks object")
+        return marks
+
+    raise ValueError(f"mark must be a list; got {type(mark_field).__name__}")
+
+
+def _mark_names(mark_field: Any) -> set[str]:
+    if isinstance(mark_field, list):
+        return {str(item) for item in mark_field if isinstance(item, str)}
+    return set()
+
+
+def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool:
+    """True for perf JSON cases intended for ``run_diffusion_benchmark.py``."""
+    if cfg.get("server_type") is not None:
+        return True
+    return "diffusion" in _mark_names(cfg.get("mark"))
+
+
+def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]:
+    return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs}
+
+
+def create_unique_server_pytest_params(
+    configs: list[dict[str, Any]],
+    stage_configs_dir: Path,
+) -> list[Any]:
+    """Like :func:`create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks."""
+    marks_by_name = _marks_by_test_name(configs)
+    return [
+        pytest.param(
+            row,
+            marks=marks_by_name.get(row[0], []),
+            id=row[0],
+        )
+        for row in create_unique_server_params(configs, stage_configs_dir)
+    ]
+
+
+def create_benchmark_pytest_params(
+    benchmark_configs: list[dict[str, Any]],
+    server_to_benchmark_mapping: dict[str, dict],
+) -> list[Any]:
+    """Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks."""
+    marks_by_name = _marks_by_test_name(benchmark_configs)
+    params: list[Any] = []
+    seen: set[str] = set()
+    for config in benchmark_configs:
+        test_name = config["test_name"]
+        if test_name in seen:
+            continue
+        seen.add(test_name)
+        params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
+        id_suffixes = _unique_benchmark_param_id_suffixes(params_list)
+        for idx, id_suffix in enumerate(id_suffixes):
+            params.append(
+                pytest.param(
+                    (test_name, idx),
+                    marks=marks_by_name.get(test_name, []),
+                    id=f"{test_name}-{id_suffix}",
+                )
+            )
+    return params
+
+
+def create_paired_benchmark_pytest_params(
+    server_entries: list[tuple[Any, str]],
+    params_by_test_name: dict[str, list[Any]],
+    marks_by_name: dict[str, list[pytest.MarkDecorator]],
+) -> list[Any]:
+    """One ``pytest.param`` per ``(server entry, benchmark index)`` pair.
+
+    Pass ``server_entry`` and ``(test_name, idx)`` as separate ``pytest.param``
+    arguments for ``@pytest.mark.parametrize("server_fixture,benchmark_params", ...)``.
+    """
+    pairs: list[Any] = []
+    for server_entry, test_name in server_entries:
+        params_list = params_by_test_name.get(test_name, [])
+        id_suffixes = _unique_benchmark_param_id_suffixes(params_list)
+        for idx, id_suffix in enumerate(id_suffixes):
+            pairs.append(
+                pytest.param(
+                    server_entry,
+                    (test_name, idx),
+                    marks=marks_by_name.get(test_name, []),
+                    id=f"{test_name}-{id_suffix}",
+                )
+            )
+    return pairs
+
+
+def create_paired_omni_benchmark_pytest_params(
+    configs: list[dict[str, Any]],
+    stage_configs_dir: Path,
+) -> list[Any]:
+    """Paired params for ``run_benchmark.py`` (omni/tts)."""
+    mapping = create_test_parameter_mapping(configs)
+    marks_by_name = _marks_by_test_name(configs)
+    server_entries = [(row, row[0]) for row in create_unique_server_params(configs, stage_configs_dir)]
+    params_by_test_name = {
+        test_name: get_benchmark_params_for_server(test_name, mapping) for _, test_name in server_entries
+    }
+    return create_paired_benchmark_pytest_params(server_entries, params_by_test_name, marks_by_name)
+
+
 def load_configs(config_path: str) -> list[dict[str, Any]]:
     try:
         abs_path = Path(config_path).resolve()
@@ -32,6 +201,29 @@ def load_configs(config_path: str) -> list[dict[str, Any]]:
         raise RuntimeError(f"Failed to load configuration file: {str(e)}")
 
 
+def load_benchmark_configs(
+    config_path: str | None = None,
+    *,
+    config_dir: Path | None = None,
+) -> list[dict[str, Any]]:
+    """Load one benchmark JSON file, or merge all ``*.json`` under *config_dir*.
+
+    When *config_path* is omitted, every ``*.json`` in *config_dir* is loaded and
+    concatenated. Pytest ``-m`` expressions then select cases via each entry's
+    ``mark`` field (e.g. ``-m tts`` after bulk load).
+    """
+    if config_path is not None:
+        return load_configs(config_path)
+    if config_dir is None:
+        raise ValueError("load_benchmark_configs requires config_path or config_dir")
+    configs: list[dict[str, Any]] = []
+    for path in sorted(config_dir.glob("*.json")):
+        configs.extend(load_configs(str(path)))
+    if not configs:
+        raise ValueError(f"No benchmark JSON files found under {config_dir}")
+    return configs
+
+
 def modify_stage(default_path: str, updates: dict[str, Any] | None, deletes: dict[str, Any] | None) -> str:
     kwargs: dict[str, Any] = {}
     if updates is not None:
@@ -293,6 +485,158 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     return s if s else default
 
 
+def _benchmark_param_id_suffix(param: dict[str, Any], *, idx: int) -> str:
+    """Derive a readable pytest id suffix from one ``benchmark_params`` entry."""
+    name = param.get("name")
+    if isinstance(name, str) and name.strip():
+        return _safe_filename_token(name.strip())
+
+    parts: list[str] = []
+    for key in ("task", "eval_phase", "dataset_name", "dataset"):
+        value = param.get(key)
+        if value is None or value == "":
+            continue
+        token = _safe_filename_token(str(value))
+        if token != "na":
+            parts.append(token)
+    if parts:
+        return "_".join(parts)
+
+    return f"case{idx}"
+
+
+def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> list[str]:
+    """Return unique pytest id suffixes for a server's benchmark param list."""
+    raw = [_benchmark_param_id_suffix(param, idx=idx) for idx, param in enumerate(params_list)]
+    seen: dict[str, int] = {}
+    unique: list[str] = []
+    for suffix in raw:
+        count = seen.get(suffix, 0)
+        seen[suffix] = count + 1
+        if count == 0:
+            unique.append(suffix)
+        else:
+            unique.append(f"{suffix}_{count}")
+    return unique
+
+
+def extract_mark_resource_label(mark_field: Any) -> str:
+    """Return a filename-safe hardware label from ``mark.hardware_marks.res`` values.
+
+    Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``.
+
+    Prefer :func:`get_runtime_resource_label` for perf result filenames so labels
+    reflect the machine that actually ran the benchmark.
+    """
+    if isinstance(mark_field, list):
+        for item in mark_field:
+            if isinstance(item, dict) and "hardware_marks" in item:
+                return extract_mark_resource_label(item)
+        return "na"
+    if not isinstance(mark_field, dict):
+        return "na"
+    hw = mark_field.get("hardware_marks")
+    if not isinstance(hw, dict):
+        return "na"
+    res = hw.get("res")
+    if not isinstance(res, dict) or not res:
+        return "na"
+    labels = [_safe_filename_token(value) for value in res.values()]
+    return "-".join(labels) if labels else "na"
+
+
+_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = (
+    "H100",
+    "H800",
+    "H200",
+    "H20",
+    "L40S",
+    "L40",
+    "L4",
+    "A100",
+    "A800",
+    "A10G",
+    "A10",
+    "A30",
+    "MI325",
+    "MI300",
+    "MI250",
+    "B60",
+    "S5000",
+    "910B4",
+    "910B",
+    "910",
+    "310P",
+    "A2",
+    "A3",
+)
+_RUNTIME_RESOURCE_LABEL: str | None = None
+
+
+def _normalize_runtime_device_label(raw: str) -> str:
+    """Map a platform device name to a short filename-safe resource token."""
+    if not raw or not str(raw).strip():
+        return "na"
+    upper = str(raw).upper()
+    for token in _KNOWN_RUNTIME_RESOURCE_TOKENS:
+        if token.upper() in upper:
+            return _safe_filename_token(token)
+    compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw))
+    for prefix in ("NVIDIA", "AMD", "ASCEND", "HUAWEI"):
+        if compact.upper().startswith(prefix):
+            compact = compact[len(prefix) :]
+            break
+    return _safe_filename_token(compact[:48]) if compact else "na"
+
+
+def _read_runtime_device_name(*, device_id: int = 0) -> str | None:
+    """Device name from the active Omni platform."""
+    if current_omni_platform.device_count() <= device_id:
+        return None
+    get_name = getattr(current_omni_platform, "get_device_name", None)
+    if not callable(get_name):
+        return None
+    raw = get_name(device_id)
+    if isinstance(raw, str) and raw.strip():
+        return raw.strip()
+    return None
+
+
+def get_runtime_resource_label(*, device_id: int = 0, refresh: bool = False) -> str:
+    """Return a filename-safe hardware label detected on the running machine."""
+    global _RUNTIME_RESOURCE_LABEL
+    if not refresh and _RUNTIME_RESOURCE_LABEL is not None:
+        return _RUNTIME_RESOURCE_LABEL
+    raw = _read_runtime_device_name(device_id=device_id)
+    label = _normalize_runtime_device_label(raw) if raw else "na"
+    if not refresh:
+        _RUNTIME_RESOURCE_LABEL = label
+    return label
+
+
+_FILENAME_OMIT_RESOURCE_LABELS = frozenset({"H100"})
+
+
+def hardware_json_value(resource_label: str | None) -> str:
+    """Hardware token stored in perf result JSON (empty when unknown)."""
+    token = _safe_filename_token(resource_label)
+    return "" if token == "na" else token
+
+
+def resource_label_for_filename(resource_label: str | None) -> str:
+    """Hardware token embedded in result filenames (H100 omitted on default CI pool)."""
+    token = _safe_filename_token(resource_label)
+    if token in _FILENAME_OMIT_RESOURCE_LABELS:
+        return ""
+    return token
+
+
+def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str:
+    """Return runtime hardware label for perf result filenames."""
+    del configs
+    return get_runtime_resource_label()
+
+
 def resolve_baseline_value(
     baseline_raw: Any,
     *,
@@ -357,6 +701,7 @@ def run_benchmark(
     max_concurrency: Any | None = None,
     random_input_len: Any | None = None,
     random_output_len: Any | None = None,
+    resource_label: str | None = None,
 ) -> dict[str, Any]:
     """Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
 
@@ -367,7 +712,11 @@ def run_benchmark(
     current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
     ri = _safe_filename_token(random_input_len)
     ro = _safe_filename_token(random_output_len)
-    result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
+    hw = resource_label_for_filename(resource_label)
+    if hw:
+        result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
+    else:
+        result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
     if "--result-filename" in args:
         print(f"The result file will be overwritten by {result_filename}")
     command = (
@@ -437,6 +786,7 @@ def run_benchmark(
         result["random_input_len"] = random_input_len
     if random_output_len is not None:
         result["random_output_len"] = random_output_len
+    result["Hardware"] = hardware_json_value(resource_label)
     with open(result_path, "w", encoding="utf-8") as f:
         json.dump(result, f, ensure_ascii=False, indent=2)
     return result

tests/dfx/perf/scripts/run_benchmark.py

@@ -7,21 +7,24 @@ from typing import Any
 import pytest
 
 from tests.dfx.conftest import (
-    create_benchmark_indices,
+    create_paired_omni_benchmark_pytest_params,
     create_test_parameter_mapping,
-    create_unique_server_params,
     get_benchmark_params_for_server,
-    load_configs,
+    get_runtime_resource_label,
+    is_diffusion_perf_config,
+    load_benchmark_configs,
     resolve_baseline_value,
     run_benchmark,
 )
 from tests.helpers.runtime import OmniServer
 
-pytestmark = [pytest.mark.full_model]
-
 # Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
 # (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
 # defined in the same module.
+#
+# Optional JSON field ``mark`` is applied as pytest marks via
+# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
+# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
 
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -40,22 +43,22 @@ def _get_config_file_from_argv() -> str | None:
 
 
 _PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests"
-_DEFAULT_CONFIG_FILE = str(_PERF_TESTS_DIR / "test_qwen_omni.json")
 
 CONFIG_FILE_PATH = _get_config_file_from_argv()
 if CONFIG_FILE_PATH is None:
+    _all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR)
+    BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)]
     print(
-        "No --test-config-file in argv, using default: tests/dfx/perf/tests/test_qwen_omni.json "
-        "(override with e.g. --test-config-file tests/dfx/perf/tests/test_tts.json)"
+        f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from "
+        f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; "
+        f"use -m to filter, e.g. -m tts)"
     )
-    CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
-
-BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
-
+else:
+    BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH)
 
 DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
+paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
 
 _omni_server_lock = threading.Lock()
 
@@ -91,17 +94,11 @@ def omni_server(request):
         print("OmniServer stopped")
 
 
-benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
-
-
 @pytest.fixture
-def benchmark_params(request, omni_server):
-    """Benchmark parameters fixture with proper parametrization"""
+def benchmark_params(request):
+    """Benchmark parameters fixture; paired with ``omni_server`` via parametrization."""
     test_name, param_index = request.param
 
-    if test_name != omni_server.test_name:
-        pytest.skip(f"Skipping parameter for {test_name} - current server is {omni_server.test_name}")
-
     all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
 
     if not all_params:
@@ -153,8 +150,11 @@ def assert_result(
 
 
 @pytest.mark.benchmark
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
+@pytest.mark.parametrize(
+    "omni_server,benchmark_params",
+    paired_benchmark_params,
+    indirect=["omni_server", "benchmark_params"],
+)
 def test_performance_benchmark(omni_server, benchmark_params, request):
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
@@ -168,6 +168,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
     print(f"Benchmark parameters: {benchmark_params}")
 
     assert_baseline = request.config.getoption("--assert-baseline", default=False)
+    resource_label = get_runtime_resource_label()
 
     def to_list(value, default=None):
         if value is None:
@@ -239,6 +240,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
             max_concurrency=None,
             random_input_len=params.get("random_input_len"),
             random_output_len=params.get("random_output_len"),
+            resource_label=resource_label,
         )
         assert_result(
             result,
@@ -264,6 +266,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
             max_concurrency=concurrency,
             random_input_len=params.get("random_input_len"),
             random_output_len=params.get("random_output_len"),
+            resource_label=resource_label,
         )
         assert_result(
             result,

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -8,15 +8,21 @@ This runner separates two concepts:
 2. ``benchmark_endpoint``: which serving API the benchmark client calls.
    Examples: ``/v1/chat/completions`` and ``/v1/videos``.
 
-A config JSON file is REQUIRED via --test-config-file:
+A config JSON file may be passed via --test-config-file. If omitted, every ``*.json`` under
+``tests/dfx/perf/tests/`` is loaded and pytest ``-m`` filters by each case's ``mark``:
+  pytest run_diffusion_benchmark.py -m "diffusion"
   pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 
 Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
 
-All benchmark results for a session are consolidated into a single JSON file under
-BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
-Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
-timestamp) together with the raw metrics returned by the benchmark script.
+Optional JSON field ``mark`` is applied as pytest marks on that case via
+``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``).
+
+All benchmark results are written under BENCHMARK_RESULT_DIR (override via the
+DIFFUSION_BENCHMARK_DIR environment variable). Each source JSON file gets one
+aggregated ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` (JSON array
+of all runs from cases in that file). Bulk load without ``--test-config-file`` uses
+the same per-file aggregation; ``-m`` only selects which cases run.
 """
 
 import json
@@ -35,8 +41,15 @@ import psutil
 import pytest
 
 from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint
-
-pytestmark = [pytest.mark.diffusion, pytest.mark.full_model, pytest.mark.local_model]
+from tests.dfx.conftest import (
+    create_paired_benchmark_pytest_params,
+    get_runtime_resource_label,
+    hardware_json_value,
+    is_diffusion_perf_config,
+    resolve_pytest_marks,
+    resource_label_for_filename,
+)
+from tests.helpers.runtime import get_open_port
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN")
@@ -126,6 +139,10 @@ _BRANCHPOINT_COMMIT_SHA: str | None = None
 DIFFUSION_RESULT_TEMPLATE_PATH = Path(__file__).parent / "diffusion_result_template.json"
 
 
+_DIFFUSION_SOURCE_CONFIG_KEY = "_source_config_file"
+_PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests"
+
+
 def _get_config_file_from_argv() -> str | None:
     """Read --test-config-file from sys.argv at import time so pytest parametrize can use it.
 
@@ -142,9 +159,6 @@ def _get_config_file_from_argv() -> str | None:
 
 
 CONFIG_FILE_PATH = _get_config_file_from_argv()
-if CONFIG_FILE_PATH is None:
-    print("No config file provided, using default config file: tests/dfx/perf/tests/test_qwen_image_vllm_omni.json")
-    CONFIG_FILE_PATH = "tests/dfx/perf/tests/test_qwen_image_vllm_omni.json"
 
 # ---------------------------------------------------------------------------
 # Config loading
@@ -184,28 +198,85 @@ def load_configs(config_path: str) -> list[dict[str, Any]]:
         raise RuntimeError(f"Failed to load configuration file: {str(e)}")
 
 
-BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
-
-_config_stem = Path(CONFIG_FILE_PATH).stem  # e.g. "test_qwen_image_vllm_omni"
-AGGREGATED_RESULT_FILE = BENCHMARK_RESULT_DIR / f"diffusion_result_{_config_stem}_{_SESSION_TIMESTAMP}.json"
+def load_diffusion_benchmark_configs(
+    config_path: str | None = None,
+    *,
+    config_dir: Path | None = None,
+) -> list[dict[str, Any]]:
+    """Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*."""
+    if config_path is not None:
+        configs = load_configs(config_path)
+        source = str(Path(config_path).resolve())
+        for cfg in configs:
+            cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source)
+        return configs
+    if config_dir is None:
+        raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir")
+    configs: list[dict[str, Any]] = []
+    for path in sorted(config_dir.glob("*.json")):
+        source = str(path.resolve())
+        for cfg in load_configs(str(path)):
+            cfg[_DIFFUSION_SOURCE_CONFIG_KEY] = source
+            configs.append(cfg)
+    if not configs:
+        raise ValueError(f"No benchmark JSON files found under {config_dir}")
+    return configs
 
 
-def _append_to_aggregated_file(record: dict[str, Any]) -> None:
-    """Thread-safe append of *record* to the session-level aggregated JSON file.
+if CONFIG_FILE_PATH is None:
+    _all_configs = load_diffusion_benchmark_configs(config_dir=_PERF_TESTS_DIR)
+    BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if is_diffusion_perf_config(cfg)]
+    print(
+        f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s) from "
+        f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} omni/tts; "
+        f"use -m to filter, e.g. -m diffusion)"
+    )
+else:
+    BENCHMARK_CONFIGS = load_diffusion_benchmark_configs(CONFIG_FILE_PATH)
 
-    The file contains a JSON array; each call loads the existing array (or
-    starts a new one), appends the record, and writes the file back atomically.
-    """
+_AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {}
+
+
+def _normalized_source_path(source_file: str) -> str:
+    return str(Path(source_file).resolve())
+
+
+def _aggregated_result_file_for_source(source_file: str) -> Path:
+    """One session aggregate per source JSON (same naming as single ``--test-config-file``)."""
+    key = _normalized_source_path(source_file)
+    if key not in _AGGREGATED_RESULT_FILES_BY_SOURCE:
+        stem = Path(key).stem
+        resource = resource_label_for_filename(get_runtime_resource_label())
+        if resource:
+            result_name = f"diffusion_result_{stem}_{resource}_{_SESSION_TIMESTAMP}.json"
+        else:
+            result_name = f"diffusion_result_{stem}_{_SESSION_TIMESTAMP}.json"
+        _AGGREGATED_RESULT_FILES_BY_SOURCE[key] = BENCHMARK_RESULT_DIR / result_name
+    return _AGGREGATED_RESULT_FILES_BY_SOURCE[key]
+
+
+def _write_result_record(record: dict[str, Any]) -> Path:
+    """Append one benchmark record to the aggregate file for its source JSON."""
+    source_file = record.get("source_file") or CONFIG_FILE_PATH
+    if not source_file:
+        raise ValueError("benchmark record missing source_file")
+    target = _aggregated_result_file_for_source(str(source_file))
     with _RESULT_LOCK:
         BENCHMARK_RESULT_DIR.mkdir(parents=True, exist_ok=True)
-        if AGGREGATED_RESULT_FILE.exists():
-            with open(AGGREGATED_RESULT_FILE, encoding="utf-8") as f:
+        if target.exists():
+            with open(target, encoding="utf-8") as f:
                 records: list[dict] = json.load(f)
         else:
             records = []
         records.append(record)
-        with open(AGGREGATED_RESULT_FILE, "w", encoding="utf-8") as f:
+        with open(target, "w", encoding="utf-8") as f:
             json.dump(records, f, indent=2, ensure_ascii=False)
+    return target
+
+
+def _append_to_aggregated_file(record: dict[str, Any]) -> None:
+    """Backward-compatible wrapper; prefer :func:`_write_result_record`."""
+    _write_result_record(record)
 
 
 _server_lock = threading.Lock()
@@ -215,14 +286,6 @@ _server_lock = threading.Lock()
 # ---------------------------------------------------------------------------
 
 
-def _get_open_port() -> int:
-    """Return an available TCP port on localhost."""
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("", 0))
-        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        return s.getsockname()[1]
-
-
 def _wait_for_port(host: str, port: int, timeout: int = 1200, proc: subprocess.Popen | None = None) -> None:
     """Block until the given host:port accepts connections or timeout expires.
 
@@ -312,7 +375,7 @@ class DiffusionServer:
         self.model = server_cfg["model"]
         self.serve_args = server_cfg["serve_args"]
         self.host = "127.0.0.1"
-        self.port = port if port is not None else _get_open_port()
+        self.port = port if port is not None else get_open_port(self.host)
         self.proc: subprocess.Popen | None = None
         self.test_name: str = ""
 
@@ -482,6 +545,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]]
                 "serve_args": _build_serve_args(serve_args_dict),
                 "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")),
                 "server_params": cfg["server_params"],
+                "mark": cfg.get("mark"),
             }
         )
     return result
@@ -496,19 +560,29 @@ def _test_param_mapping(configs: list[dict[str, Any]]) -> dict[str, list[dict]]:
     return mapping
 
 
-def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer:
-    """Factory: return a vLLM-Omni diffusion server instance for the config."""
-    return DiffusionServer(server_cfg=server_cfg)
+def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]:
+    return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs}
+
+
+def _paired_diffusion_benchmark_pytest_params(configs: list[dict[str, Any]]) -> list[Any]:
+    """Paired params for ``run_diffusion_benchmark.py``; same shape as omni runner."""
+    test_param_map = _test_param_mapping(configs)
+    server_entries = [(cfg, cfg["test_name"]) for cfg in _unique_server_params(configs)]
+    return create_paired_benchmark_pytest_params(server_entries, test_param_map, _marks_by_test_name(configs))
 
 
 # ---------------------------------------------------------------------------
 # Parametrize data
 # ---------------------------------------------------------------------------
 
-server_params = _unique_server_params(BENCHMARK_CONFIGS)
 test_param_map = _test_param_mapping(BENCHMARK_CONFIGS)
+paired_benchmark_params = _paired_diffusion_benchmark_pytest_params(BENCHMARK_CONFIGS)
+
+
+def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer:
+    """Factory: return a vLLM-Omni diffusion server instance for the config."""
+    return DiffusionServer(server_cfg=server_cfg)
 
-benchmark_indices: list[int] = list(range(max(len(v) for v in test_param_map.values())))
 
 # ---------------------------------------------------------------------------
 # Pytest fixtures
@@ -534,16 +608,13 @@ def diffusion_server(request):
 
 
 @pytest.fixture
-def benchmark_params(request, diffusion_server):
-    """Yield the benchmark params dict for the current (server, index) pair."""
-    param_index: int = request.param
-    test_name = diffusion_server.test_name
+def benchmark_params(request):
+    """Benchmark params for the paired server/index parametrization."""
+    test_name, param_index = request.param
 
     params_list = test_param_map.get(test_name, [])
     if not params_list:
         raise ValueError(f"No benchmark params for test: {test_name}")
-    if param_index >= len(params_list):
-        pytest.skip(f"Param index {param_index} out of range for {test_name} (has {len(params_list)} params)")
 
     current = param_index + 1
     total = len(params_list)
@@ -575,8 +646,8 @@ def run_benchmark(
     The raw metrics are written to a temporary file by the subprocess.  After
     the run completes the metrics are merged with full metadata (test_name,
     endpoint, benchmark_params, timestamp, flat reporting fields) and appended
-    to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE).  The
-    temporary file is removed afterwards.  Subprocess stdout/stderr are tee'd
+    to ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` for the
+    source JSON file. The temporary file is removed afterwards.  Subprocess stdout/stderr are tee'd
     to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in
     the record.
     """
@@ -586,7 +657,12 @@ def run_benchmark(
     log_dir = BENCHMARK_RESULT_DIR / "logs"
     log_dir.mkdir(parents=True, exist_ok=True)
     endpoint_label = endpoint_filename_token(endpoint)
-    log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log"
+    resource_label = get_runtime_resource_label()
+    hw_for_filename = resource_label_for_filename(resource_label)
+    if hw_for_filename:
+        log_file = log_dir / f"{test_name}_{hw_for_filename}_{endpoint_label}_{timestamp}.log"
+    else:
+        log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log"
 
     with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
         tmp_result_file = Path(tmp.name)
@@ -696,7 +772,7 @@ def run_benchmark(
         "Model": model,
         "Framework": server_type,
         "API Endpoint": endpoint,
-        "Hardware": "",
+        "Hardware": hardware_json_value(resource_label),
         "Deployment": "",
         "Task": params.get("task", "t2i"),
         "Dataset": params.get("dataset", "random"),
@@ -728,8 +804,8 @@ def run_benchmark(
         "build_url": os.environ.get("BUILDKITE_BUILD_URL", ""),
         "source_file": source_file,
     }
-    _append_to_aggregated_file(record)
-    print(f"\n  Result appended to: {AGGREGATED_RESULT_FILE}")
+    result_path = _write_result_record(record)
+    print(f"\n  Result saved to: {result_path}")
     print(f"  Log saved to:       {log_file}")
 
     return metrics
@@ -932,12 +1008,10 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
 # ---------------------------------------------------------------------------
 @pytest.mark.benchmark
 @pytest.mark.parametrize(
-    "diffusion_server",
-    server_params,
-    ids=[p["test_name"] for p in server_params],
-    indirect=True,
+    "diffusion_server,benchmark_params",
+    paired_benchmark_params,
+    indirect=["diffusion_server", "benchmark_params"],
 )
-@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
 def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
     """Run the diffusion performance benchmark and verify request completion.
 
@@ -962,7 +1036,10 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req
             test_name=test_name,
             endpoint=endpoint,
             server_cfg=server_cfg,
-            source_file=cast(str, CONFIG_FILE_PATH),
+            source_file=server_cfg.get(
+                _DIFFUSION_SOURCE_CONFIG_KEY,
+                CONFIG_FILE_PATH or f"{_PERF_TESTS_DIR}/*.json",
+            ),
         )
 
         print(f"\n{'=' * 60}")
@@ -980,7 +1057,9 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req
             if key in result:
                 print(f"  {key}: {result[key]:.4f}")
 
-        print(f"\n  Aggregated results: {AGGREGATED_RESULT_FILE}")
+        source = server_cfg.get(_DIFFUSION_SOURCE_CONFIG_KEY) or CONFIG_FILE_PATH
+        if source:
+            print(f"\n  Aggregated results: {_aggregated_result_file_for_source(str(source))}")
         print("=" * 60)
 
         assert_baseline = request.config.getoption("--assert-baseline", default=False)

tests/dfx/perf/tests/test_bagel_vllm_omni.json

@@ -1,6 +1,18 @@
 [
   {
     "test_name": "test_bagel_single_device_single_stage_t2i",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
     "description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task",
     "server_type": "vllm-omni",
     "server_params": {
@@ -38,6 +50,18 @@
   },
   {
     "test_name": "test_bagel_single_device_single_stage_i2i",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
     "description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task",
     "server_type": "vllm-omni",
     "server_params": {
@@ -76,6 +100,18 @@
   },
   {
     "test_name": "test_bagel_single_device_multi_stage_t2i",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
     "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task",
     "server_type": "vllm-omni",
     "server_params": {
@@ -113,6 +149,18 @@
   },
   {
     "test_name": "test_bagel_single_device_multi_stage_i2i",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
     "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task",
     "server_type": "vllm-omni",
     "server_params": {

tests/dfx/perf/tests/test_cosmos3_vllm_omni.json

@@ -1,213 +1,261 @@
 [
-    {
-        "test_name": "test_cosmos3_t2i_official_demo_2gpu",
-        "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs",
-        "server_type": "vllm-omni",
-        "benchmark_endpoint": "/v1/images/generations",
-        "server_params": {
-            "model": "nvidia/Cosmos3-Nano",
-            "serve_args": {
-                "cfg-parallel-size": 2,
-                "use-hsdp": true,
-                "hsdp-shard-size": 2,
-                "vae-patch-parallel-size": 2,
-                "vae-use-tiling": true,
-                "model-class-name": "Cosmos3OmniDiffusersPipeline",
-                "no-guardrails": true,
-                "stage-init-timeout": 900,
-                "init-timeout": 1200,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1024x1024_steps4",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1024,
-                "height": 1024,
-                "num-inference-steps": 4,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "seed": 42,
-                "extra-body": {
-                    "guidance_scale": 7.0,
-                    "flow_shift": 3.0,
-                    "guardrails": false
-                },
-                "baseline": {
-                    "throughput_qps": 1.107,
-                    "latency_mean": 0.8929,
-                    "peak_memory_mb_mean": 80000.0
-                }
-            }
-        ]
+  {
+    "test_name": "test_cosmos3_t2i_official_demo_2gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/images/generations",
+    "server_params": {
+      "model": "nvidia/Cosmos3-Nano",
+      "serve_args": {
+        "cfg-parallel-size": 2,
+        "use-hsdp": true,
+        "hsdp-shard-size": 2,
+        "vae-patch-parallel-size": 2,
+        "vae-use-tiling": true,
+        "model-class-name": "Cosmos3OmniDiffusersPipeline",
+        "no-guardrails": true,
+        "stage-init-timeout": 900,
+        "init-timeout": 1200,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_cosmos3_t2v_official_demo_2gpu",
-        "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs",
-        "server_type": "vllm-omni",
-        "benchmark_endpoint": "/v1/videos",
-        "server_params": {
-            "model": "nvidia/Cosmos3-Nano",
-            "serve_args": {
-                "cfg-parallel-size": 2,
-                "use-hsdp": true,
-                "hsdp-shard-size": 2,
-                "vae-patch-parallel-size": 2,
-                "vae-use-tiling": true,
-                "model-class-name": "Cosmos3OmniDiffusersPipeline",
-                "no-guardrails": true,
-                "stage-init-timeout": 900,
-                "init-timeout": 1200,
-                "enable-diffusion-pipeline-profiler": true
-            }
+    "benchmark_params": [
+      {
+        "name": "1024x1024_steps4",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1024,
+        "height": 1024,
+        "num-inference-steps": 4,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "seed": 42,
+        "extra-body": {
+          "guidance_scale": 7.0,
+          "flow_shift": 3.0,
+          "guardrails": false
         },
-        "benchmark_params": [
-            {
-                "name": "1280x720_frames189_steps4",
-                "dataset": "random",
-                "task": "t2v",
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "seed": 42,
-                "random-request-config": [
-                    {
-                        "width": 1280,
-                        "height": 720,
-                        "prompt": "A robot arm is cleaning a plate in the kitchen.",
-                        "num_inference_steps": 4,
-                        "num_frames": 189,
-                        "fps": 24,
-                        "weight": 1
-                    }
-                ],
-                "extra-body": {
-                    "guidance_scale": 6.0,
-                    "flow_shift": 10.0,
-                    "max_sequence_length": 4096,
-                    "guardrails": false,
-                    "use_resolution_template": false,
-                    "use_duration_template": false
-                },
-                "baseline": {
-                    "throughput_qps": 0.027,
-                    "latency_mean": 36.6824,
-                    "peak_memory_mb_mean": 30155.4
-                }
-            }
-        ]
+        "baseline": {
+          "throughput_qps": 1.107,
+          "latency_mean": 0.8929,
+          "peak_memory_mb_mean": 80000.0
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_cosmos3_t2v_official_demo_2gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "nvidia/Cosmos3-Nano",
+      "serve_args": {
+        "cfg-parallel-size": 2,
+        "use-hsdp": true,
+        "hsdp-shard-size": 2,
+        "vae-patch-parallel-size": 2,
+        "vae-use-tiling": true,
+        "model-class-name": "Cosmos3OmniDiffusersPipeline",
+        "no-guardrails": true,
+        "stage-init-timeout": 900,
+        "init-timeout": 1200,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_cosmos3_i2v_official_demo_2gpu",
-        "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs",
-        "server_type": "vllm-omni",
-        "benchmark_endpoint": "/v1/videos",
-        "server_params": {
-            "model": "nvidia/Cosmos3-Nano",
-            "serve_args": {
-                "cfg-parallel-size": 2,
-                "use-hsdp": true,
-                "hsdp-shard-size": 2,
-                "vae-patch-parallel-size": 2,
-                "vae-use-tiling": true,
-                "model-class-name": "Cosmos3OmniDiffusersPipeline",
-                "no-guardrails": true,
-                "stage-init-timeout": 900,
-                "init-timeout": 1200,
-                "enable-diffusion-pipeline-profiler": true
-            }
+    "benchmark_params": [
+      {
+        "name": "1280x720_frames189_steps4",
+        "dataset": "random",
+        "task": "t2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1280,
+            "height": 720,
+            "prompt": "A robot arm is cleaning a plate in the kitchen.",
+            "num_inference_steps": 4,
+            "num_frames": 189,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "guidance_scale": 6.0,
+          "flow_shift": 10.0,
+          "max_sequence_length": 4096,
+          "guardrails": false,
+          "use_resolution_template": false,
+          "use_duration_template": false
         },
-        "benchmark_params": [
-            {
-                "name": "1280x720_frames189_steps4",
-                "dataset": "random",
-                "task": "i2v",
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "num-input-images": 1,
-                "seed": 42,
-                "random-request-config": [
-                    {
-                        "width": 1280,
-                        "height": 720,
-                        "prompt": "The scene comes to life with smooth, natural motion.",
-                        "num_inference_steps": 4,
-                        "num_frames": 189,
-                        "fps": 24,
-                        "weight": 1
-                    }
-                ],
-                "extra-body": {
-                    "guidance_scale": 6.0,
-                    "flow_shift": 10.0,
-                    "max_sequence_length": 4096,
-                    "guardrails": false
-                },
-                "baseline": {
-                    "throughput_qps": 0.027,
-                    "latency_mean": 36.968,
-                    "peak_memory_mb_mean": 30443.6
-                }
-            }
-        ]
+        "baseline": {
+          "throughput_qps": 0.027,
+          "latency_mean": 36.6824,
+          "peak_memory_mb_mean": 30155.4
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_cosmos3_i2v_official_demo_2gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "nvidia/Cosmos3-Nano",
+      "serve_args": {
+        "cfg-parallel-size": 2,
+        "use-hsdp": true,
+        "hsdp-shard-size": 2,
+        "vae-patch-parallel-size": 2,
+        "vae-use-tiling": true,
+        "model-class-name": "Cosmos3OmniDiffusersPipeline",
+        "no-guardrails": true,
+        "stage-init-timeout": 900,
+        "init-timeout": 1200,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_cosmos3_v2v_official_demo_2gpu",
-        "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs",
-        "server_type": "vllm-omni",
-        "benchmark_endpoint": "/v1/videos",
-        "server_params": {
-            "model": "nvidia/Cosmos3-Nano",
-            "serve_args": {
-                "cfg-parallel-size": 2,
-                "use-hsdp": true,
-                "hsdp-shard-size": 2,
-                "vae-patch-parallel-size": 2,
-                "vae-use-tiling": true,
-                "model-class-name": "Cosmos3OmniDiffusersPipeline",
-                "no-guardrails": true,
-                "stage-init-timeout": 900,
-                "init-timeout": 1200,
-                "enable-diffusion-pipeline-profiler": true
-            }
+    "benchmark_params": [
+      {
+        "name": "1280x720_frames189_steps4",
+        "dataset": "random",
+        "task": "i2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "num-input-images": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1280,
+            "height": 720,
+            "prompt": "The scene comes to life with smooth, natural motion.",
+            "num_inference_steps": 4,
+            "num_frames": 189,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "guidance_scale": 6.0,
+          "flow_shift": 10.0,
+          "max_sequence_length": 4096,
+          "guardrails": false
         },
-        "benchmark_params": [
-            {
-                "name": "1280x720_frames189_steps4",
-                "dataset": "random",
-                "task": "v2v",
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "seed": 42,
-                "random-request-config": [
-                    {
-                        "width": 1280,
-                        "height": 720,
-                        "prompt": "Continue the same scene with smooth natural motion.",
-                        "num_inference_steps": 4,
-                        "num_frames": 189,
-                        "fps": 24,
-                        "weight": 1
-                    }
-                ],
-                "extra-body": {
-                    "guidance_scale": 6.0,
-                    "flow_shift": 10.0,
-                    "max_sequence_length": 4096,
-                    "guardrails": false,
-                    "condition_frame_indexes_vision": [
-                        0,
-                        1
-                    ],
-                    "condition_video_keep": "first"
-                },
-                "baseline": {
-                    "throughput_qps": 0.027,
-                    "latency_mean": 37.2117,
-                    "peak_memory_mb_mean": 30135.6
-                }
-            }
-        ]
-    }
+        "baseline": {
+          "throughput_qps": 0.027,
+          "latency_mean": 36.968,
+          "peak_memory_mb_mean": 30443.6
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_cosmos3_v2v_official_demo_2gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "nvidia/Cosmos3-Nano",
+      "serve_args": {
+        "cfg-parallel-size": 2,
+        "use-hsdp": true,
+        "hsdp-shard-size": 2,
+        "vae-patch-parallel-size": 2,
+        "vae-use-tiling": true,
+        "model-class-name": "Cosmos3OmniDiffusersPipeline",
+        "no-guardrails": true,
+        "stage-init-timeout": 900,
+        "init-timeout": 1200,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1280x720_frames189_steps4",
+        "dataset": "random",
+        "task": "v2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1280,
+            "height": 720,
+            "prompt": "Continue the same scene with smooth natural motion.",
+            "num_inference_steps": 4,
+            "num_frames": 189,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "guidance_scale": 6.0,
+          "flow_shift": 10.0,
+          "max_sequence_length": 4096,
+          "guardrails": false,
+          "condition_frame_indexes_vision": [
+            0,
+            1
+          ],
+          "condition_video_keep": "first"
+        },
+        "baseline": {
+          "throughput_qps": 0.027,
+          "latency_mean": 37.2117,
+          "peak_memory_mb_mean": 30135.6
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_higgs_audio_v3.json

@@ -1,6 +1,18 @@
 [
   {
     "test_name": "test_higgs_audio_v3",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "tts"
+    ],
     "server_params": {
       "model": "bosonai/higgs-audio-v3-tts-4b",
       "stage_overrides": {

tests/dfx/perf/tests/test_hunyuan_image3_it2i.json

@@ -1,140 +1,152 @@
 [
-    {
-        "test_name": "test_hunyuan_image3_ti2i_inline",
-        "description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
-        "server_type": "vllm-omni",
-        "benchmark_endpoint": "/v1/images/edits",
-        "server_params": {
-            "model": "tencent/HunyuanImage-3.0-Instruct",
-            "serve_args": {
-                "trust-remote-code": true,
-                "deploy-config-inline": {
-                    "type": "yaml",
-                    "content": {
-                        "pipeline": "hunyuan_image_3_moe",
-                        "async_chunk": false,
-                        "trust_remote_code": true,
-                        "connectors": {
-                            "rdma_connector": {
-                                "name": "MooncakeTransferEngineConnector",
-                                "extra": {
-                                    "host": "auto",
-                                    "zmq_port": 50051,
-                                    "protocol": "rdma",
-                                    "device_name": "",
-                                    "memory_pool_size": 17179869184,
-                                    "memory_pool_device": "cpu"
-                                }
-                            }
-                        },
-                        "stages": [
-                            {
-                                "stage_id": 0,
-                                "is_comprehension": true,
-                                "final_output": true,
-                                "final_output_type": "text",
-                                "max_num_seqs": 16,
-                                "gpu_memory_utilization": 0.95,
-                                "enforce_eager": false,
-                                "enable_prefix_caching": false,
-                                "max_num_batched_tokens": 32768,
-                                "devices": "0,1",
-                                "tensor_parallel_size": 2,
-                                "hf_overrides": {
-                                    "rope_parameters": {
-                                        "mrope_section": [
-                                            0,
-                                            32,
-                                            32
-                                        ],
-                                        "rope_type": "default"
-                                    }
-                                },
-                                "omni_kv_config": {
-                                    "need_send_cache": true
-                                },
-                                "output_connectors": {
-                                    "to_stage_1": "rdma_connector"
-                                },
-                                "default_sampling_params": {
-                                    "temperature": 0.0,
-                                    "top_p": 1,
-                                    "top_k": -1,
-                                    "max_tokens": 8192,
-                                    "detokenize": true,
-                                    "skip_special_tokens": false,
-                                    "stop_token_ids": [
-                                        128025
-                                    ]
-                                }
-                            },
-                            {
-                                "stage_id": 1,
-                                "max_num_seqs": 1,
-                                "enforce_eager": true,
-                                "devices": "2,3",
-                                "distributed_executor_backend": "mp",
-                                "omni_kv_config": {
-                                    "need_recv_cache": true
-                                },
-                                "parallel_config": {
-                                    "tensor_parallel_size": 2,
-                                    "vae_patch_parallel_size": 2,
-                                    "enable_expert_parallel": true
-                                },
-                                "input_connectors": {
-                                    "from_stage_0": "rdma_connector"
-                                },
-                                "default_sampling_params": {
-                                    "num_inference_steps": 50,
-                                    "guidance_scale": 0
-                                }
-                            }
-                        ],
-                        "edges": [
-                            {
-                                "from": 0,
-                                "to": 1,
-                                "window_size": -1,
-                                "max_inflight": 1
-                            }
-                        ]
-                    }
-                },
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "inline_dataset_5_requests",
-                "dataset": "custom",
-                "dataset-path-inline": {
-                    "type": "jsonl",
-                    "content": [
-                        {
-                            "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。",
-                            "image_urls": [
-                                "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"
-                            ],
-                            "num_inference_steps": 8,
-                            "seed": 42,
-                            "guidance_scale": 1.0
-                        }
-                    ]
-                },
-                "task": "ti2i",
-                "bot-task": "think_recaption",
-                "max-concurrency": 16,
-                "num-prompts": 96,
-                "skip-performance-assertion": true,
-                "baseline": {
-                    "throughput_qps": 0.1874,
-                    "latency_p99": 101.852,
-                    "latency_mean": 79.0385,
-                    "peak_memory_mb_max": 70000,
-                    "peak_memory_mb_mean": 70000
+  {
+    "test_name": "test_hunyuan_image3_ti2i_inline",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "diffusion",
+      "local_model"
+    ],
+    "description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/images/edits",
+    "server_params": {
+      "model": "tencent/HunyuanImage-3.0-Instruct",
+      "serve_args": {
+        "trust-remote-code": true,
+        "deploy-config-inline": {
+          "type": "yaml",
+          "content": {
+            "pipeline": "hunyuan_image_3_moe",
+            "async_chunk": false,
+            "trust_remote_code": true,
+            "connectors": {
+              "rdma_connector": {
+                "name": "MooncakeTransferEngineConnector",
+                "extra": {
+                  "host": "auto",
+                  "zmq_port": 50051,
+                  "protocol": "rdma",
+                  "device_name": "",
+                  "memory_pool_size": 17179869184,
+                  "memory_pool_device": "cpu"
                 }
+              }
+            },
+            "stages": [
+              {
+                "stage_id": 0,
+                "is_comprehension": true,
+                "final_output": true,
+                "final_output_type": "text",
+                "max_num_seqs": 16,
+                "gpu_memory_utilization": 0.95,
+                "enforce_eager": false,
+                "enable_prefix_caching": false,
+                "max_num_batched_tokens": 32768,
+                "devices": "0,1",
+                "tensor_parallel_size": 2,
+                "hf_overrides": {
+                  "rope_parameters": {
+                    "mrope_section": [
+                      0,
+                      32,
+                      32
+                    ],
+                    "rope_type": "default"
+                  }
+                },
+                "omni_kv_config": {
+                  "need_send_cache": true
+                },
+                "output_connectors": {
+                  "to_stage_1": "rdma_connector"
+                },
+                "default_sampling_params": {
+                  "temperature": 0.0,
+                  "top_p": 1,
+                  "top_k": -1,
+                  "max_tokens": 8192,
+                  "detokenize": true,
+                  "skip_special_tokens": false,
+                  "stop_token_ids": [
+                    128025
+                  ]
+                }
+              },
+              {
+                "stage_id": 1,
+                "max_num_seqs": 1,
+                "enforce_eager": true,
+                "devices": "2,3",
+                "distributed_executor_backend": "mp",
+                "omni_kv_config": {
+                  "need_recv_cache": true
+                },
+                "parallel_config": {
+                  "tensor_parallel_size": 2,
+                  "vae_patch_parallel_size": 2,
+                  "enable_expert_parallel": true
+                },
+                "input_connectors": {
+                  "from_stage_0": "rdma_connector"
+                },
+                "default_sampling_params": {
+                  "num_inference_steps": 50,
+                  "guidance_scale": 0
+                }
+              }
+            ],
+            "edges": [
+              {
+                "from": 0,
+                "to": 1,
+                "window_size": -1,
+                "max_inflight": 1
+              }
+            ]
+          }
+        },
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "inline_dataset_5_requests",
+        "dataset": "custom",
+        "dataset-path-inline": {
+          "type": "jsonl",
+          "content": [
+            {
+              "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。",
+              "image_urls": [
+                "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png"
+              ],
+              "num_inference_steps": 8,
+              "seed": 42,
+              "guidance_scale": 1.0
             }
-        ]
-    }
+          ]
+        },
+        "task": "ti2i",
+        "bot-task": "think_recaption",
+        "max-concurrency": 16,
+        "num-prompts": 96,
+        "skip-performance-assertion": true,
+        "baseline": {
+          "throughput_qps": 0.1874,
+          "latency_p99": 101.852,
+          "latency_mean": 79.0385,
+          "peak_memory_mb_max": 70000,
+          "peak_memory_mb_mean": 70000
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json

@@ -1,35 +1,47 @@
 [
-    {
-        "test_name": "test_hunyuan_image_tp2_cfgp2",
-        "description": "TP=2 CfgP=2 baseline",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "tencent/HunyuanImage-3.0-Instruct",
-            "serve_args": {
-                "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
-                "tensor-parallel-size": 2,
-                "cfg-parallel-size": 2,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1024x1024_steps8",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1024,
-                "height": 1024,
-                "num-inference-steps": 8,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "skip-performance-assertion": true,
-                "baseline": {
-                    "throughput_qps": 0.21,
-                    "latency_p99": 4.7469,
-                    "peak_memory_mb_max": 101100,
-                    "peak_memory_mb_mean": 101100
-                }
-            }
-        ]
-    }
+  {
+    "test_name": "test_hunyuan_image_tp2_cfgp2",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "diffusion",
+      "local_model"
+    ],
+    "description": "TP=2 CfgP=2 baseline",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "tencent/HunyuanImage-3.0-Instruct",
+      "serve_args": {
+        "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
+        "tensor-parallel-size": 2,
+        "cfg-parallel-size": 2,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1024x1024_steps8",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1024,
+        "height": 1024,
+        "num-inference-steps": 8,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "skip-performance-assertion": true,
+        "baseline": {
+          "throughput_qps": 0.21,
+          "latency_p99": 4.7469,
+          "peak_memory_mb_max": 101100,
+          "peak_memory_mb_mean": 101100
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json

@@ -1,35 +1,47 @@
 [
-    {
-        "test_name": "test_hunyuan_image_tp2_sp2",
-        "description": "TP=2 SP=2 baseline",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "tencent/HunyuanImage-3.0-Instruct",
-            "serve_args": {
-                "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
-                "tensor-parallel-size": 2,
-                "ulysses-degree": 2,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1024x1024_steps8",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1024,
-                "height": 1024,
-                "num-inference-steps": 8,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "skip-performance-assertion": true,
-                "baseline": {
-                    "throughput_qps": 0.20,
-                    "latency_p99": 5.1025,
-                    "peak_memory_mb_max": 97402,
-                    "peak_memory_mb_mean": 97402
-                }
-            }
-        ]
-    }
+  {
+    "test_name": "test_hunyuan_image_tp2_sp2",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "diffusion",
+      "local_model"
+    ],
+    "description": "TP=2 SP=2 baseline",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "tencent/HunyuanImage-3.0-Instruct",
+      "serve_args": {
+        "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
+        "tensor-parallel-size": 2,
+        "ulysses-degree": 2,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1024x1024_steps8",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1024,
+        "height": 1024,
+        "num-inference-steps": 8,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "skip-performance-assertion": true,
+        "baseline": {
+          "throughput_qps": 0.20,
+          "latency_p99": 5.1025,
+          "peak_memory_mb_max": 97402,
+          "peak_memory_mb_mean": 97402
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_hunyuan_image_tp4.json

@@ -1,35 +1,47 @@
 [
-    {
-        "test_name": "test_hunyuan_image_tp4",
-        "description": "TP=4 baseline",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "tencent/HunyuanImage-3.0-Instruct",
-            "serve_args": {
-                "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
-                "tensor-parallel-size": 4,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1024x1024_steps8",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1024,
-                "height": 1024,
-                "num-inference-steps": 8,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "skip-performance-assertion": true,
-                "baseline": {
-                    "throughput_qps": 0.213,
-                    "latency_mean": 4.6953,
-                    "latency_p99": 4.8601,
-                    "peak_memory_mb_max": 56912.0,
-                    "peak_memory_mb_mean": 56912.0
-                }
-            }
-        ]
-    }
+  {
+    "test_name": "test_hunyuan_image_tp4",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "diffusion",
+      "local_model"
+    ],
+    "description": "TP=4 baseline",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "tencent/HunyuanImage-3.0-Instruct",
+      "serve_args": {
+        "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml",
+        "tensor-parallel-size": 4,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1024x1024_steps8",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1024,
+        "height": 1024,
+        "num-inference-steps": 8,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "skip-performance-assertion": true,
+        "baseline": {
+          "throughput_qps": 0.213,
+          "latency_mean": 4.6953,
+          "latency_p99": 4.8601,
+          "peak_memory_mb_max": 56912.0,
+          "peak_memory_mb_mean": 56912.0
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_ltx2_vllm_omni.json

@@ -1,217 +1,273 @@
 [
-    {
-        "test_name": "test_ltx2_baseline_eager",
-        "description": "Single-device baseline with enforce-eager (no torch.compile)",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Lightricks/LTX-2",
-            "serve_args": {
-                "enforce-eager": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "256x256_145f_steps6",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 256,
-                "height": 256,
-                "num-frames": 145,
-                "fps": 24,
-                "num-inference-steps": 6,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            },
-            {
-                "name": "480x768_41f_steps20",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 768,
-                "height": 480,
-                "num-frames": 41,
-                "fps": 24,
-                "num-inference-steps": 20,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            }
-        ]
+  {
+    "test_name": "test_ltx2_baseline_eager",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline with enforce-eager (no torch.compile)",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Lightricks/LTX-2",
+      "serve_args": {
+        "enforce-eager": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-
-    {
-        "test_name": "test_ltx2_torch_compile",
-        "description": "Single-device with torch.compile (default, no enforce-eager)",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Lightricks/LTX-2",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "256x256_145f_steps6",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 256,
-                "height": 256,
-                "num-frames": 145,
-                "fps": 24,
-                "num-inference-steps": 6,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            },
-            {
-                "name": "480x768_41f_steps20",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 768,
-                "height": 480,
-                "num-frames": 41,
-                "fps": 24,
-                "num-inference-steps": 20,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "256x256_145f_steps6",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 256,
+        "height": 256,
+        "num-frames": 145,
+        "fps": 24,
+        "num-inference-steps": 6,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      },
+      {
+        "name": "480x768_41f_steps20",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 768,
+        "height": 480,
+        "num-frames": 41,
+        "fps": 24,
+        "num-inference-steps": 20,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      }
+    ]
+  },
+  {
+    "test_name": "test_ltx2_torch_compile",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device with torch.compile (default, no enforce-eager)",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Lightricks/LTX-2",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-
-    {
-        "test_name": "test_ltx2_cfg2_eager",
-        "description": "CFG-parallel=2 with enforce-eager",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Lightricks/LTX-2",
-            "serve_args": {
-                "cfg-parallel-size": 2,
-                "enforce-eager": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "256x256_145f_steps6",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 256,
-                "height": 256,
-                "num-frames": 145,
-                "fps": 24,
-                "num-inference-steps": 6,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            },
-            {
-                "name": "480x768_41f_steps20",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 768,
-                "height": 480,
-                "num-frames": 41,
-                "fps": 24,
-                "num-inference-steps": 20,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "256x256_145f_steps6",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 256,
+        "height": 256,
+        "num-frames": 145,
+        "fps": 24,
+        "num-inference-steps": 6,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      },
+      {
+        "name": "480x768_41f_steps20",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 768,
+        "height": 480,
+        "num-frames": 41,
+        "fps": 24,
+        "num-inference-steps": 20,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      }
+    ]
+  },
+  {
+    "test_name": "test_ltx2_cfg2_eager",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "CFG-parallel=2 with enforce-eager",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Lightricks/LTX-2",
+      "serve_args": {
+        "cfg-parallel-size": 2,
+        "enforce-eager": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-
-    {
-        "test_name": "test_ltx2_cfg2_compile",
-        "description": "CFG-parallel=2 with torch.compile",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Lightricks/LTX-2",
-            "serve_args": {
-                "cfg-parallel-size": 2,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "256x256_145f_steps6",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 256,
-                "height": 256,
-                "num-frames": 145,
-                "fps": 24,
-                "num-inference-steps": 6,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            },
-            {
-                "name": "480x768_41f_steps20",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 768,
-                "height": 480,
-                "num-frames": 41,
-                "fps": 24,
-                "num-inference-steps": 20,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "256x256_145f_steps6",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 256,
+        "height": 256,
+        "num-frames": 145,
+        "fps": 24,
+        "num-inference-steps": 6,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      },
+      {
+        "name": "480x768_41f_steps20",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 768,
+        "height": 480,
+        "num-frames": 41,
+        "fps": 24,
+        "num-inference-steps": 20,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      }
+    ]
+  },
+  {
+    "test_name": "test_ltx2_cfg2_compile",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "CFG-parallel=2 with torch.compile",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Lightricks/LTX-2",
+      "serve_args": {
+        "cfg-parallel-size": 2,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-
-    {
-        "test_name": "test_ltx2_cache_dit_eager",
-        "description": "CacheDiT with enforce-eager",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Lightricks/LTX-2",
-            "serve_args": {
-                "cache-backend": "cache_dit",
-                "enforce-eager": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "256x256_145f_steps6",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 256,
-                "height": 256,
-                "num-frames": 145,
-                "fps": 24,
-                "num-inference-steps": 6,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            },
-            {
-                "name": "480x768_41f_steps20",
-                "dataset": "random",
-                "task": "t2v",
-                "backend": "v1/videos",
-                "width": 768,
-                "height": 480,
-                "num-frames": 41,
-                "fps": 24,
-                "num-inference-steps": 20,
-                "num-prompts": 3,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true
-            }
-        ]
-    }
+    "benchmark_params": [
+      {
+        "name": "256x256_145f_steps6",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 256,
+        "height": 256,
+        "num-frames": 145,
+        "fps": 24,
+        "num-inference-steps": 6,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      },
+      {
+        "name": "480x768_41f_steps20",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 768,
+        "height": 480,
+        "num-frames": 41,
+        "fps": 24,
+        "num-inference-steps": 20,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      }
+    ]
+  },
+  {
+    "test_name": "test_ltx2_cache_dit_eager",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "CacheDiT with enforce-eager",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Lightricks/LTX-2",
+      "serve_args": {
+        "cache-backend": "cache_dit",
+        "enforce-eager": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "256x256_145f_steps6",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 256,
+        "height": 256,
+        "num-frames": 145,
+        "fps": 24,
+        "num-inference-steps": 6,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      },
+      {
+        "name": "480x768_41f_steps20",
+        "dataset": "random",
+        "task": "t2v",
+        "backend": "v1/videos",
+        "width": 768,
+        "height": 480,
+        "num-frames": 41,
+        "fps": 24,
+        "num-inference-steps": 20,
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json

@@ -1,331 +1,343 @@
 [
-    {
-        "test_name": "test_qwen3_omni_chunk",
-        "server_params": {
-            "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": [
-                "--async-chunk"
-            ]
+  {
+    "test_name": "test_qwen3_omni_chunk",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "omni"
+    ],
+    "server_params": {
+      "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+      "extra_cli_args": [
+        "--async-chunk"
+      ]
+    },
+    "benchmark_params": [
+      {
+        "dataset_name": "random",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          4,
+          16,
+          32,
+          64,
+          128
+        ],
+        "max_concurrency": [
+          1,
+          4,
+          8,
+          16,
+          32
+        ],
+        "random_input_len": 2500,
+        "random_output_len": 900,
+        "ignore_eos": true,
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            135.4379,
+            187.0896,
+            400,
+            435.675,
+            724.6836
+          ],
+          "mean_e2el_ms": [
+            17619.9851,
+            34811.8335,
+            43656.9501,
+            56657.1548,
+            85097.6016
+          ],
+          "mean_audio_rtf": [
+            0.1279,
+            0.2238,
+            0.2735,
+            0.3712,
+            0.5634
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          10
+        ],
+        "request_rate": [
+          0.1
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 1,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "audio": 1
         },
-        "benchmark_params": [
-            {
-                "dataset_name": "random",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    4,
-                    16,
-                    32,
-                    64,
-                    128
-                ],
-                "max_concurrency": [
-                    1,
-                    4,
-                    8,
-                    16,
-                    32
-                ],
-                "random_input_len": 2500,
-                "random_output_len": 900,
-                "ignore_eos": true,
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        135.4379,
-                        187.0896,
-                        400,
-                        435.675,
-                        724.6836
-                    ],
-                    "mean_e2el_ms": [
-                        17619.9851,
-                        34811.8335,
-                        43656.9501,
-                        56657.1548,
-                        85097.6016
-                    ],
-                    "mean_audio_rtf": [
-                        0.1279,
-                        0.2238,
-                        0.2735,
-                        0.3712,
-                        0.5634
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    10
-                ],
-                "request_rate": [
-                    0.1
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 1,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(0, 60, 3)": 1.0
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        340.329
-                    ],
-                    "mean_e2el_ms": [
-                        4305.477
-                    ],
-                    "mean_audio_rtf": [
-                        0.1305
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    40
-                ],
-                "request_rate": [
-                    0.5
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 2,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.5,
-                    "(720, 1280, 2)": 0.5
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        220.0909
-                    ],
-                    "mean_e2el_ms": [
-                        5475.9565
-                    ],
-                    "mean_audio_rtf": [
-                        0.1818
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    100
-                ],
-                "request_rate": [
-                    1.0
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 3,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1,
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.34,
-                    "(720, 1280, 2)": 0.33,
-                    "(0, 60, 3)": 0.33
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        450.0155
-                    ],
-                    "mean_e2el_ms": [
-                        8516.8316
-                    ],
-                    "mean_audio_rtf": [
-                        0.2717
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    4,
-                    16,
-                    32,
-                    64,
-                    128
-                ],
-                "max_concurrency": [
-                    1,
-                    4,
-                    8,
-                    16,
-                    32
-                ],
-                "random_input_len": 2500,
-                "random_output_len": 900,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        78.4278,
-                        195.8037,
-                        333.7318,
-                        561.805,
-                        1038.043
-                    ],
-                    "mean_e2el_ms": [
-                        4288.3029,
-                        7429.9429,
-                        9451.6184,
-                        12607.5292,
-                        18153.2547
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    10
-                ],
-                "request_rate": [
-                    0.1
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "random_mm_base_items_per_request": 1,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(0, 60, 3)": 1.0
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        275.3875
-                    ],
-                    "mean_e2el_ms": [
-                        730.6995
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    40
-                ],
-                "request_rate": [
-                    0.5
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "random_mm_base_items_per_request": 2,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.5,
-                    "(720, 1280, 2)": 0.5
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        166.2201
-                    ],
-                    "mean_e2el_ms": [
-                        673.5789
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    100
-                ],
-                "request_rate": [
-                    1.0
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "random_mm_base_items_per_request": 3,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1,
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.34,
-                    "(720, 1280, 2)": 0.33,
-                    "(0, 60, 3)": 0.33
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        357.7781
-                    ],
-                    "mean_e2el_ms": [
-                        953.3493
-                    ]
-                }
-            }
-        ]
-    }
+        "random_mm_bucket_config": {
+          "(0, 60, 3)": 1.0
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            340.329
+          ],
+          "mean_e2el_ms": [
+            4305.477
+          ],
+          "mean_audio_rtf": [
+            0.1305
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          40
+        ],
+        "request_rate": [
+          0.5
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 2,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.5,
+          "(720, 1280, 2)": 0.5
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            220.0909
+          ],
+          "mean_e2el_ms": [
+            5475.9565
+          ],
+          "mean_audio_rtf": [
+            0.1818
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          1.0
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 3,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1,
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.34,
+          "(720, 1280, 2)": 0.33,
+          "(0, 60, 3)": 0.33
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            450.0155
+          ],
+          "mean_e2el_ms": [
+            8516.8316
+          ],
+          "mean_audio_rtf": [
+            0.2717
+          ]
+        }
+      },
+      {
+        "dataset_name": "random",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          4,
+          16,
+          32,
+          64,
+          128
+        ],
+        "max_concurrency": [
+          1,
+          4,
+          8,
+          16,
+          32
+        ],
+        "random_input_len": 2500,
+        "random_output_len": 900,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            78.4278,
+            195.8037,
+            333.7318,
+            561.805,
+            1038.043
+          ],
+          "mean_e2el_ms": [
+            4288.3029,
+            7429.9429,
+            9451.6184,
+            12607.5292,
+            18153.2547
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          10
+        ],
+        "request_rate": [
+          0.1
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "random_mm_base_items_per_request": 1,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(0, 60, 3)": 1.0
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            275.3875
+          ],
+          "mean_e2el_ms": [
+            730.6995
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          40
+        ],
+        "request_rate": [
+          0.5
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "random_mm_base_items_per_request": 2,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.5,
+          "(720, 1280, 2)": 0.5
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            166.2201
+          ],
+          "mean_e2el_ms": [
+            673.5789
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          1.0
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "random_mm_base_items_per_request": 3,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1,
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.34,
+          "(720, 1280, 2)": 0.33,
+          "(0, 60, 3)": 0.33
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            357.7781
+          ],
+          "mean_e2el_ms": [
+            953.3493
+          ]
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json

@@ -1,115 +1,127 @@
 [
-    {
-        "test_name": "test_qwen3_omni_3gpu_replica2",
-        "server_params": {
-            "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "stage_overrides": {
-                "1": {"num_replicas": 2, "devices": "1,2"},
-                "2": {"num_replicas": 2, "devices": "1,2"}
-            },
-            "extra_cli_args": [
-                "--async-chunk"
-            ]
+  {
+    "test_name": "test_qwen3_omni_3gpu_replica2",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 3
+        }
+      },
+      "full_model",
+      "omni"
+    ],
+    "server_params": {
+      "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+      "stage_overrides": {
+          "1": {"num_replicas": 2, "devices": "1,2"},
+          "2": {"num_replicas": 2, "devices": "1,2"}
+      },
+      "extra_cli_args": [
+        "--async-chunk"
+      ]
+    },
+    "benchmark_params": [
+      {
+        "dataset_name": "random",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [128, 128, 128, 128],
+        "max_concurrency": [8, 16, 24, 32],
+        "random_input_len": 2500,
+        "random_output_len": 900,
+        "ignore_eos": true,
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+            "mean_ttft_ms": [203, 283, 470, 869],
+            "mean_tpot_ms": [28, 36, 41, 45],
+            "mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
+            "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [10],
+        "request_rate": [0.1],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 1,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "audio": 1
         },
-        "benchmark_params": [
-            {
-                "dataset_name": "random",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [128, 128, 128, 128],
-                "max_concurrency": [8, 16, 24, 32],
-                "random_input_len": 2500,
-                "random_output_len": 900,
-                "ignore_eos": true,
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [203, 283, 470, 869],
-                    "mean_tpot_ms": [28, 36, 41, 45],
-                    "mean_audio_ttfp_ms": [861, 1200, 1528, 2100],
-                    "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 1,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(0, 60, 3)": 1.0
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [377],
-                    "mean_audio_ttfp_ms": [684],
-                    "mean_audio_rtf": [0.14]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 2,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.5,
-                    "(720, 1280, 2)": 0.5
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [228],
-                    "mean_audio_ttfp_ms": [686],
-                    "mean_audio_rtf": [0.16]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 3,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1,
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.34,
-                    "(720, 1280, 2)": 0.33,
-                    "(0, 60, 3)": 0.33
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [522],
-                    "mean_audio_ttfp_ms": [1119],
-                    "mean_audio_rtf": [0.21]
-                }
-            }
-        ]
-    }
+        "random_mm_bucket_config": {
+          "(0, 60, 3)": 1.0
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+            "mean_ttft_ms": [377],
+            "mean_audio_ttfp_ms": [684],
+            "mean_audio_rtf": [0.14]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [40],
+        "request_rate": [0.5],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 2,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.5,
+          "(720, 1280, 2)": 0.5
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+            "mean_ttft_ms": [228],
+            "mean_audio_ttfp_ms": [686],
+            "mean_audio_rtf": [0.16]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [100],
+        "request_rate": [1.0],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 3,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1,
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.34,
+          "(720, 1280, 2)": 0.33,
+          "(0, 60, 3)": 0.33
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+            "mean_ttft_ms": [522],
+            "mean_audio_ttfp_ms": [1119],
+            "mean_audio_rtf": [0.21]
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json

@@ -1,170 +1,182 @@
 [
-    {
-        "test_name": "test_qwen3_omni",
-        "server_params": {
-            "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": [
-                "--no-async-chunk"
-            ]
+  {
+    "test_name": "test_qwen3_omni",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "omni"
+    ],
+    "server_params": {
+      "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+      "extra_cli_args": [
+        "--no-async-chunk"
+      ]
+    },
+    "benchmark_params": [
+      {
+        "dataset_name": "random",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          4,
+          16,
+          32,
+          64,
+          128
+        ],
+        "max_concurrency": [
+          1,
+          4,
+          8,
+          16,
+          32
+        ],
+        "random_input_len": 2500,
+        "random_output_len": 900,
+        "ignore_eos": true,
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            109.1218,
+            247.4411,
+            354.9384,
+            447.7885,
+            694.7487
+          ],
+          "mean_e2el_ms": [
+            22413.667,
+            30808.9715,
+            35677.844,
+            46345.8978,
+            68350.4853
+          ],
+          "mean_audio_rtf": [
+            0.1695,
+            0.2033,
+            0.2467,
+            0.3234,
+            0.4438
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          10
+        ],
+        "request_rate": [
+          0.1
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 1,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "audio": 1
         },
-        "benchmark_params": [
-            {
-                "dataset_name": "random",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    4,
-                    16,
-                    32,
-                    64,
-                    128
-                ],
-                "max_concurrency": [
-                    1,
-                    4,
-                    8,
-                    16,
-                    32
-                ],
-                "random_input_len": 2500,
-                "random_output_len": 900,
-                "ignore_eos": true,
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        109.1218,
-                        247.4411,
-                        354.9384,
-                        447.7885,
-                        694.7487
-                    ],
-                    "mean_e2el_ms": [
-                        22413.667,
-                        30808.9715,
-                        35677.844,
-                        46345.8978,
-                        68350.4853
-                    ],
-                    "mean_audio_rtf": [
-                        0.1695,
-                        0.2033,
-                        0.2467,
-                        0.3234,
-                        0.4438
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    10
-                ],
-                "request_rate": [
-                    0.1
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 1,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(0, 60, 3)": 1.0
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        323.3573
-                    ],
-                    "mean_e2el_ms": [
-                        4623.4173
-                    ],
-                    "mean_audio_rtf": [
-                        0.1406
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    40
-                ],
-                "request_rate": [
-                    0.5
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 2,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.5,
-                    "(720, 1280, 2)": 0.5
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        215.4502
-                    ],
-                    "mean_e2el_ms": [
-                        4924.149
-                    ],
-                    "mean_audio_rtf": [
-                        0.1637
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "openai-chat-omni",
-                "endpoint": "/v1/chat/completions",
-                "num_prompts": [
-                    100
-                ],
-                "request_rate": [
-                    1.0
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "random_mm_base_items_per_request": 3,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1,
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.34,
-                    "(720, 1280, 2)": 0.33,
-                    "(0, 60, 3)": 0.33
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        442.5815
-                    ],
-                    "mean_e2el_ms": [
-                        6441.0986
-                    ],
-                    "mean_audio_rtf": [
-                        0.2091
-                    ]
-                }
-            }
-        ]
-    }
+        "random_mm_bucket_config": {
+          "(0, 60, 3)": 1.0
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            323.3573
+          ],
+          "mean_e2el_ms": [
+            4623.4173
+          ],
+          "mean_audio_rtf": [
+            0.1406
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          40
+        ],
+        "request_rate": [
+          0.5
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 2,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.5,
+          "(720, 1280, 2)": 0.5
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            215.4502
+          ],
+          "mean_e2el_ms": [
+            4924.149
+          ],
+          "mean_audio_rtf": [
+            0.1637
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "openai-chat-omni",
+        "endpoint": "/v1/chat/completions",
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          1.0
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "random_mm_base_items_per_request": 3,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1,
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.34,
+          "(720, 1280, 2)": 0.33,
+          "(0, 60, 3)": 0.33
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "mean_ttft_ms": [
+            442.5815
+          ],
+          "mean_e2el_ms": [
+            6441.0986
+          ],
+          "mean_audio_rtf": [
+            0.2091
+          ]
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json

@@ -1,180 +1,192 @@
 [
-    {
-        "test_name": "test_qwen3_omni_vllm_text",
-        "server_params": {
-            "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "use_omni": false,
-            "extra_cli_args": [
-                "--no-enable-prefix-caching",
-                "--skip-mm-profiling"
-            ]
+  {
+    "test_name": "test_qwen3_omni_vllm_text",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "omni"
+    ],
+    "server_params": {
+      "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+      "use_omni": false,
+      "extra_cli_args": [
+        "--no-enable-prefix-caching",
+        "--skip-mm-profiling"
+      ]
+    },
+    "benchmark_params": [
+      {
+        "dataset_name": "random",
+        "backend": "vllm",
+        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+        "endpoint": "/v1/completions",
+        "num_prompts": [
+          4,
+          16,
+          32,
+          64,
+          128
+        ],
+        "max_concurrency": [
+          1,
+          4,
+          8,
+          16,
+          32
+        ],
+        "random_input_len": 2500,
+        "random_output_len": 900,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
         },
-        "benchmark_params": [
-            {
-                "dataset_name": "random",
-                "backend": "vllm",
-                "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-                "endpoint": "/v1/completions",
-                "num_prompts": [
-                    4,
-                    16,
-                    32,
-                    64,
-                    128
-                ],
-                "max_concurrency": [
-                    1,
-                    4,
-                    8,
-                    16,
-                    32
-                ],
-                "random_input_len": 2500,
-                "random_output_len": 900,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        69.1276,
-                        255.1882,
-                        310.639,
-                        422.0337,
-                        532.0582
-                    ],
-                    "mean_e2el_ms": [
-                        4382.7211,
-                        7510.3815,
-                        9440.4854,
-                        12553.0186,
-                        16932.6065
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "vllm",
-                "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-                "endpoint": "/v1/completions",
-                "num_prompts": [
-                    10
-                ],
-                "request_rate": [
-                    0.1
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "random_mm_base_items_per_request": 1,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(0, 60, 3)": 1.0
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        26.3381
-                    ],
-                    "mean_e2el_ms": [
-                        485.2436
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "vllm",
-                "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-                "endpoint": "/v1/completions",
-                "num_prompts": [
-                    40
-                ],
-                "request_rate": [
-                    0.5
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "random_mm_base_items_per_request": 2,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.5,
-                    "(720, 1280, 2)": 0.5
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        26.4517
-                    ],
-                    "mean_e2el_ms": [
-                        517.2017
-                    ]
-                }
-            },
-            {
-                "dataset_name": "random-mm",
-                "backend": "vllm",
-                "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-                "endpoint": "/v1/completions",
-                "num_prompts": [
-                    100
-                ],
-                "request_rate": [
-                    1.0
-                ],
-                "random_input_len": 100,
-                "random_output_len": 100,
-                "random_range_ratio": 0.0,
-                "ignore_eos": true,
-                "extra_body": {
-                    "modalities": [
-                        "text"
-                    ]
-                },
-                "random_mm_base_items_per_request": 3,
-                "random_mm_num_mm_items_range_ratio": 0.5,
-                "random_mm_limit_mm_per_prompt": {
-                    "image": 1,
-                    "video": 1,
-                    "audio": 1
-                },
-                "random_mm_bucket_config": {
-                    "(256, 256, 1)": 0.34,
-                    "(720, 1280, 2)": 0.33,
-                    "(0, 60, 3)": 0.33
-                },
-                "percentile-metrics": "ttft,tpot,itl,e2el",
-                "baseline": {
-                    "mean_ttft_ms": [
-                        27.6972
-                    ],
-                    "mean_e2el_ms": [
-                        561.3684
-                    ]
-                }
-            }
-        ]
-    }
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            69.1276,
+            255.1882,
+            310.639,
+            422.0337,
+            532.0582
+          ],
+          "mean_e2el_ms": [
+            4382.7211,
+            7510.3815,
+            9440.4854,
+            12553.0186,
+            16932.6065
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "vllm",
+        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+        "endpoint": "/v1/completions",
+        "num_prompts": [
+          10
+        ],
+        "request_rate": [
+          0.1
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "random_mm_base_items_per_request": 1,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(0, 60, 3)": 1.0
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            26.3381
+          ],
+          "mean_e2el_ms": [
+            485.2436
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "vllm",
+        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+        "endpoint": "/v1/completions",
+        "num_prompts": [
+          40
+        ],
+        "request_rate": [
+          0.5
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "random_mm_base_items_per_request": 2,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.5,
+          "(720, 1280, 2)": 0.5
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            26.4517
+          ],
+          "mean_e2el_ms": [
+            517.2017
+          ]
+        }
+      },
+      {
+        "dataset_name": "random-mm",
+        "backend": "vllm",
+        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+        "endpoint": "/v1/completions",
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          1.0
+        ],
+        "random_input_len": 100,
+        "random_output_len": 100,
+        "random_range_ratio": 0.0,
+        "ignore_eos": true,
+        "extra_body": {
+          "modalities": [
+            "text"
+          ]
+        },
+        "random_mm_base_items_per_request": 3,
+        "random_mm_num_mm_items_range_ratio": 0.5,
+        "random_mm_limit_mm_per_prompt": {
+          "image": 1,
+          "video": 1,
+          "audio": 1
+        },
+        "random_mm_bucket_config": {
+          "(256, 256, 1)": 0.34,
+          "(720, 1280, 2)": 0.33,
+          "(0, 60, 3)": 0.33
+        },
+        "percentile-metrics": "ttft,tpot,itl,e2el",
+        "baseline": {
+          "mean_ttft_ms": [
+            27.6972
+          ],
+          "mean_e2el_ms": [
+            561.3684
+          ]
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json

@@ -1,149 +1,185 @@
 [
-    {
-        "test_name": "test_qwen_image_edit_2511_single_device",
-        "description": "Single-device baseline (two input images)",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Edit-2511",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20_i2i_2img",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 2,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0721,
-                    "latency_mean": 13.8398,
-                    "peak_memory_mb_max": 54731.4,
-                    "peak_memory_mb_mean": 54731.4
-                }
-            },
-            {
-                "name": "1536x1536_steps35_i2i_2img",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 2,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0186,
-                    "latency_mean": 53.6942,
-                    "peak_memory_mb_max": 64402.4,
-                    "peak_memory_mb_mean": 64402.4
-                }
-            }
-        ]
+  {
+    "test_name": "test_qwen_image_edit_2511_single_device",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline (two input images)",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Edit-2511",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4",
-        "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Edit-2511",
-            "serve_args": {
-                "ulysses-degree": 2,
-                "cfg-parallel-size": 2,
-                "vae-patch-parallel-size": 4,
-                "vae-use-tiling": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1536x1536_steps35_i2i_2img",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 2,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0538,
-                    "latency_mean": 18.5591,
-                    "peak_memory_mb_max": 53485.0,
-                    "peak_memory_mb_mean": 53485.0
-                }
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20_i2i_2img",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 2,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0721,
+          "latency_mean": 13.8398,
+          "peak_memory_mb_max": 54731.4,
+          "peak_memory_mb_mean": 54731.4
+        }
+      },
+      {
+        "name": "1536x1536_steps35_i2i_2img",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 2,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0186,
+          "latency_mean": 53.6942,
+          "peak_memory_mb_max": 64402.4,
+          "peak_memory_mb_mean": 64402.4
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Edit-2511",
+      "serve_args": {
+        "ulysses-degree": 2,
+        "cfg-parallel-size": 2,
+        "vae-patch-parallel-size": 4,
+        "vae-use-tiling": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit",
-        "description": "Ulysses SP=2 + CFG=2 + CacheDiT",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Edit-2511",
-            "serve_args": {
-                "ulysses-degree": 2,
-                "cfg-parallel-size": 2,
-                "cache-backend": "cache_dit",
-                "cache-config": {
-                    "Fn_compute_blocks": 1,
-                    "Bn_compute_blocks": 0,
-                    "max_warmup_steps": 4,
-                    "residual_diff_threshold": 0.24,
-                    "max_continuous_cached_steps": 3,
-                    "enable_taylorseer": false,
-                    "taylorseer_order": 1,
-                    "scm_steps_mask_policy": null,
-                    "scm_steps_policy": "dynamic"
-                },
-                "enable-diffusion-pipeline-profiler": true
-            }
+    "benchmark_params": [
+      {
+        "name": "1536x1536_steps35_i2i_2img",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 2,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0538,
+          "latency_mean": 18.5591,
+          "peak_memory_mb_max": 53485.0,
+          "peak_memory_mb_mean": 53485.0
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Ulysses SP=2 + CFG=2 + CacheDiT",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Edit-2511",
+      "serve_args": {
+        "ulysses-degree": 2,
+        "cfg-parallel-size": 2,
+        "cache-backend": "cache_dit",
+        "cache-config": {
+          "Fn_compute_blocks": 1,
+          "Bn_compute_blocks": 0,
+          "max_warmup_steps": 4,
+          "residual_diff_threshold": 0.24,
+          "max_continuous_cached_steps": 3,
+          "enable_taylorseer": false,
+          "taylorseer_order": 1,
+          "scm_steps_mask_policy": null,
+          "scm_steps_policy": "dynamic"
         },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20_i2i_2img",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 2,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.2776,
-                    "latency_mean": 3.5938,
-                    "peak_memory_mb_max": 55214.0,
-                    "peak_memory_mb_mean": 55214.0
-                }
-            },
-            {
-                "name": "1536x1536_steps35_i2i_2img",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 2,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.1051,
-                    "latency_mean": 9.4918,
-                    "peak_memory_mb_max": 64265.6,
-                    "peak_memory_mb_mean": 64265.6
-                }
-            }
-        ]
-    }
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20_i2i_2img",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 2,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.2776,
+          "latency_mean": 3.5938,
+          "peak_memory_mb_max": 55214.0,
+          "peak_memory_mb_mean": 55214.0
+        }
+      },
+      {
+        "name": "1536x1536_steps35_i2i_2img",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 2,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.1051,
+          "latency_mean": 9.4918,
+          "peak_memory_mb_max": 64265.6,
+          "peak_memory_mb_mean": 64265.6
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json

@@ -1,144 +1,180 @@
 [
-    {
-        "test_name": "test_qwen_image_edit_single_device",
-        "description": "Single-device baseline",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Edit",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.1129,
-                    "latency_mean": 8.8352,
-                    "peak_memory_mb_max": 55356.5,
-                    "peak_memory_mb_mean": 55356.5
-                }
-            },
-            {
-                "name": "1536x1536_steps35_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0240,
-                    "latency_mean": 41.5686,
-                    "peak_memory_mb_max": 62762.7,
-                    "peak_memory_mb_mean": 62762.7
-                }
-            }
-        ]
+  {
+    "test_name": "test_qwen_image_edit_single_device",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Edit",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4",
-        "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Edit",
-            "serve_args": {
-                "ulysses-degree": 2,
-                "cfg-parallel-size": 2,
-                "vae-patch-parallel-size": 4,
-                "vae-use-tiling": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1536x1536_steps35_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0672,
-                    "latency_mean": 14.8517,
-                    "peak_memory_mb_max": 53931.5,
-                    "peak_memory_mb_mean": 53931.5
-                }
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.1129,
+          "latency_mean": 8.8352,
+          "peak_memory_mb_max": 55356.5,
+          "peak_memory_mb_mean": 55356.5
+        }
+      },
+      {
+        "name": "1536x1536_steps35_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0240,
+          "latency_mean": 41.5686,
+          "peak_memory_mb_max": 62762.7,
+          "peak_memory_mb_mean": 62762.7
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Edit",
+      "serve_args": {
+        "ulysses-degree": 2,
+        "cfg-parallel-size": 2,
+        "vae-patch-parallel-size": 4,
+        "vae-use-tiling": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit",
-        "description": "Ulysses SP=2 + CFG=2 + CacheDiT",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Edit",
-            "serve_args": {
-                "ulysses-degree": 2,
-                "cfg-parallel-size": 2,
-                "cache-backend": "cache_dit",
-                "cache-config": {
-                    "Fn_compute_blocks": 1,
-                    "Bn_compute_blocks": 0,
-                    "max_warmup_steps": 4,
-                    "residual_diff_threshold": 0.24,
-                    "max_continuous_cached_steps": 3,
-                    "enable_taylorseer": false,
-                    "taylorseer_order": 1,
-                    "scm_steps_mask_policy": null,
-                    "scm_steps_policy": "dynamic"
-                },
-                "enable-diffusion-pipeline-profiler": true
-            }
+    "benchmark_params": [
+      {
+        "name": "1536x1536_steps35_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0672,
+          "latency_mean": 14.8517,
+          "peak_memory_mb_max": 53931.5,
+          "peak_memory_mb_mean": 53931.5
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Ulysses SP=2 + CFG=2 + CacheDiT",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Edit",
+      "serve_args": {
+        "ulysses-degree": 2,
+        "cfg-parallel-size": 2,
+        "cache-backend": "cache_dit",
+        "cache-config": {
+          "Fn_compute_blocks": 1,
+          "Bn_compute_blocks": 0,
+          "max_warmup_steps": 4,
+          "residual_diff_threshold": 0.24,
+          "max_continuous_cached_steps": 3,
+          "enable_taylorseer": false,
+          "taylorseer_order": 1,
+          "scm_steps_mask_policy": null,
+          "scm_steps_policy": "dynamic"
         },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.2879,
-                    "latency_mean": 3.4701,
-                    "peak_memory_mb_max": 55525.6,
-                    "peak_memory_mb_mean": 55525.6
-                }
-            },
-            {
-                "name": "1536x1536_steps35_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.1113,
-                    "latency_mean": 8.9656,
-                    "peak_memory_mb_max": 63758.3,
-                    "peak_memory_mb_mean": 63758.3
-                }
-            }
-        ]
-    }
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.2879,
+          "latency_mean": 3.4701,
+          "peak_memory_mb_max": 55525.6,
+          "peak_memory_mb_mean": 55525.6
+        }
+      },
+      {
+        "name": "1536x1536_steps35_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.1113,
+          "latency_mean": 8.9656,
+          "peak_memory_mb_max": 63758.3,
+          "peak_memory_mb_mean": 63758.3
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json

@@ -1,49 +1,61 @@
 [
-    {
-        "test_name": "test_qwen_image_layered_single_device",
-        "description": "Single-device baseline",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image-Layered",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "640x640_steps20_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 640,
-                "height": 640,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0703,
-                    "latency_mean": 14.208,
-                    "peak_memory_mb_max": 60543.5,
-                    "peak_memory_mb_mean": 60543.5
-                }
-            },
-            {
-                "name": "1024x1024_steps35_i2i",
-                "dataset": "random",
-                "task": "i2i",
-                "width": 1024,
-                "height": 1024,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0417,
-                    "latency_mean": 23.914,
-                    "peak_memory_mb_max": 60543.5,
-                    "peak_memory_mb_mean": 60543.5
-                }
-            }
-        ]
-    }
+  {
+    "test_name": "test_qwen_image_layered_single_device",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image-Layered",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "640x640_steps20_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 640,
+        "height": 640,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0703,
+          "latency_mean": 14.208,
+          "peak_memory_mb_max": 60543.5,
+          "peak_memory_mb_mean": 60543.5
+        }
+      },
+      {
+        "name": "1024x1024_steps35_i2i",
+        "dataset": "random",
+        "task": "i2i",
+        "width": 1024,
+        "height": 1024,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0417,
+          "latency_mean": 23.914,
+          "peak_memory_mb_max": 60543.5,
+          "peak_memory_mb_mean": 60543.5
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_qwen_image_vllm_omni.json

@@ -1,219 +1,279 @@
 [
-    {
-        "test_name": "test_qwen_image_single_device",
-        "description": "Single-device baseline (no parallelism)",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.4153,
-                    "latency_mean": 2.4065,
-                    "peak_memory_mb_mean": 52289.9
-                }
-            },
-            {
-                "name": "1536x1536_steps35",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0423,
-                    "latency_mean": 23.5991,
-                    "peak_memory_mb_mean": 61360.5
-                }
-            }
-        ]
+  {
+    "test_name": "test_qwen_image_single_device",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline (no parallelism)",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_single_device_step_execution",
-        "description": "Single-device baseline (no parallelism) with step execution",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true,
-                "step-execution": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.4175,
-                    "latency_mean": 2.3953,
-                    "peak_memory_mb_mean": 56000
-                }
-            },
-            {
-                "name": "1536x1536_steps35",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.0423,
-                    "latency_mean": 23.5768,
-                    "peak_memory_mb_mean": 65000
-                }
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.4153,
+          "latency_mean": 2.4065,
+          "peak_memory_mb_mean": 52289.9
+        }
+      },
+      {
+        "name": "1536x1536_steps35",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0423,
+          "latency_mean": 23.5991,
+          "peak_memory_mb_mean": 61360.5
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_single_device_step_execution",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline (no parallelism) with step execution",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true,
+        "step-execution": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4",
-        "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image",
-            "serve_args": {
-                "ulysses-degree": 2,
-                "cfg-parallel-size": 2,
-                "vae-patch-parallel-size": 4,
-                "vae-use-tiling": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "1536x1536_steps35",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.1214,
-                    "latency_mean": 8.2211,
-                    "peak_memory_mb_mean": 51704.7
-                }
-            }
-        ]
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.4175,
+          "latency_mean": 2.3953,
+          "peak_memory_mb_mean": 56000
+        }
+      },
+      {
+        "name": "1536x1536_steps35",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.0423,
+          "latency_mean": 23.5768,
+          "peak_memory_mb_mean": 65000
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image",
+      "serve_args": {
+        "ulysses-degree": 2,
+        "cfg-parallel-size": 2,
+        "vae-patch-parallel-size": 4,
+        "vae-use-tiling": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit",
-        "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image",
-            "serve_args": {
-                "ulysses-degree": 2,
-                "cfg-parallel-size": 2,
-                "cache-backend": "cache_dit",
-                "cache-config": {
-                    "Fn_compute_blocks": 1,
-                    "Bn_compute_blocks": 0,
-                    "max_warmup_steps": 4,
-                    "residual_diff_threshold": 0.24,
-                    "max_continuous_cached_steps": 3,
-                    "enable_taylorseer": false,
-                    "taylorseer_order": 1,
-                    "scm_steps_mask_policy": null,
-                    "scm_steps_policy": "dynamic"
-                },
-                "enable-diffusion-pipeline-profiler": true
-            }
+    "benchmark_params": [
+      {
+        "name": "1536x1536_steps35",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.1214,
+          "latency_mean": 8.2211,
+          "peak_memory_mb_mean": 51704.7
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image",
+      "serve_args": {
+        "ulysses-degree": 2,
+        "cfg-parallel-size": 2,
+        "cache-backend": "cache_dit",
+        "cache-config": {
+          "Fn_compute_blocks": 1,
+          "Bn_compute_blocks": 0,
+          "max_warmup_steps": 4,
+          "residual_diff_threshold": 0.24,
+          "max_continuous_cached_steps": 3,
+          "enable_taylorseer": false,
+          "taylorseer_order": 1,
+          "scm_steps_mask_policy": null,
+          "scm_steps_policy": "dynamic"
         },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.559,
-                    "latency_mean": 1.794,
-                    "peak_memory_mb_mean": 52397.6775
-                }
-            },
-            {
-                "name": "1536x1536_steps35",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 1536,
-                "height": 1536,
-                "num-inference-steps": 35,
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": 0.1918,
-                    "latency_mean": 5.2022,
-                    "peak_memory_mb_mean": 61468.8
-                }
-            }
-        ]
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_qwen_image_single_device_step_execution_high_concurrency",
-        "description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Qwen/Qwen-Image",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true,
-                "step-execution": true,
-                "max-num-seqs": 8
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "512x512_steps20_high_concurrency",
-                "dataset": "random",
-                "task": "t2i",
-                "width": 512,
-                "height": 512,
-                "num-inference-steps": 20,
-                "num-prompts": [20, 40, 80, 160],
-                "max-concurrency": [1, 2, 4, 8],
-                "warmup-requests": 8,
-                "warmup-concurrency": 8,
-                "warmup-num-inference-steps": 20,
-                "enable-negative-prompt": true,
-                "baseline": {
-                    "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
-                    "latency_mean": [2.4215, 2.992, 5.9293, 9.656],
-                    "peak_memory_mb_mean": [67000, 67000, 67000, 67000]
-                }
-            }
-        ]
-    }
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.559,
+          "latency_mean": 1.794,
+          "peak_memory_mb_mean": 52397.6775
+        }
+      },
+      {
+        "name": "1536x1536_steps35",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 1536,
+        "height": 1536,
+        "num-inference-steps": 35,
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "enable-negative-prompt": true,
+        "baseline": {
+          "throughput_qps": 0.1918,
+          "latency_mean": 5.2022,
+          "peak_memory_mb_mean": 61468.8
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_qwen_image_single_device_step_execution_high_concurrency",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Qwen/Qwen-Image",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true,
+        "step-execution": true,
+        "max-num-seqs": 8
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "512x512_steps20_high_concurrency",
+        "dataset": "random",
+        "task": "t2i",
+        "width": 512,
+        "height": 512,
+        "num-inference-steps": 20,
+        "num-prompts": [20, 40, 80, 160],
+        "max-concurrency": [1, 2, 4, 8],
+        "warmup-requests": 8,
+        "warmup-concurrency": 8,
+        "warmup-num-inference-steps": 20,
+        "enable-negative-prompt": true,
+        "baseline": {
+            "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
+            "latency_mean": [2.4215, 2.992, 5.9293, 9.656],
+            "peak_memory_mb_mean": [67000, 67000, 67000, 67000]
+        }
+      }
+    ]
+  }
 ]

tests/dfx/perf/tests/test_runner_metadata.py

@@ -86,3 +86,202 @@ def test_enabled_false_entry_is_skipped():
     # Only the enabled=True entry should appear
     assert len(params) == 1
     assert params[0].get("task") == "voice_clone"
+
+
+def test_resolve_pytest_marks_hardware_dict_with_extra():
+    from tests.dfx.conftest import resolve_pytest_marks
+
+    marks = resolve_pytest_marks(
+        [
+            {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}},
+            "full_model",
+            "diffusion",
+            "local_model",
+        ]
+    )
+    names = {m.name for m in marks}
+    assert "H100" in names
+    assert "cuda" in names
+    assert "gpu" in names
+    assert "distributed_cuda" in names
+    assert "full_model" in names
+    assert "diffusion" in names
+    assert "local_model" in names
+
+
+def test_resolve_pytest_marks_rejects_legacy_object_format():
+    from tests.dfx.conftest import resolve_pytest_marks
+
+    with pytest.raises(ValueError, match="mark must be a list"):
+        resolve_pytest_marks(
+            {
+                "hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1},
+                "marks": ["full_model"],
+            }
+        )
+
+
+def test_extract_mark_resource_label():
+    from tests.dfx.conftest import extract_mark_resource_label
+
+    assert extract_mark_resource_label(None) == "na"
+    assert (
+        extract_mark_resource_label(
+            [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "full_model",
+            ]
+        )
+        == "H100"
+    )
+    assert (
+        extract_mark_resource_label(
+            [
+                {
+                    "hardware_marks": {
+                        "res": {"cuda": "H100", "rocm": "MI325"},
+                        "num_cards": 2,
+                    }
+                },
+                "diffusion",
+            ]
+        )
+        == "H100-MI325"
+    )
+
+
+def test_resource_label_for_filename():
+    from tests.dfx.conftest import resource_label_for_filename
+
+    assert resource_label_for_filename("H100") == ""
+    assert resource_label_for_filename("L4") == "L4"
+    assert resource_label_for_filename("910B") == "910B"
+    assert resource_label_for_filename("na") == "na"
+
+
+def test_hardware_json_value():
+    from tests.dfx.conftest import hardware_json_value
+
+    assert hardware_json_value("H100") == "H100"
+    assert hardware_json_value("na") == ""
+    assert hardware_json_value(None) == ""
+
+
+def test_extract_configs_resource_label(monkeypatch):
+    from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
+
+    monkeypatch.setattr(
+        "tests.dfx.conftest._read_runtime_device_name",
+        lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
+    )
+    get_runtime_resource_label(refresh=True)
+    assert extract_configs_resource_label([]) == "H100"
+    get_runtime_resource_label(refresh=True)
+    monkeypatch.setattr(
+        "tests.dfx.conftest._read_runtime_device_name",
+        lambda *, device_id=0: "Ascend910B2",
+    )
+    assert get_runtime_resource_label(refresh=True) == "910B"
+
+
+def test_load_benchmark_configs_from_dir(tmp_path):
+    from tests.dfx.conftest import load_benchmark_configs
+
+    (tmp_path / "a.json").write_text(
+        json.dumps([{"test_name": "test_a", "server_params": {"model": "m/a"}, "benchmark_params": []}]),
+        encoding="utf-8",
+    )
+    (tmp_path / "b.json").write_text(
+        json.dumps([{"test_name": "test_b", "server_params": {"model": "m/b"}, "benchmark_params": []}]),
+        encoding="utf-8",
+    )
+    configs = load_benchmark_configs(config_dir=tmp_path)
+    assert [c["test_name"] for c in configs] == ["test_a", "test_b"]
+
+
+def test_create_unique_server_pytest_params_applies_marks(tmp_path):
+    from tests.dfx.conftest import create_unique_server_pytest_params
+
+    configs = [
+        {
+            "test_name": "test_with_mark",
+            "mark": [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "full_model",
+            ],
+            "server_params": {"model": "some/model"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}],
+        },
+        {
+            "test_name": "test_without_mark",
+            "server_params": {"model": "other/model"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}],
+        },
+    ]
+    params = create_unique_server_pytest_params(configs, tmp_path)
+    by_id = {p.id: p for p in params}
+    assert len(by_id["test_with_mark"].values) == 1
+    assert isinstance(by_id["test_with_mark"].values[0], tuple)
+    assert any(m.name == "H100" for m in by_id["test_with_mark"].marks)
+    assert not any(m.name == "H100" for m in by_id["test_without_mark"].marks)
+
+
+def test_is_diffusion_perf_config():
+    from tests.dfx.conftest import is_diffusion_perf_config
+
+    assert not is_diffusion_perf_config(
+        {"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]}
+    )
+    assert is_diffusion_perf_config(
+        {
+            "test_name": "diff_a",
+            "server_type": "vllm-omni",
+            "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"],
+        }
+    )
+
+
+def test_benchmark_param_id_suffix_from_task_eval_phase():
+    from tests.dfx.conftest import _unique_benchmark_param_id_suffixes
+
+    params = [
+        {"task": "default_voice", "eval_phase": "latency"},
+        {"task": "default_voice", "eval_phase": "throughput"},
+    ]
+    assert _unique_benchmark_param_id_suffixes(params) == [
+        "default_voice_latency",
+        "default_voice_throughput",
+    ]
+
+
+def test_create_paired_omni_benchmark_pytest_params(tmp_path):
+    from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params
+
+    configs = [
+        {
+            "test_name": "test_omni",
+            "mark": [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "omni",
+            ],
+            "server_params": {"model": "m/omni"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}, {"name": "p1", "num_prompts": 2}],
+        },
+        {
+            "test_name": "test_tts",
+            "mark": [
+                {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
+                "tts",
+            ],
+            "server_params": {"model": "m/tts"},
+            "benchmark_params": [{"name": "p0", "num_prompts": 1}],
+        },
+    ]
+    params = create_paired_omni_benchmark_pytest_params(configs, tmp_path)
+    by_id = {p.id: p for p in params}
+    assert set(by_id) == {"test_omni-p0", "test_omni-p1", "test_tts-p0"}
+    omni_row, bench_row = by_id["test_tts-p0"].values
+    assert omni_row[0] == "test_tts"
+    assert bench_row == ("test_tts", 0)
+    assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
+    assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)

tests/dfx/perf/tests/test_tts.json

@@ -1,6 +1,18 @@
 [
   {
     "test_name": "test_qwen3_tts_base",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "tts"
+    ],
     "server_params": {
       "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
       "stage_overrides": {
@@ -98,6 +110,18 @@
   },
   {
     "test_name": "test_qwen3_tts_customvoice",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "tts"
+    ],
     "server_params": {
       "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
       "stage_overrides": {
@@ -342,6 +366,18 @@
   },
   {
     "test_name": "test_voxtral_tts",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "tts"
+    ],
     "server_params": {
       "model": "mistralai/Voxtral-4B-TTS-2603"
     },

tests/dfx/perf/tests/test_voxcpm2.json

@@ -1,6 +1,18 @@
 [
   {
     "test_name": "test_voxcpm2",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "tts"
+    ],
     "server_params": {
       "model": "openbmb/VoxCPM2",
       "trust_remote_code": true,

tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json

@@ -1,107 +1,131 @@
 [
-    {
-        "test_name": "test_wan22_i2v_single_device",
-        "description": "Single-device baseline",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
-            "serve_args": {
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "832x480_frames81_steps4",
-                "dataset": "random",
-                "task": "i2v",
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 1,
-                "seed": 42,
-                "enable-negative-prompt": true,
-                "random-request-config": [
-                    {
-                        "width": 832,
-                        "height": 480,
-                        "num_inference_steps": 4,
-                        "num_frames": 81,
-                        "fps": 16,
-                        "weight": 1
-                    }
-                ],
-                "baseline": {
-                    "throughput_qps": 0.0465,
-                    "latency_mean": 21.4354,
-                    "peak_memory_mb_mean": 72557.58
-                }
-            }
-        ]
+  {
+    "test_name": "test_wan22_i2v_single_device",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 1
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "Single-device baseline",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+      "serve_args": {
+        "enable-diffusion-pipeline-profiler": true
+      }
     },
-    {
-        "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing",
-        "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
-        "server_type": "vllm-omni",
-        "server_params": {
-            "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
-            "serve_args": {
-                "usp": 2,
-                "vae-patch-parallel-size": 2,
-                "use-hsdp": true,
-                "vae-use-slicing": true,
-                "enable-diffusion-pipeline-profiler": true
-            }
-        },
-        "benchmark_params": [
-            {
-                "name": "832x480_frames81_steps4",
-                "dataset": "random",
-                "task": "i2v",
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 1,
-                "seed": 42,
-                "enable-negative-prompt": true,
-                "random-request-config": [
-                    {
-                        "width": 832,
-                        "height": 480,
-                        "num_inference_steps": 4,
-                        "num_frames": 81,
-                        "fps": 16,
-                        "weight": 1
-                    }
-                ],
-                "baseline": {
-                    "throughput_qps": 0.0642,
-                    "latency_mean": 15.5589,
-                    "peak_memory_mb_mean": 45742.88
-                }
-            },
-            {
-                "name": "1280x720_frames121_steps4",
-                "dataset": "random",
-                "task": "i2v",
-                "num-prompts": 10,
-                "max-concurrency": 1,
-                "num-input-images": 1,
-                "seed": 42,
-                "enable-negative-prompt": true,
-                "random-request-config": [
-                    {
-                        "width": 1280,
-                        "height": 720,
-                        "num_inference_steps": 4,
-                        "num_frames": 121,
-                        "fps": 16,
-                        "weight": 1
-                    }
-                ],
-                "baseline": {
-                    "throughput_qps": 0.0128,
-                    "latency_mean": 77.8565,
-                    "peak_memory_mb_mean": 53947.84
-                }
-            }
-        ]
-    }
+    "benchmark_params": [
+      {
+        "name": "832x480_frames81_steps4",
+        "dataset": "random",
+        "task": "i2v",
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 1,
+        "seed": 42,
+        "enable-negative-prompt": true,
+        "random-request-config": [
+          {
+            "width": 832,
+            "height": 480,
+            "num_inference_steps": 4,
+            "num_frames": 81,
+            "fps": 16,
+            "weight": 1
+          }
+        ],
+        "baseline": {
+          "throughput_qps": 0.0465,
+          "latency_mean": 21.4354,
+          "peak_memory_mb_mean": 72557.58
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 2
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
+    "server_type": "vllm-omni",
+    "server_params": {
+      "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+      "serve_args": {
+        "usp": 2,
+        "vae-patch-parallel-size": 2,
+        "use-hsdp": true,
+        "vae-use-slicing": true,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "832x480_frames81_steps4",
+        "dataset": "random",
+        "task": "i2v",
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 1,
+        "seed": 42,
+        "enable-negative-prompt": true,
+        "random-request-config": [
+          {
+            "width": 832,
+            "height": 480,
+            "num_inference_steps": 4,
+            "num_frames": 81,
+            "fps": 16,
+            "weight": 1
+          }
+        ],
+        "baseline": {
+          "throughput_qps": 0.0642,
+          "latency_mean": 15.5589,
+          "peak_memory_mb_mean": 45742.88
+        }
+      },
+      {
+        "name": "1280x720_frames121_steps4",
+        "dataset": "random",
+        "task": "i2v",
+        "num-prompts": 10,
+        "max-concurrency": 1,
+        "num-input-images": 1,
+        "seed": 42,
+        "enable-negative-prompt": true,
+        "random-request-config": [
+          {
+            "width": 1280,
+            "height": 720,
+            "num_inference_steps": 4,
+            "num_frames": 121,
+            "fps": 16,
+            "weight": 1
+          }
+        ],
+        "baseline": {
+          "throughput_qps": 0.0128,
+          "latency_mean": 77.8565,
+          "peak_memory_mb_mean": 53947.84
+        }
+      }
+    ]
+  }
 ]