import json import os import re import subprocess import threading from datetime import datetime from pathlib import Path from typing import Any import pytest from tests.helpers.mark import get_hardware_mark_list, hardware_marks from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import modify_stage_config from vllm_omni.platforms import current_omni_platform def _named_pytest_marks(names: list[str]) -> list[pytest.MarkDecorator]: marks: list[pytest.MarkDecorator] = [] for name in names: name = name.strip() if not name: raise ValueError("mark name must be a non-empty string") marks.append(getattr(pytest.mark, name)) return marks def _hardware_marks_from_dict(hw: Any) -> list[pytest.MarkDecorator]: if not isinstance(hw, dict): raise ValueError(f"mark.hardware_marks must be a dict, got {type(hw).__name__}") res = hw.get("res") if not isinstance(res, dict): raise ValueError(f"mark.hardware_marks.res must be a dict, got {type(res).__name__}") num_cards = hw.get("num_cards", 1) return list(hardware_marks(res=res, num_cards=num_cards)) def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]: """Convert a JSON ``mark`` field into pytest mark decorators. Supported form (per test-case object in perf/stability JSON):: "mark": [ {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion" ] Exactly one ``hardware_marks`` object is required when ``mark`` is present. ``hardware_marks`` delegates to :func:`tests.helpers.mark.hardware_marks` (same shape as ``@hardware_test``). Additional array entries are registered pytest marker names (strings). """ if mark_field is None: return [] if isinstance(mark_field, list): marks: list[pytest.MarkDecorator] = [] hw_seen = False for item in mark_field: if isinstance(item, dict) and "hardware_marks" in item: if hw_seen: raise ValueError("mark array must contain at most one hardware_marks object") hw_seen = True marks.extend(_hardware_marks_from_dict(item["hardware_marks"])) unknown_keys = set(item) - {"hardware_marks"} if unknown_keys: raise ValueError( f"mark hardware_marks object only allows hardware_marks; unknown keys: {sorted(unknown_keys)}" ) elif isinstance(item, str): item = item.strip() if not item: raise ValueError("mark name must be a non-empty string") marks.extend(_named_pytest_marks([item])) else: raise ValueError( f"mark array entries must be hardware_marks objects or marker name strings; got {type(item).__name__}" ) if not hw_seen: raise ValueError("mark array must contain a hardware_marks object") return marks raise ValueError(f"mark must be a list; got {type(mark_field).__name__}") def _mark_names(mark_field: Any) -> set[str]: if isinstance(mark_field, list): return {str(item) for item in mark_field if isinstance(item, str)} return set() def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool: """True for perf JSON cases intended for ``run_diffusion_benchmark.py``.""" if cfg.get("server_type") is not None: return True return "diffusion" in _mark_names(cfg.get("mark")) def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs} def create_unique_server_pytest_params( configs: list[dict[str, Any]], stage_configs_dir: Path, ) -> list[Any]: """Like :func:`_create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks.""" marks_by_name = _marks_by_test_name(configs) return [ pytest.param( row, marks=marks_by_name.get(row[0], []), id=row[0], ) for row in _create_unique_server_params(configs, stage_configs_dir) ] def create_paired_benchmark_pytest_params( server_entries: list[tuple[Any, str]], params_by_test_name: dict[str, list[Any]], marks_by_name: dict[str, list[pytest.MarkDecorator]], ) -> list[Any]: """One ``pytest.param`` per ``(server entry, benchmark index)`` pair. Pass ``server_entry`` and ``(test_name, idx)`` as separate ``pytest.param`` arguments for ``@pytest.mark.parametrize("server_fixture,benchmark_params", ...)``. """ pairs: list[Any] = [] for server_entry, test_name in server_entries: params_list = params_by_test_name.get(test_name, []) id_suffixes = _unique_benchmark_param_id_suffixes(params_list) for idx, id_suffix in enumerate(id_suffixes): pairs.append( pytest.param( server_entry, (test_name, idx), marks=marks_by_name.get(test_name, []), id=f"{test_name}-{id_suffix}", ) ) return pairs def create_paired_omni_benchmark_pytest_params( configs: list[dict[str, Any]], stage_configs_dir: Path, ) -> list[Any]: """Paired params for ``run_benchmark.py`` (omni/tts).""" mapping = create_test_parameter_mapping(configs) marks_by_name = _marks_by_test_name(configs) server_entries = [(row, row[0]) for row in _create_unique_server_params(configs, stage_configs_dir)] params_by_test_name = { test_name: get_benchmark_params_for_server(test_name, mapping) for _, test_name in server_entries } return create_paired_benchmark_pytest_params(server_entries, params_by_test_name, marks_by_name) def load_configs(config_path: str) -> list[dict[str, Any]]: try: abs_path = Path(config_path).resolve() with open(abs_path, encoding="utf-8") as f: configs = json.load(f) return configs except json.JSONDecodeError as e: raise ValueError(f"JSON parsing error: {str(e)}") except FileNotFoundError: raise ValueError(f"Configuration file not found: {config_path}") except Exception as e: raise RuntimeError(f"Failed to load configuration file: {str(e)}") def load_benchmark_configs( config_path: str | None = None, *, config_dir: Path | None = None, ) -> list[dict[str, Any]]: """Load one benchmark JSON file, or merge all ``*.json`` under *config_dir*. When *config_path* is omitted, every ``*.json`` in *config_dir* is loaded and concatenated. Pytest ``-m`` expressions then select cases via each entry's ``mark`` field (e.g. ``-m tts`` after bulk load). """ if config_path is not None: return load_configs(config_path) if config_dir is None: raise ValueError("load_benchmark_configs requires config_path or config_dir") configs: list[dict[str, Any]] = [] for path in sorted(config_dir.glob("*.json")): configs.extend(load_configs(str(path))) if not configs: raise ValueError(f"No benchmark JSON files found under {config_dir}") return configs def modify_stage(default_path: str, updates: dict[str, Any] | None, deletes: dict[str, Any] | None) -> str: kwargs: dict[str, Any] = {} if updates is not None: kwargs["updates"] = updates if deletes is not None: kwargs["deletes"] = deletes if kwargs: return modify_stage_config(default_path, **kwargs) return default_path def _build_serve_args(serve_args: Any) -> list[str]: """Convert server_params.serve_args to a flat CLI args list.""" if serve_args is None: return [] if isinstance(serve_args, list): return [str(item) for item in serve_args] if not isinstance(serve_args, dict): raise TypeError(f"serve_args must be dict/list/None, got {type(serve_args).__name__}") args: list[str] = [] for key, value in serve_args.items(): flag = f"--{str(key).replace('_', '-')}" if isinstance(value, bool): if value: args.append(flag) continue if value is None: continue if isinstance(value, (dict, list)): args.extend([flag, json.dumps(value, ensure_ascii=False, separators=(",", ":"))]) continue args.extend([flag, str(value)]) return args def _create_unique_server_params( configs: list[dict[str, Any]], stage_configs_dir: Path, ) -> list[tuple[str, str, str | None, str | None, tuple[str, ...], bool]]: """Return one row per unique server configuration. ``(test_name, model, deploy_yaml_path, stage_overrides_json, extra_cli_args, use_omni)``. JSON ``server_params.serve_args`` (dict/list) is expanded via ``_build_serve_args`` and **prepended** to ``extra_cli_args`` so perf / stability ``omni_server`` fixtures stay identical to main while still honoring ``serve_args`` in benchmark JSON. """ unique_params: list[tuple[str, str, str | None, str | None, tuple[str, ...], bool]] = [] seen: set[tuple[str, str, str | None, str | None, tuple[str, ...], bool]] = set() for config in configs: test_name = config["test_name"] server_params = config["server_params"] model = server_params["model"] stage_config_name = server_params.get("stage_config_name") if stage_config_name: stage_config_path = str(stage_configs_dir / stage_config_name) delete = server_params.get("delete", None) update = server_params.get("update", None) stage_config_path = modify_stage(stage_config_path, update, delete) else: stage_config_path = None stage_overrides = server_params.get("stage_overrides") stage_overrides_json = json.dumps(stage_overrides) if stage_overrides else None # ``extra_cli_args`` passes raw CLI flags straight through to # ``vllm_omni.entrypoints.cli.main serve`` — used for flags that # don't map to stage-level overrides, e.g. ``--async-chunk`` / # ``--no-async-chunk`` toggling the deploy-level async_chunk bool. serve_flat = _build_serve_args(server_params.get("serve_args")) raw_extra = tuple(server_params.get("extra_cli_args") or ()) extra_cli_args = tuple(serve_flat) + raw_extra use_omni = bool(server_params.get("use_omni", True)) server_param = ( test_name, model, stage_config_path, stage_overrides_json, extra_cli_args, use_omni, ) if server_param not in seen: seen.add(server_param) unique_params.append(server_param) return unique_params def configs_with_platform_stage_configs(configs: list[dict[str, Any]]) -> list[dict[str, Any]]: """Resolve stage config for XPU vs CUDA/ROCm.""" out: list[dict[str, Any]] = [] for config in configs: config_copy = json.loads(json.dumps(config)) if current_omni_platform.is_xpu(): config_copy["server_params"]["stage_config_name"] = "xpu/qwen3_omni_ci.yaml" out.append(config_copy) return out def extract_server_args_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[str] | None]: mapping: dict[str, list[str] | None] = {} for cfg in configs: test_name = str(cfg.get("test_name")) server_params = cfg.get("server_params") or {} raw_args = server_params.get("server_args") mapping[test_name] = [str(item) for item in raw_args] if isinstance(raw_args, list) else None return mapping def create_reliability_omni_server_params( configs: list[dict[str, Any]], stage_configs_dir: Path ) -> list[OmniServerParams]: adjusted_configs = configs_with_platform_stage_configs(configs) unique_params = _create_unique_server_params(adjusted_configs, stage_configs_dir) server_args_by_name = extract_server_args_by_test_name(adjusted_configs) return [ OmniServerParams( model=model, stage_config_path=stage_config_path, server_args=server_args_by_name.get(test_name), use_omni=use_omni, ) for test_name, model, stage_config_path, _stage_overrides_json, _extra_cli_args, use_omni in unique_params ] def supports_video_generation(model_name: str) -> bool: lower = model_name.lower() return any(key in lower for key in ("wan", "video", "i2v", "t2v")) def parse_stage_devices(stage_config_path: str) -> str: text = Path(stage_config_path).read_text(encoding="utf-8") raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE) devices: set[int] = set() for item in raw_devices: for token in item.split(","): token = token.strip() if token: devices.add(int(token)) if not devices: raise ValueError(f"No runtime.devices found in stage config: {stage_config_path}") return ",".join(str(x) for x in sorted(devices)) def resolve_oom_device_spec(config: dict[str, Any], stage_config_path: str | None) -> str: explicit = config.get("device") if explicit is not None: return str(explicit) if not stage_config_path: return "0" return parse_stage_devices(stage_config_path) def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> None: text = str(exc).lower() assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}" def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]: mapping = {} for config in configs: test_name = config["test_name"] if test_name not in mapping: mapping[test_name] = { "test_name": test_name, "benchmark_params": [], } for entry in config["benchmark_params"]: # Skip disabled entries if not entry.get("enabled", True): continue mapping[test_name]["benchmark_params"].append(entry) return mapping def get_benchmark_params_for_server(test_name: str, server_to_benchmark_mapping: dict[str, dict]) -> list: if test_name not in server_to_benchmark_mapping: return [] return server_to_benchmark_mapping[test_name]["benchmark_params"] def create_benchmark_indices( benchmark_configs: list[dict[str, Any]], server_to_benchmark_mapping: dict[str, dict], ) -> list[tuple[str, int]]: indices = [] seen = set() for config in benchmark_configs: test_name = config["test_name"] if test_name not in seen: seen.add(test_name) params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) for idx in range(len(params_list)): indices.append((test_name, idx)) return indices # Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON). OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json" def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: """Make a single path segment safe for result filenames on common filesystems.""" if value is None: return default s = str(value).strip() for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"): s = s.replace(bad, "_") return s if s else default def _benchmark_param_id_suffix(param: dict[str, Any], *, idx: int) -> str: """Derive a readable pytest id suffix from one ``benchmark_params`` entry.""" name = param.get("name") if isinstance(name, str) and name.strip(): return _safe_filename_token(name.strip()) parts: list[str] = [] for key in ("task", "eval_phase", "dataset_name", "dataset"): value = param.get(key) if value is None or value == "": continue token = _safe_filename_token(str(value)) if token != "na": parts.append(token) if parts: return "_".join(parts) return f"case{idx}" def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> list[str]: """Return unique pytest id suffixes for a server's benchmark param list.""" raw = [_benchmark_param_id_suffix(param, idx=idx) for idx, param in enumerate(params_list)] seen: dict[str, int] = {} unique: list[str] = [] for suffix in raw: count = seen.get(suffix, 0) seen[suffix] = count + 1 if count == 0: unique.append(suffix) else: unique.append(f"{suffix}_{count}") return unique # Device marketing-name tokens for ``get_runtime_resource_label`` only. # Longer / more specific tokens must appear before shorter substrings # (e.g. ``H200`` before ``H20``, ``910B4`` before ``910``). _RUNTIME_DEVICE_ALIASES: tuple[str, ...] = ( "H100", "H800", "H200", "H20", "B200", "GB200", "L40S", "L40", "L4", "A100", "A800", "A10G", "A10", "A30", "MI325", "MI300", "MI250", "B60", "S5000", "910B4", "910B", "910", "310P", "A2", "A3", ) _RUNTIME_RESOURCE_LABEL: str | None = None def _normalize_runtime_device_label(raw: str) -> str: """Map a platform device name to a short filename-safe resource token.""" if not raw or not str(raw).strip(): return "na" upper = str(raw).upper() for token in _RUNTIME_DEVICE_ALIASES: if token.upper() in upper: return _safe_filename_token(token) compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw)) for prefix in ("NVIDIA", "AMD", "ASCEND", "HUAWEI"): if compact.upper().startswith(prefix): compact = compact[len(prefix) :] break return _safe_filename_token(compact[:48]) if compact else "na" def _read_runtime_device_name(*, device_id: int = 0) -> str | None: """Device name from the active Omni platform.""" if current_omni_platform.device_count() <= device_id: return None get_name = getattr(current_omni_platform, "get_device_name", None) if not callable(get_name): return None raw = get_name(device_id) if isinstance(raw, str) and raw.strip(): return raw.strip() return None def get_runtime_resource_label(*, device_id: int = 0, refresh: bool = False) -> str: """Return a filename-safe hardware label detected on the running machine.""" global _RUNTIME_RESOURCE_LABEL if not refresh and _RUNTIME_RESOURCE_LABEL is not None: return _RUNTIME_RESOURCE_LABEL raw = _read_runtime_device_name(device_id=device_id) label = _normalize_runtime_device_label(raw) if raw else "na" if not refresh: _RUNTIME_RESOURCE_LABEL = label return label _FILENAME_OMIT_RESOURCE_LABELS = frozenset({"H100"}) def hardware_json_value(resource_label: str | None) -> str: """Hardware token stored in perf result JSON (empty when unknown).""" token = _safe_filename_token(resource_label) return "" if token == "na" else token def resource_label_for_filename(resource_label: str | None) -> str: """Hardware token embedded in result filenames (H100 omitted on default CI pool).""" token = _safe_filename_token(resource_label) if token in _FILENAME_OMIT_RESOURCE_LABELS: return "" return token def is_hardware_nested_baseline(baseline: dict[str, Any]) -> bool: """True for ``{"H100": {"metric": ...}, "A3": {...}}`` baselines. Detection is structural + hardware-label allowlist (not metric-name hints): - every top-level value is a non-empty dict (per-hardware metric map); - every top-level key is in :func:`tests.helpers.mark.get_hardware_mark_list` (``[hardware-resource]`` markers from ``pyproject.toml``). Metric names under each hardware bucket are unconstrained. Runtime device aliases (:data:`_RUNTIME_DEVICE_ALIASES`) are unrelated. """ if not baseline: return False if not all(isinstance(value, dict) and bool(value) for value in baseline.values()): return False allowed = {label.upper() for label in get_hardware_mark_list()} return all((token := str(key).strip()) and token.upper() in allowed for key in baseline.keys()) def resolve_baseline_value( baseline_raw: Any, *, sweep_index: int | None, ) -> Any: """Pick the baseline threshold for one metric at this sweep step. Accepts a sweep-aligned list/tuple (indexed by ``sweep_index``) or a scalar. """ if baseline_raw is None: return None if isinstance(baseline_raw, dict): raise TypeError( "per-metric baseline dict keyed by concurrency/request-rate is not supported; " "use a sweep-aligned list or a scalar under each hardware bucket" ) if isinstance(baseline_raw, (list, tuple)): if sweep_index is None: raise ValueError("list baseline requires sweep_index") if not (0 <= sweep_index < len(baseline_raw)): raise IndexError(f"baseline list len={len(baseline_raw)} has no index {sweep_index}") return baseline_raw[sweep_index] return baseline_raw def resolve_baseline_for_sweep( baseline_config: dict[str, Any] | None, *, sweep_index: int | None = None, ) -> dict[str, Any]: """Resolve sweep-dependent baselines while keeping every hardware bucket. Expects hardware-nested input only:: {"H100": {"throughput_qps": [a, b, c]}, "A3": {"throughput_qps": [x, y, z]}} with ``sweep_index=2`` (e.g. concurrency 32) becomes:: {"H100": {"throughput_qps": c}, "A3": {"throughput_qps": z}} Per-metric values may be a sweep-aligned list or a scalar. """ if not baseline_config: return {} if not is_hardware_nested_baseline(baseline_config): allowed = sorted(get_hardware_mark_list()) unknown = [ str(key).strip() for key in baseline_config.keys() if str(key).strip().upper() not in {label.upper() for label in allowed} ] nested_shape = all(isinstance(value, dict) and bool(value) for value in baseline_config.values()) if nested_shape and unknown: raise ValueError( "baseline top-level keys must be [hardware-resource] pytest markers from " "pyproject.toml (tool.pytest.ini_options.markers). " f"Unknown hardware label(s): {unknown!r}. " f"Known markers: {allowed!r}. " "Add a marker such as '\"A100: [hardware-resource] Tests that require A100 GPU\"' " "to pyproject.toml, then retry." ) raise ValueError( "baseline must be hardware-nested with [hardware-resource] marker names as " "top-level keys, e.g. " '{"H100": {"throughput_qps": [...]}, "A3": {"custom_metric": 1.0}}. ' f"got top-level keys={list(baseline_config.keys())!r}; " f"known markers={allowed!r}. " "If you need a new hardware bucket, add it to pyproject.toml markers with " "the [hardware-resource] tag." ) return { hardware: { metric_name: resolve_baseline_value( baseline_raw, sweep_index=sweep_index, ) for metric_name, baseline_raw in metrics.items() } for hardware, metrics in baseline_config.items() if isinstance(metrics, dict) } def run_benchmark( args: list[str], test_name: str, flow: Any, dataset_name: str, num_prompt: int, *, baseline_config: dict[str, Any] | None = None, sweep_index: int | None = None, random_input_len: Any | None = None, random_output_len: Any | None = None, resource_label: str | None = None, num_warmups: int = 2, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the sweep-resolved baseline (when ``baseline_config`` is provided). Hardware-nested maps keep every hardware bucket, but each metric is reduced to the value for this concurrency / request-rate step. If the benchmark exits without writing a result file, ``result_omni_template.json`` is used as a fallback. """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) ro = _safe_filename_token(random_output_len) hw = resource_label_for_filename(resource_label) if hw: result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" else: result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" if "--result-filename" in args: print(f"The result file will be overwritten by {result_filename}") command = ( ["vllm", "bench", "serve", "--omni"] + args + [ "--num-warmups", str(num_warmups), "--save-result", "--result-dir", os.environ.get("BENCHMARK_DIR", "tests"), "--result-filename", result_filename, ] ) process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True ) if process.stdout is None or process.stderr is None: raise RuntimeError("Failed to capture benchmark process output streams") def _forward_stream(stream) -> None: try: for line in iter(stream.readline, ""): print(line, end="") finally: stream.close() stdout_thread = threading.Thread(target=_forward_stream, args=(process.stdout,)) stderr_thread = threading.Thread(target=_forward_stream, args=(process.stderr,)) stdout_thread.start() stderr_thread.start() stdout_thread.join() stderr_thread.join() process.wait() if "--result-dir" in command: index = command.index("--result-dir") result_dir = command[index + 1] else: result_dir = "./" result_path = os.path.join(result_dir, result_filename) if not os.path.exists(result_path): with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f: template_result: dict[str, Any] = json.load(f) Path(result_path).parent.mkdir(parents=True, exist_ok=True) with open(result_path, "w", encoding="utf-8") as f: json.dump(template_result, f, ensure_ascii=False, indent=2) print(f"Benchmark result file not generated, fallback to template: {result_path}") result = template_result else: with open(result_path, encoding="utf-8") as f: result = json.load(f) if baseline_config: # Keep every hardware bucket; resolve list metrics to this sweep step. result["baseline"] = resolve_baseline_for_sweep( baseline_config, sweep_index=sweep_index, ) else: result["baseline"] = {} if random_input_len is not None: result["random_input_len"] = random_input_len if random_output_len is not None: result["random_output_len"] = random_output_len result["Hardware"] = hardware_json_value(resource_label) with open(result_path, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) return result def pytest_addoption(parser: pytest.Parser) -> None: """Register shared CLI options for DFX benchmark suites.""" parser.addoption( "--test-config-file", action="store", default=None, help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"), ) parser.addoption( "--assert-baseline", action="store_true", default=False, help=( "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config " "(default: off)." ), )