[Benchmark]Omni-modality model accuracy benchmark(Daily-Omni & seed-tts-eval) (#2558)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>

pyproject.toml

@@ -61,6 +61,17 @@ demo = [
     "gradio>=6.7.0",
 ]
 
+# Seed-TTS serve benchmark WER (BytedanceSpeech/seed-tts-eval run_wer.py protocol).
+seed-tts-eval = [
+    "jiwer>=3.0.0",
+    "zhon>=2.0.0",
+    "zhconv>=1.4.2",
+    "scipy>=1.10.0",
+    "soundfile>=0.12.0",
+    "transformers>=4.36.0",
+    "funasr>=1.0.0",
+]
+
 docs = [
     "mkdocs>=1.5.0",
     "mkdocs-api-autonav",

vllm_omni/benchmarks/data_modules/daily_omni_dataset.py

@@ -0,0 +1,887 @@
+"""Daily-Omni Dataset loader for benchmark.
+
+Daily-Omni is an audio-visual reasoning benchmark with 684 videos
+and 1,197 multiple-choice QA pairs across 6 major task types.
+
+Dataset source: https://huggingface.co/datasets/liarliar/Daily-Omni
+
+Supports loading QA metadata from:
+- Local JSON file (``qa_json_path``): recommended for offline/air-gapped environments
+- HuggingFace datasets (``dataset_path``): legacy online mode
+
+The videos must be separately downloaded and extracted from Videos.tar.
+
+Why ``BenchmarkDataset`` instead of ``HuggingFaceDataset``?
+    vLLM's ``HuggingFaceDataset`` is a thin wrapper whose ``__init__`` always ends by calling
+    ``load_data()`` → ``datasets.load_dataset(...)`` with a required Hub id and split. That
+    contract fits "Hub-only" benches, but Daily-Omni also needs **offline QA metadata** from a
+    local ``qa.json`` without touching the network. Subclassing ``HuggingFaceDataset`` would
+    mean fighting the parent constructor (fake ``dataset_path``, reordering ``load_data``, or
+    duplicating half the parent) and would still imply ``datasets`` is always relevant.
+
+    This class therefore inherits only ``BenchmarkDataset`` (minimal: ``dataset_path``,
+    ``random_seed``, ``self.data``) and implements **two explicit loaders**:
+    ``_load_from_local_json`` (default path for air-gapped runs) and ``_load_from_huggingface``
+    (optional legacy path for users who prefer ``datasets`` + Hub cache). The latter is **not**
+    inheritance; it is the same Hub rows as before, factored into a helper so one class can
+    serve both deployment modes without mandatory ``datasets`` when using ``qa_json_path``.
+
+Usage:
+    from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset
+
+    # Local JSON mode (recommended)
+    dataset = DailyOmniDataset(
+        qa_json_path="/path/to/qa.json",
+        video_dir="/path/to/Videos",
+        random_seed=42,
+    )
+
+    # HuggingFace mode (legacy, requires network)
+    dataset = DailyOmniDataset(
+        dataset_path="liarliar/Daily-Omni",
+        dataset_split="train",
+        random_seed=42,
+    )
+    requests = dataset.sample(
+        tokenizer=tokenizer,
+        num_requests=100,
+        output_len=256,
+    )
+"""
+
+import base64
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+try:
+    from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest
+except ImportError:
+    # Fallback: if BenchmarkDataset not available, use base class from same module
+    from vllm.benchmarks.datasets import HuggingFaceDataset as BenchmarkDataset
+    from vllm.benchmarks.datasets import SampleRequest
+from vllm.tokenizers import TokenizerLike
+from vllm.tokenizers.hf import get_cached_tokenizer
+
+try:
+    from datasets import load_dataset
+except ImportError:
+    load_dataset = None
+
+logger = logging.getLogger(__name__)
+
+
+class _ListDatasetIterator:
+    """Simple iterator wrapper around a list to mimic HuggingFace streaming dataset behavior."""
+
+    def __init__(self, data: list[dict[str, Any]]) -> None:
+        self._data = data
+        self._index = 0
+
+    def __iter__(self):
+        self._index = 0
+        return self
+
+    def __next__(self) -> dict[str, Any]:
+        if self._index >= len(self._data):
+            raise StopIteration
+        item = self._data[self._index]
+        self._index += 1
+        return item
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    def __getitem__(self, idx: int | slice) -> dict[str, Any] | list[dict[str, Any]]:
+        return self._data[idx]
+
+
+# Aligns with Lliar-liar/Daily-Omni CLI ``--input_mode`` (test_model/*/testmodel.py).
+DailyOmniInputMode = Literal["all", "visual", "audio"]
+
+# ``build_conversation()`` in Daily-Omni ``test_model/Qwen2.5-Omni/testmodel.py`` (verbatim).
+DAILY_OMNI_SYSTEM_TEXT = (
+    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+    "capable of perceiving auditory and visual inputs, as well as generating text and speech."
+)
+
+
+@dataclass
+class DailyOmniSampleRequest(SampleRequest):
+    """``SampleRequest`` with Daily-Omni gold labels for post-run accuracy scoring."""
+
+    daily_omni_gold_answer: str = ""
+    daily_omni_video_id: str = ""
+    daily_omni_task_type: str = ""
+    #: Official qa.json ``video_duration`` (e.g. ``30s``, ``60s``) for leaderboard-style breakdown.
+    daily_omni_video_duration: str = ""
+    #: Official ``video_category`` (YouTube-style category string) for per-category accuracy.
+    daily_omni_video_category: str = ""
+    #: Extra JSON fields merged into chat-completions ``extra_body`` (e.g. ``mm_processor_kwargs``).
+    omni_extra_body: dict[str, Any] | None = None
+    #: Full OpenAI ``messages`` (system + user) mirroring upstream Daily-Omni conversation.
+    omni_chat_messages: list[dict[str, Any]] | None = None
+    #: Used only when ``omni_chat_messages`` is None (non-Daily-Omni-style requests).
+    omni_chat_mm_position: Literal["first", "last"] = "last"
+
+
+class DailyOmniDataset(BenchmarkDataset):
+    """Daily-Omni audio-visual QA dataset for benchmarking.
+
+    Inherits ``BenchmarkDataset`` only (not ``HuggingFaceDataset``): see module docstring for why
+    Hub loading lives in ``_load_from_huggingface`` instead of subclassing the HF base class.
+
+    The dataset includes:
+    - 684 videos from daily life scenarios (available in Videos.tar)
+    - 1,197 multiple-choice QA pairs in qa.json
+    - 6 major task categories
+
+    QA metadata can be loaded from:
+    - Local JSON file (``qa_json_path``): recommended for offline/air-gapped environments
+    - HuggingFace datasets (``dataset_path``): legacy online mode
+
+    The videos must be separately downloaded and extracted from Videos.tar.
+
+    Args:
+        qa_json_path: Path to local qa.json file (offline mode, preferred). When provided,
+            ``dataset_path`` and ``dataset_split`` are ignored.
+        dataset_path: HuggingFace dataset path (e.g., "liarliar/Daily-Omni"). Used only if
+            ``qa_json_path`` is not provided (legacy online mode).
+        dataset_split: Dataset split to use (default: "train"). Used only in online mode.
+        random_seed: Random seed for shuffling
+        video_dir: Directory containing extracted video files (default: None)
+        input_mode: Which modalities to send, matching upstream Daily-Omni ``--input_mode``:
+            ``all`` — video + WAV (default; official audio-visual protocol);
+            ``visual`` — video only;
+            ``audio`` — extracted WAV only (requires ``{video_id}/{video_id}_audio.wav`` under ``video_dir``).
+        max_duration_seconds: Reserved for future ffprobe-based filtering; currently **not applied**
+            when building requests (metadata ``video_duration`` is still passed through for eval).
+        dataset_subset: Optional HuggingFace subset name (``load_dataset(..., name=...)``); used by bench
+            ``--hf-subset`` / patch.
+        no_stream: If True, load the Hub split non-streaming (matches bench ``--no-stream``).
+        inline_local_video: If True, embed local MP4 as ``data:video/mp4;base64,...`` in requests so
+            the API server does not need ``--allowed-local-media-path`` (large JSON; use for small runs).
+            When ``input_mode`` is ``audio`` or ``all``, local WAV is embedded the same way
+            (``data:audio/wav;base64,...``).
+        trust_remote_code: Whether to trust remote code when loading HuggingFace dataset
+            (online mode only).
+    """
+
+    SUPPORTED_DATASET_PATHS: set[str] = {
+        "liarliar/Daily-Omni",
+    }
+    #: Default Hub id for synthetic video URLs when ``qa_json_path`` is used (``dataset_path`` None).
+    DEFAULT_HF_DATASET_ID = "liarliar/Daily-Omni"
+    IS_MULTIMODAL = True
+    DEFAULT_OUTPUT_LEN = 256
+
+    def __init__(
+        self,
+        qa_json_path: str | None = None,
+        dataset_path: str | None = None,
+        dataset_split: str = "train",
+        random_seed: int = 0,
+        video_dir: str | None = None,
+        input_mode: DailyOmniInputMode = "all",
+        inline_local_video: bool = False,
+        trust_remote_code: bool = False,
+        max_duration_seconds: float | None = None,
+        dataset_subset: str | None = None,
+        no_stream: bool = False,
+        **kwargs,
+    ) -> None:
+        if input_mode not in ("all", "visual", "audio"):
+            raise ValueError(f"input_mode must be 'all', 'visual', or 'audio', got {input_mode!r}")
+
+        # Validate arguments: need either local JSON or HF path
+        if qa_json_path is None and dataset_path is None:
+            raise ValueError(
+                "Either 'qa_json_path' (local JSON) or 'dataset_path' (HuggingFace) must be provided. "
+                "For offline/air-gapped environments, download qa.json and use qa_json_path."
+            )
+
+        # Store configuration
+        self.qa_json_path = Path(qa_json_path) if qa_json_path else None
+        self.dataset_path = dataset_path
+        self.dataset_split = dataset_split
+        self.dataset_subset = dataset_subset
+        #: Match vLLM ``HuggingFaceDataset`` / bench CLI ``--no-stream``.
+        self._hf_streaming = not no_stream
+        self.video_dir = Path(video_dir) if video_dir else None
+        self.inline_local_video = inline_local_video
+        self.input_mode: DailyOmniInputMode = input_mode
+        self.max_duration_seconds = max_duration_seconds
+        self.trust_remote_code = trust_remote_code
+
+        #: In-process cache of ffprobe durations only (no disk persistence).
+        self._video_durations: dict[str, float] = {}
+
+        # Initialize parent BenchmarkDataset
+        super().__init__(
+            dataset_path=dataset_path if qa_json_path is None else None,
+            random_seed=random_seed,
+            **kwargs,
+        )
+
+        # Load data based on mode
+        self.load_data()
+
+        # Verify dataset info
+        logger.info(
+            "Loaded Daily-Omni dataset: mode=%s, source=%s, random_seed=%d, input_mode=%s, max_duration=%s",
+            "local_json" if self.qa_json_path else "huggingface",
+            str(self.qa_json_path) if self.qa_json_path else f"{dataset_path}/{dataset_split}",
+            random_seed,
+            input_mode,
+            f"{max_duration_seconds}s" if max_duration_seconds else "unlimited",
+        )
+
+    def load_data(self) -> None:
+        """Populate ``self.data`` from either local JSON or the Hub.
+
+        See module docstring: we do not subclass ``HuggingFaceDataset`` because Daily-Omni needs
+        a first-class offline path; Hub loading is an optional branch implemented below.
+        """
+        if self.qa_json_path is not None:
+            self._load_from_local_json()
+        else:
+            self._load_from_huggingface()
+
+    def _load_from_local_json(self) -> None:
+        """Load QA data from local JSON file."""
+        if not self.qa_json_path.exists():
+            raise FileNotFoundError(f"QA JSON file not found: {self.qa_json_path}")
+
+        with open(self.qa_json_path, encoding="utf-8") as f:
+            data = json.load(f)
+
+        # Support both list format and dict with "train"/"test" splits
+        if isinstance(data, dict):
+            # Try to get the requested split, fallback to first available
+            split_data = data.get(self.dataset_split)
+            if split_data is None:
+                available = list(data.keys())
+                if available:
+                    logger.warning(
+                        "Split '%s' not found in %s, using '%s' instead",
+                        self.dataset_split,
+                        self.qa_json_path,
+                        available[0],
+                    )
+                    split_data = data[available[0]]
+                else:
+                    split_data = []
+            data = split_data
+
+        if not isinstance(data, list):
+            raise ValueError(f"Expected list of QA items in JSON, got {type(data).__name__}")
+
+        # Shuffle if requested
+        if not getattr(self, "disable_shuffle", False) and self.random_seed is not None:
+            import random
+
+            rng = random.Random(self.random_seed)
+            shuffled = data[:]
+            rng.shuffle(shuffled)
+            data = shuffled
+
+        # Create an iterator-like wrapper for compatibility
+        self.data = _ListDatasetIterator(data)
+
+    def _load_from_huggingface(self) -> None:
+        """Load QA rows via ``datasets.load_dataset`` (legacy / convenience path).
+
+        Kept for backward compatibility: callers can still pass ``dataset_path=liarliar/Daily-Omni``
+        and get the same parquet-backed rows as the Hub dataset card, with streaming (or
+        non-streaming if ``no_stream=True``) and shuffle.
+
+        This is intentionally **not** implemented by subclassing ``HuggingFaceDataset``: that base
+        always runs Hub ``load_dataset`` from its constructor and expects a Hub id as the primary
+        API; Daily-Omni instead chooses the source in ``load_data()`` (JSON vs Hub) while sharing
+        one ``sample()`` / request-building implementation for both.
+        """
+        if load_dataset is None:
+            raise ImportError(
+                "datasets library is required for HuggingFace mode. "
+                "Install with: pip install datasets, or use local JSON mode instead."
+            )
+
+        ds = load_dataset(
+            self.dataset_path,
+            name=self.dataset_subset,
+            split=self.dataset_split,
+            streaming=self._hf_streaming,
+            trust_remote_code=self.trust_remote_code,
+        )
+        if not getattr(self, "disable_shuffle", False):
+            ds = ds.shuffle(seed=self.random_seed)
+        self.data = ds
+
+    def get_task_statistics(self) -> dict[str, int]:
+        """Get distribution of task types in the dataset.
+
+        Returns:
+            Dict mapping task type to count
+        """
+        stats: dict[str, int] = {}
+        for item in self.data:
+            row = self._coerce_row(item)
+            fields = self._normalize_qa_fields(row)
+            task_type = fields["task_type"] or "unknown"
+            stats[task_type] = stats.get(task_type, 0) + 1
+        return stats
+
+    @staticmethod
+    def _coerce_row(item: Any) -> dict[str, Any]:
+        """Turn a dataset row into a plain dict (Arrow / Mapping)."""
+        if isinstance(item, dict):
+            return item
+        if hasattr(item, "as_py"):
+            return dict(item.as_py())  # pyarrow Row
+        try:
+            return dict(item)
+        except (TypeError, ValueError):
+            return {k: item[k] for k in item}  # type: ignore[misc]
+
+    @staticmethod
+    def _normalize_qa_fields(row: dict[str, Any]) -> dict[str, Any]:
+        """Map official Daily-Omni qa.json / Hub schema to internal fields.
+
+        Official fields (see liarliar/Daily-Omni ``qa.json``): ``Question``, ``Choice`` (list),
+        ``Answer``, ``video_id``, ``Type``, ``video_duration`` (``30s`` / ``60s``), ``video_category``,
+        plus other category columns. Legacy aliases (lowercase / older loaders) are still accepted.
+        """
+        out: dict[str, Any] = {}
+
+        out["question"] = str(row.get("Question") or row.get("question") or "").strip()
+        vid = row.get("video_id") if row.get("video_id") is not None else row.get("video")
+        out["video_id"] = str(vid).strip() if vid is not None else ""
+        out["task_type"] = str(row.get("Type") or row.get("task_type") or row.get("type") or "").strip()
+        vc = row.get("video_category") if row.get("video_category") is not None else row.get("videoCategory")
+        out["video_category"] = str(vc).strip() if vc is not None else ""
+        vd = row.get("video_duration") if row.get("video_duration") is not None else row.get("videoDuration")
+        out["video_duration"] = str(vd).strip() if vd is not None else ""
+        out["answer"] = str(row.get("Answer") or row.get("answer") or "").strip()
+        vu = row.get("video_url") if row.get("video_url") is not None else row.get("Video_URL")
+        out["video_url"] = str(vu).strip() if vu is not None and str(vu).strip() else None
+
+        choice = row.get("Choice")
+        if choice is None:
+            choice = row.get("options") or row.get("choice")
+        out["choice"] = choice
+
+        return out
+
+    def sample(
+        self,
+        tokenizer: TokenizerLike,
+        num_requests: int,
+        output_len: int | None = None,
+        request_id_prefix: str = "",
+        no_oversample: bool = False,
+        **kwargs,
+    ) -> list[SampleRequest]:
+        """Sample requests from Daily-Omni dataset.
+
+        Args:
+            tokenizer: Tokenizer for computing prompt length
+            num_requests: Number of requests to sample
+            output_len: Target output length in tokens (default: 256)
+            request_id_prefix: Prefix for request IDs
+            no_oversample: If True, do not oversample if fewer examples available
+            **kwargs: Additional arguments (ignored)
+
+        Returns:
+            List of SampleRequest objects with video URLs and prompts
+        """
+        if output_len is None:
+            output_len = self.DEFAULT_OUTPUT_LEN
+
+        sampled_requests: list[SampleRequest] = []
+        ind = 0
+        cached_tokenizer = get_cached_tokenizer(tokenizer)
+
+        # Iterate over shuffled dataset
+        for item in self.data:
+            if len(sampled_requests) >= num_requests:
+                break
+
+            request = self._create_sample_request(
+                self._coerce_row(item), cached_tokenizer, output_len, request_id_prefix, ind
+            )
+            if request:
+                sampled_requests.append(request)
+                ind += 1
+
+        logger.info("Created %d sample requests from Daily-Omni dataset", len(sampled_requests))
+
+        # Handle oversampling if needed
+        self.maybe_oversample_requests(sampled_requests, num_requests, request_id_prefix, no_oversample)
+
+        return sampled_requests
+
+    def _create_sample_request(
+        self,
+        qa_item: dict[str, Any],
+        tokenizer: TokenizerLike,
+        output_len: int,
+        request_id_prefix: str,
+        index: int,
+    ) -> SampleRequest | None:
+        """Create a SampleRequest from a QA item.
+
+        Args:
+            qa_item: QA pair from the dataset
+            tokenizer: Tokenizer
+            output_len: Target output length
+            request_id_prefix: Prefix for request ID
+            index: Request index
+
+        Returns:
+            SampleRequest or None if invalid
+        """
+        fields = self._normalize_qa_fields(qa_item)
+        video_id = fields["video_id"]
+        question = fields["question"]
+        choice = fields["choice"]
+        task_type = fields["task_type"]
+        video_url = fields["video_url"]
+        video_duration = fields.get("video_duration") or ""
+        video_category = fields.get("video_category") or ""
+
+        if not video_id and not video_url:
+            logger.warning("Skipping item: no video_id / video_url")
+            return None
+
+        if not question:
+            logger.warning("Skipping item: no question found")
+            return None
+
+        # Official layout after extracting Videos.tar (see Lliar-liar/Daily-Omni test_model):
+        #   {video_base_dir}/{video_id}/{video_id}_video.mp4
+        mm_payload, omni_extra, mm_pos = self._compose_daily_omni_multimodal(video_id, video_url)
+        if not mm_payload:
+            return None
+
+        messages = self._build_daily_omni_openai_messages(mm_payload, question, choice)
+        user_text = self._official_daily_omni_user_prompt(question, choice)
+        # Text-only length estimate (same as before: no MM token count in bench).
+        prompt_len = len(tokenizer.encode(f"{DAILY_OMNI_SYSTEM_TEXT}\n{user_text}"))
+
+        return DailyOmniSampleRequest(
+            prompt=user_text,
+            prompt_len=prompt_len,
+            expected_output_len=output_len,
+            multi_modal_data=None,
+            request_id=f"{request_id_prefix}{index}",
+            daily_omni_gold_answer=fields["answer"],
+            daily_omni_video_id=video_id,
+            daily_omni_task_type=task_type,
+            daily_omni_video_duration=video_duration,
+            daily_omni_video_category=video_category,
+            omni_extra_body=omni_extra,
+            omni_chat_messages=messages,
+            omni_chat_mm_position=mm_pos,
+        )
+
+    @staticmethod
+    def _official_video_relpath(video_id: str) -> str:
+        """Relative path inside extracted ``Videos/`` per upstream Daily-Omni scripts."""
+        return f"{video_id}/{video_id}_video.mp4"
+
+    @staticmethod
+    def _official_audio_relpath(video_id: str) -> str:
+        """Relative path for extracted WAV per upstream ``get_audio_path``."""
+        return f"{video_id}/{video_id}_audio.wav"
+
+    def _resolve_local_video_path(self, video_id: str) -> Path | None:
+        """Pick an existing file under ``video_dir`` (official layout + flat fallback)."""
+        if not self.video_dir or not video_id:
+            return None
+
+        candidates = [
+            self.video_dir / self._official_video_relpath(video_id),
+            self.video_dir / f"{video_id}.mp4",  # flat layout (custom mirrors / outdated docs)
+        ]
+        seen: set[Path] = set()
+        for p in candidates:
+            rp = p.resolve()
+            if rp in seen:
+                continue
+            seen.add(rp)
+            if p.exists():
+                return p
+        return None
+
+    def _resolve_local_audio_path(self, video_id: str) -> Path | None:
+        """Pick an existing WAV under ``video_dir`` (official layout + flat fallback)."""
+        if not self.video_dir or not video_id:
+            return None
+        candidates = [
+            self.video_dir / self._official_audio_relpath(video_id),
+            self.video_dir / f"{video_id}.wav",
+        ]
+        seen: set[Path] = set()
+        for p in candidates:
+            rp = p.resolve()
+            if rp in seen:
+                continue
+            seen.add(rp)
+            if p.exists():
+                return p
+        return None
+
+    def _local_file_to_video_url_payload(self, video_path: Path) -> dict[str, Any]:
+        """Build OpenAI-style video_url part for a resolved local file.
+
+        vLLM rejects ``file://`` unless the server was started with
+        ``--allowed-local-media-path`` set to a directory that **contains** the file
+        (typically the extracted ``Videos`` root). Use ``inline_local_video=True`` to
+        send base64 data URLs instead (no server path allowlist; larger requests).
+        """
+        path = video_path.expanduser().resolve()
+        if self.inline_local_video:
+            raw = path.read_bytes()
+            b64 = base64.b64encode(raw).decode("ascii")
+            return {
+                "type": "video_url",
+                "video_url": {"url": f"data:video/mp4;base64,{b64}"},
+            }
+        return {
+            "type": "video_url",
+            "video_url": {"url": path.as_uri()},
+        }
+
+    def _local_file_to_audio_url_payload(self, audio_path: Path) -> dict[str, Any]:
+        """Build OpenAI-style ``audio_url`` part for a resolved local WAV file."""
+        path = audio_path.expanduser().resolve()
+        if self.inline_local_video:
+            raw = path.read_bytes()
+            b64 = base64.b64encode(raw).decode("ascii")
+            return {
+                "type": "audio_url",
+                "audio_url": {"url": f"data:audio/wav;base64,{b64}"},
+            }
+        return {
+            "type": "audio_url",
+            "audio_url": {"url": path.as_uri()},
+        }
+
+    def _get_video_content(
+        self,
+        video_id: str,
+        video_url: str | None,
+    ) -> dict[str, Any] | None:
+        """Resolve video for OpenAI-style ``video_url`` content.
+
+        Upstream uses ``get_video_path(video_id, base) -> base/video_id/video_id_video.mp4``.
+        The Hub repo only publishes ``Videos.tar``; use ``--daily-omni-video-dir`` pointing
+        at the extracted ``Videos`` folder (parent of per-``video_id`` subdirs).
+
+        For ``file://`` URLs, start ``vllm serve`` with e.g.
+        ``--allowed-local-media-path /same/path/as/daily-omni-video-dir``.
+        """
+        if video_url:
+            url = video_url
+            if not url.startswith(("http://", "https://", "file://")):
+                url = f"https://{url.lstrip('/')}"
+            return {"type": "video_url", "video_url": {"url": url}}
+
+        if self.video_dir and video_id:
+            video_path = self._resolve_local_video_path(video_id)
+            if video_path is not None:
+                return self._local_file_to_video_url_payload(video_path)
+            logger.warning(
+                "Video not found under video_dir=%s for video_id=%r (expected %s or %s)",
+                self.video_dir,
+                video_id,
+                self._official_video_relpath(video_id),
+                f"{video_id}.mp4",
+            )
+
+        if video_id:
+            repo = self.dataset_path or self.DEFAULT_HF_DATASET_ID
+            rel = self._official_video_relpath(video_id)
+            hf_video_url = f"https://huggingface.co/datasets/{repo}/resolve/main/Videos/{rel}"
+            logger.debug(
+                "Using HF video URL (likely 404 — Hub ships Videos.tar only): %s",
+                hf_video_url,
+            )
+            return {"type": "video_url", "video_url": {"url": hf_video_url}}
+
+        logger.error("Could not determine video source for video_id=%r", video_id)
+        return None
+
+    def _get_audio_content(self, video_id: str) -> dict[str, Any] | None:
+        """Resolve extracted WAV for OpenAI-style ``audio_url`` (local files only)."""
+        if not self.video_dir or not video_id:
+            logger.warning(
+                "Daily-Omni input_mode %r requires --daily-omni-video-dir with %s",
+                self.input_mode,
+                self._official_audio_relpath(video_id),
+            )
+            return None
+        audio_path = self._resolve_local_audio_path(video_id)
+        if audio_path is not None:
+            return self._local_file_to_audio_url_payload(audio_path)
+        logger.warning(
+            "Audio not found under video_dir=%s for video_id=%r (expected %s or %s)",
+            self.video_dir,
+            video_id,
+            self._official_audio_relpath(video_id),
+            f"{video_id}.wav",
+        )
+        return None
+
+    def _compose_daily_omni_multimodal(
+        self,
+        video_id: str,
+        video_url: str | None,
+    ) -> tuple[dict[str, Any] | list[dict[str, Any]] | None, dict[str, Any] | None, Literal["first", "last"]]:
+        """Build ``multi_modal_data`` and request extras for the active ``input_mode``.
+
+        Mirrors upstream Daily-Omni: separate video + WAV with ``use_audio_in_video=False``.
+        """
+        extra: dict[str, Any] = {"mm_processor_kwargs": {"use_audio_in_video": False}}
+        mode = self.input_mode
+
+        if mode == "visual":
+            v = self._get_video_content(video_id, video_url)
+            return v, extra, "last"
+
+        if mode == "audio":
+            a = self._get_audio_content(video_id)
+            return a, extra, "first"
+
+        v = self._get_video_content(video_id, video_url)
+        a = self._get_audio_content(video_id)
+        if not v or not a:
+            return None, None, "first"
+        return [v, a], extra, "first"
+
+    @staticmethod
+    def _media_desc_for_official_prompt(mode: DailyOmniInputMode) -> str:
+        """``media_desc`` in upstream ``build_conversation``."""
+        if mode == "audio":
+            return "given audio"
+        if mode == "all":
+            return "given video and audio together"
+        return "given video"
+
+    @staticmethod
+    def _choices_repr_for_official_prompt(choice: Any) -> str:
+        """Format ``Choice`` from qa.json for the model (one option per line when possible).
+
+        Using ``str(list)`` embeds Python list brackets and quotes, which is poor for MCQ
+        reading; lists/tuples are joined with newlines instead. Other shapes fall back to
+        ``str(choice)`` for parity with exotic upstream payloads.
+        """
+        if choice is None:
+            return ""
+        if isinstance(choice, (list, tuple)):
+            lines = [str(x).strip() for x in choice if str(x).strip()]
+            return "\n".join(lines)
+        if isinstance(choice, dict):
+            return "\n".join(f"{k}. {v}" for k, v in choice.items())
+        return str(choice)
+
+    def _official_daily_omni_user_prompt(self, question: str, choice: Any) -> str:
+        """User text block from Daily-Omni ``build_conversation`` (after media parts)."""
+        task_prompt = self._media_desc_for_official_prompt(self.input_mode)
+        choices = self._choices_repr_for_official_prompt(choice)
+        # Single f-string with explicit newlines avoids accidental implicit concatenation
+        # gluing sentences (e.g. ``...media_desc.Select...``) when editing.
+        return (
+            "Your task is to accurately answer multiple-choice questions "
+            f"based on the {task_prompt}.\n"
+            "Select the single most accurate answer from the given choices.\n"
+            f"Question: {question}\n"
+            f"Choices: {choices}\n"
+            "Your answer should be a capital letter representing your choice: "
+            "A, B, C, or D. Don't generate any other text.\n"
+        )
+
+    def _build_daily_omni_openai_messages(
+        self,
+        mm_payload: dict[str, Any] | list[dict[str, Any]],
+        question: str,
+        choice: Any,
+    ) -> list[dict[str, Any]]:
+        """Map upstream conversation to OpenAI Chat Completions ``messages`` (video_url / audio_url parts)."""
+        user_text = self._official_daily_omni_user_prompt(question, choice)
+        mm_list: list[dict[str, Any]] = mm_payload if isinstance(mm_payload, list) else [mm_payload]
+        user_content: list[dict[str, Any]] = [*mm_list, {"type": "text", "text": user_text}]
+        return [
+            {"role": "system", "content": [{"type": "text", "text": DAILY_OMNI_SYSTEM_TEXT}]},
+            {"role": "user", "content": user_content},
+        ]
+
+    def sample_by_task_type(
+        self,
+        tokenizer: TokenizerLike,
+        task_type: str,
+        num_samples: int,
+        output_len: int | None = None,
+        request_id_prefix: str = "",
+        **kwargs,
+    ) -> list[SampleRequest]:
+        """Sample requests filtered by task type.
+
+        Args:
+            tokenizer: Tokenizer
+            task_type: Task type to filter by
+            num_samples: Number of samples
+            output_len: Target output length
+            request_id_prefix: Prefix for request IDs
+            **kwargs: Additional sampling arguments
+
+        Returns:
+            List of SampleRequest objects matching the task type
+        """
+        if output_len is None:
+            output_len = self.DEFAULT_OUTPUT_LEN
+
+        filtered = [
+            item for item in self.data if self._normalize_qa_fields(self._coerce_row(item))["task_type"] == task_type
+        ]
+
+        available = len(filtered)
+        if available < num_samples:
+            logger.warning(
+                "Only %d samples available for task type '%s', requested %d",
+                available,
+                task_type,
+                num_samples,
+            )
+            num_samples = available
+
+        sampled_requests: list[SampleRequest] = []
+        cached_tokenizer = get_cached_tokenizer(tokenizer)
+
+        for i, item in enumerate(filtered[:num_samples]):
+            request = self._create_sample_request(item, cached_tokenizer, output_len, request_id_prefix, i)
+            if request:
+                sampled_requests.append(request)
+
+        return sampled_requests
+
+    def __repr__(self) -> str:
+        return (
+            f"DailyOmniDataset("
+            f"dataset_path={self.dataset_path!r}, "
+            f"dataset_split={self.dataset_split!r}, "
+            f"video_dir={self.video_dir!r}, "
+            f"input_mode={self.input_mode!r}, "
+            f"inline_local_video={self.inline_local_video!r}, "
+            f"max_duration_seconds={self.max_duration_seconds}, "
+            f"random_seed={self.random_seed}"
+            f")"
+        )
+
+
+def load_daily_omni_dataset(
+    qa_json_path: str | None = None,
+    dataset_path: str | None = None,
+    dataset_split: str = "train",
+    random_seed: int = 0,
+    video_dir: str | None = None,
+    input_mode: DailyOmniInputMode = "all",
+    max_duration_seconds: float | None = None,
+    dataset_subset: str | None = None,
+    no_stream: bool = False,
+    **kwargs,
+) -> DailyOmniDataset:
+    """Convenience function to load Daily-Omni dataset.
+
+    Args:
+        qa_json_path: Path to local qa.json file (recommended for offline/air-gapped environments).
+            When provided, ``dataset_path`` is ignored.
+        dataset_path: HuggingFace dataset path (default: liarliar/Daily-Omni). Used only if
+            ``qa_json_path`` is not provided (legacy online mode).
+        dataset_split: Dataset split to use (default: "train")
+        random_seed: Random seed for shuffling
+        video_dir: Directory containing extracted ``Videos/`` tree (MP4 and, for ``all``/``audio``, WAV)
+        input_mode: ``visual`` | ``audio`` | ``all`` (same semantics as upstream Daily-Omni)
+        max_duration_seconds: Maximum video duration in seconds (e.g., 30 for 30s subset, 60 for 60s subset);
+            uses ffprobe on local files under ``video_dir`` (in-memory cache only for this process).
+        **kwargs: Additional arguments passed to DailyOmniDataset
+
+    Returns:
+        DailyOmniDataset instance
+
+    Example:
+        >>> from vllm_omni.benchmarks.data_modules.daily_omni_dataset import load_daily_omni_dataset
+
+        # Local JSON mode (recommended for offline)
+        >>> dataset = load_daily_omni_dataset(
+        ...     qa_json_path="/path/to/qa.json",
+        ...     video_dir="/path/to/Daily-Omni/Videos",
+        ...     random_seed=42,
+        ...     max_duration_seconds=30,
+        ... )
+
+        # HuggingFace mode (legacy online)
+        >>> dataset = load_daily_omni_dataset(
+        ...     dataset_path="liarliar/Daily-Omni",
+        ...     video_dir="/path/to/Daily-Omni/Videos",
+        ...     random_seed=42,
+        ... )
+        >>> requests = dataset.sample(tokenizer, num_requests=100)
+    """
+    return DailyOmniDataset(
+        qa_json_path=qa_json_path,
+        dataset_path=dataset_path,
+        dataset_split=dataset_split,
+        random_seed=random_seed,
+        video_dir=video_dir,
+        input_mode=input_mode,
+        max_duration_seconds=max_duration_seconds,
+        dataset_subset=dataset_subset,
+        no_stream=no_stream,
+        **kwargs,
+    )
+
+
+def get_daily_omni_statistics(
+    qa_json_path: str | None = None,
+    dataset_path: str | None = DailyOmniDataset.DEFAULT_HF_DATASET_ID,
+    dataset_split: str = "train",
+) -> dict[str, Any]:
+    """Get statistics about the Daily-Omni dataset.
+
+    Args:
+        qa_json_path: Path to local qa.json file (recommended for offline/air-gapped environments).
+            When provided, ``dataset_path`` is ignored.
+        dataset_path: HuggingFace dataset path. Defaults to ``DailyOmniDataset.DEFAULT_HF_DATASET_ID``
+            when ``qa_json_path`` is omitted. Pass ``None`` only together with ``qa_json_path``.
+        dataset_split: Dataset split to use (default: "train")
+
+    Returns:
+        Statistics dict with task type distribution and other info
+
+    Example:
+        >>> from vllm_omni.benchmarks.data_modules.daily_omni_dataset import get_daily_omni_statistics
+
+        # Local JSON mode
+        >>> stats = get_daily_omni_statistics(qa_json_path="/path/to/qa.json")
+
+        # HuggingFace mode
+        >>> stats = get_daily_omni_statistics(dataset_path="liarliar/Daily-Omni")
+        >>> print(f"Total QA pairs: {stats['total_qa_pairs']}")
+        >>> print(f"Task distribution: {stats['task_distribution']}")
+    """
+    dataset = DailyOmniDataset(
+        qa_json_path=qa_json_path,
+        dataset_path=dataset_path,
+        dataset_split=dataset_split,
+    )
+    task_stats = dataset.get_task_statistics()
+
+    source = str(qa_json_path) if qa_json_path else f"{dataset_path}/{dataset_split}"
+    return {
+        "source": source,
+        "total_qa_pairs": len(list(dataset.data)),
+        "task_distribution": task_stats,
+    }

vllm_omni/benchmarks/data_modules/daily_omni_eval.py

@@ -0,0 +1,406 @@
+"""Daily-Omni multiple-choice accuracy scoring for vLLM-Omni bench serve.
+
+Compares model ``generated_text`` to dataset ``Answer`` (A/B/C/D).
+
+**Alignment with open-source** (`Lliar-liar/Daily-Omni` ``test_model/.../testmodel.py``):
+
+- Answer extraction defaults to the same rules as ``extract_choice_letter`` (strip after an
+  ``assistant`` marker, then leading ``A``–``D``, else first ``\\b[A-D]\\b``). Set env
+  ``DAILY_OMNI_EXTRACT_MODE=relaxed`` to use the older vLLM-Omni heuristics (last ``answer:``,
+  tail scan, etc.).
+- Overall accuracy comparable to the official script uses **successful HTTP responses only** as
+  the denominator (their ``valid_questions = total - failed`` excludes inference / I/O skips).
+  We also report ``daily_omni_accuracy_incl_http_fail`` where each failed request counts as a
+  wrong answer in the denominator (stricter throughput-bench view).
+- **By video length:** mirrors upstream ``--- Accuracy by Video Duration ---`` for ``30s`` /
+  ``60s`` (``qa.json`` ``video_duration``): ``daily_omni_per_duration*`` metrics and a printed block.
+- **By video category:** mirrors ``--- Accuracy by Video Category ---`` using ``video_category``
+  from ``qa.json`` (``daily_omni_per_category*``; empty category is bucketed as ``unknown``).
+- **Correctness:** uses the same ``evaluate_answer`` rule as upstream (truthy extracted letter vs
+  raw ``Answer`` string, both ``strip().upper()``). Rows with empty ``Answer`` are skipped
+  (``no_gold``), matching missing-field skips in the official loop.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+
+from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniSampleRequest
+
+_VALID = frozenset("ABCD")
+
+# Official ``testmodel.py`` buckets (``qa.json`` ``video_duration``).
+DAILY_OMNI_DURATION_KEYS: tuple[str, ...] = ("30s", "60s")
+
+
+def extract_choice_letter_official(text: str | None) -> str | None:
+    """Port of Daily-Omni ``extract_choice_letter`` (first A–D, assistant-tail semantics)."""
+    if not text:
+        return None
+    raw = str(text).strip()
+    if not raw:
+        return None
+    match = re.search(r"assistant\s*([\s\S]*)$", raw, flags=re.IGNORECASE)
+    candidate = match.group(1).strip() if match else raw
+    direct = re.match(r"(?i)^\s*([A-D])(?:[\s\.\):：]|$)", candidate)
+    if direct:
+        return direct.group(1).upper()
+    fallback = re.search(r"\b([A-D])\b", candidate.upper())
+    if fallback:
+        return fallback.group(1)
+    return None
+
+
+def evaluate_answer_official(model_answer: str | None, correct_answer: str) -> bool:
+    """Port of Daily-Omni ``evaluate_answer`` (strict string match after strip/upper)."""
+    if not model_answer:
+        return False
+    return model_answer.strip().upper() == (correct_answer or "").strip().upper()
+
+
+def normalize_gold_answer(gold: str) -> str | None:
+    """Best-effort single letter from ``Answer`` (for ``gold_normalized`` in saved items only)."""
+    g = (gold or "").strip().upper()
+    if len(g) == 1 and g in _VALID:
+        return g
+    m = re.search(r"([ABCD])\b", g)
+    if m:
+        return m.group(1).upper()
+    return None
+
+
+def _extract_predicted_choice_relaxed(text: str) -> str | None:
+    """Legacy vLLM-Omni heuristics (last ``answer:`` patterns, tail scan)."""
+    if not text or not str(text).strip():
+        return None
+    t = str(text).strip()
+
+    strong_patterns = [
+        r"(?i)\*\*answer\*\*\s*[:：]?\s*\(?([ABCD])\)?",
+        r"(?i)\banswer\s*[:：]?\s*\(?([ABCD])\)?",
+        r"(?i)\bfinal\s+answer\s*[:：]?\s*\(?([ABCD])\)?",
+        r"(?i)\bcorrect\s+(?:answer|option)\s*[:：]?\s*\(?([ABCD])\)?",
+        r"(?i)\bthe\s+(?:correct\s+)?option\s+(?:is|would\s+be)\s*\(?([ABCD])\)?",
+        r"(?i)\bI\s+(?:would\s+)?(?:choose|select|pick)\s*\(?([ABCD])\)?",
+    ]
+    last_letter: str | None = None
+    for pat in strong_patterns:
+        for m in re.finditer(pat, t):
+            last_letter = m.group(1).upper()
+    if last_letter:
+        return last_letter
+
+    # Weaker phrases: first match can be spurious; still prefer last occurrence.
+    weak_patterns = [
+        r"(?i)\boption\s*[:：]?\s*\(?([ABCD])\)?",
+        r"(?i)\bchoice\s*[:：]?\s*\(?([ABCD])\)?",
+    ]
+    for pat in weak_patterns:
+        for m in re.finditer(pat, t):
+            last_letter = m.group(1).upper()
+    if last_letter:
+        return last_letter
+
+    paren = list(re.finditer(r"\(([ABCD])\)", t))
+    if paren:
+        return paren[-1].group(1).upper()
+
+    # First line sometimes is just "B" or "B." — allow if whole output is short
+    one_line = t.split("\n", 1)[0].strip()
+    if len(t) < 120 and len(one_line) <= 6:
+        m0 = re.match(r"^([ABCD])\s*[.:\)]?\s*$", one_line, re.I)
+        if m0:
+            return m0.group(1).upper()
+
+    # Tail-only: avoids matching echoed "A. ..." option blocks at the start
+    tail_len = min(500, len(t))
+    tail = t[-tail_len:]
+    # ``\b`` after the letter avoids "Because"/"Definitely" false positives
+    m = re.search(r"(?:^|[^\w])([ABCD])\b", tail, re.I)
+    if m:
+        return m.group(1).upper()
+
+    return None
+
+
+def extract_predicted_choice(text: str | None) -> str | None:
+    """Parse model output to A–D (official Daily-Omni rules by default)."""
+    if not text or not str(text).strip():
+        return None
+    mode = os.environ.get("DAILY_OMNI_EXTRACT_MODE", "official").strip().lower()
+    if mode in ("relaxed", "heuristic", "legacy"):
+        return _extract_predicted_choice_relaxed(str(text))
+    return extract_choice_letter_official(text)
+
+
+def compute_daily_omni_accuracy_metrics(
+    input_requests: list[Any],
+    outputs: list[RequestFuncOutput],
+    *,
+    include_per_item: bool = False,
+) -> dict[str, Any] | None:
+    """If all requests are :class:`DailyOmniSampleRequest`, compute accuracy stats.
+
+    Rows with empty ``Answer`` (after strip) are skipped as ``no_gold``, like upstream missing
+    ``correct_answer``.
+
+    **Denominators:** The open-source script excludes items that hit inference / I/O failures
+    from ``valid_questions``; we mirror that with ``daily_omni_accuracy`` (= correct /
+    successful responses). Failed HTTP requests are also tracked and used in
+    ``daily_omni_accuracy_incl_http_fail`` (each failure counts as incorrect in the
+    denominator).
+    """
+    if not input_requests or len(input_requests) != len(outputs):
+        return None
+    if not all(isinstance(r, DailyOmniSampleRequest) for r in input_requests):
+        return None
+
+    # total / correct: all rows with gold (incl. HTTP fail in total)
+    # total_ok / correct_ok: successful HTTP only (GitHub-style per-type denominator)
+    per_task: dict[str, dict[str, int]] = {}
+    per_category: dict[str, dict[str, int]] = {}
+    per_duration: dict[str, dict[str, int]] = {
+        k: {"correct": 0, "total": 0, "correct_ok": 0, "total_ok": 0} for k in DAILY_OMNI_DURATION_KEYS
+    }
+    items: list[dict[str, Any]] = []
+    correct = 0
+    evaluated = 0
+    no_gold = 0
+    request_failed = 0
+    parse_failed = 0  # success but could not extract A–D
+
+    for req, out in zip(input_requests, outputs, strict=True):
+        assert isinstance(req, DailyOmniSampleRequest)
+        gold_raw = (req.daily_omni_gold_answer or "").strip()
+        gold_norm = normalize_gold_answer(req.daily_omni_gold_answer)
+        tt = (req.daily_omni_task_type or "unknown").strip() or "unknown"
+        dur_key = (req.daily_omni_video_duration or "").strip()
+        dur_active = dur_key in per_duration
+        cat_key = (req.daily_omni_video_category or "").strip() or "unknown"
+        if tt not in per_task:
+            per_task[tt] = {"correct": 0, "total": 0, "correct_ok": 0, "total_ok": 0}
+        if cat_key not in per_category:
+            per_category[cat_key] = {"correct": 0, "total": 0, "correct_ok": 0, "total_ok": 0}
+
+        if not gold_raw:
+            no_gold += 1
+            items.append(
+                {
+                    "request_id": req.request_id,
+                    "skipped": True,
+                    "reason": "no_gold",
+                    "task_type": tt,
+                    "video_id": req.daily_omni_video_id,
+                    "video_duration": dur_key or None,
+                    "video_category": cat_key if cat_key != "unknown" else None,
+                }
+            )
+            continue
+
+        if not out.success:
+            request_failed += 1
+            evaluated += 1
+            per_task[tt]["total"] += 1
+            per_category[cat_key]["total"] += 1
+            if dur_active:
+                per_duration[dur_key]["total"] += 1
+            # GitHub: failed inference not in valid_questions — do not increment total_ok
+            items.append(
+                {
+                    "request_id": req.request_id,
+                    "gold": gold_raw,
+                    "gold_normalized": gold_norm,
+                    "predicted": None,
+                    "correct": False,
+                    "task_type": tt,
+                    "video_id": req.daily_omni_video_id,
+                    "video_duration": dur_key or None,
+                    "video_category": cat_key if cat_key != "unknown" else None,
+                    "error": (out.error or "")[:500],
+                }
+            )
+            continue
+
+        pred = extract_predicted_choice(out.generated_text)
+        evaluated += 1
+        per_task[tt]["total"] += 1
+        per_task[tt]["total_ok"] += 1
+        per_category[cat_key]["total"] += 1
+        per_category[cat_key]["total_ok"] += 1
+        if dur_active:
+            per_duration[dur_key]["total"] += 1
+            per_duration[dur_key]["total_ok"] += 1
+        if pred is None:
+            parse_failed += 1
+        is_correct = evaluate_answer_official(pred, req.daily_omni_gold_answer)
+        if is_correct:
+            correct += 1
+            per_task[tt]["correct"] += 1
+            per_task[tt]["correct_ok"] += 1
+            per_category[cat_key]["correct"] += 1
+            per_category[cat_key]["correct_ok"] += 1
+            if dur_active:
+                per_duration[dur_key]["correct"] += 1
+                per_duration[dur_key]["correct_ok"] += 1
+
+        items.append(
+            {
+                "request_id": req.request_id,
+                "gold": gold_raw,
+                "gold_normalized": gold_norm,
+                "predicted": pred,
+                "correct": is_correct,
+                "parse_failed": pred is None,
+                "task_type": tt,
+                "video_id": req.daily_omni_video_id,
+                "video_duration": dur_key or None,
+                "video_category": cat_key if cat_key != "unknown" else None,
+            }
+        )
+
+    evaluated_ok = evaluated - request_failed
+    accuracy_github = (correct / evaluated_ok) if evaluated_ok else None
+    accuracy_incl_fail = (correct / evaluated) if evaluated else None
+
+    per_task_accuracy: dict[str, float | None] = {}
+    per_task_accuracy_github: dict[str, float | None] = {}
+    for name, st in per_task.items():
+        tot = st["total"]
+        per_task_accuracy[name] = (st["correct"] / tot) if tot else None
+        tok = st["total_ok"]
+        per_task_accuracy_github[name] = (st["correct_ok"] / tok) if tok else None
+
+    per_category_accuracy: dict[str, float | None] = {}
+    per_category_accuracy_github: dict[str, float | None] = {}
+    for name, st in per_category.items():
+        tot = st["total"]
+        per_category_accuracy[name] = (st["correct"] / tot) if tot else None
+        tok = st["total_ok"]
+        per_category_accuracy_github[name] = (st["correct_ok"] / tok) if tok else None
+
+    per_duration_accuracy: dict[str, float | None] = {}
+    per_duration_accuracy_github: dict[str, float | None] = {}
+    for name, st in per_duration.items():
+        tot = st["total"]
+        per_duration_accuracy[name] = (st["correct"] / tot) if tot else None
+        tok = st["total_ok"]
+        per_duration_accuracy_github[name] = (st["correct_ok"] / tok) if tok else None
+
+    out: dict[str, Any] = {
+        # Comparable to GitHub testmodel.py: correct / successful inferences
+        "daily_omni_accuracy": accuracy_github,
+        "daily_omni_accuracy_incl_http_fail": accuracy_incl_fail,
+        "daily_omni_correct": correct,
+        "daily_omni_evaluated": evaluated,
+        "daily_omni_evaluated_ok": evaluated_ok,
+        "daily_omni_no_gold": no_gold,
+        "daily_omni_request_failed": request_failed,
+        "daily_omni_parse_failed": parse_failed,
+        "daily_omni_per_task": {k: dict(v) for k, v in per_task.items()},
+        "daily_omni_per_task_accuracy": per_task_accuracy,
+        "daily_omni_per_task_accuracy_github_style": per_task_accuracy_github,
+        "daily_omni_per_category": {k: dict(v) for k, v in per_category.items()},
+        "daily_omni_per_category_accuracy": per_category_accuracy,
+        "daily_omni_per_category_accuracy_github_style": per_category_accuracy_github,
+        "daily_omni_per_duration": {k: dict(v) for k, v in per_duration.items()},
+        "daily_omni_per_duration_accuracy": per_duration_accuracy,
+        "daily_omni_per_duration_accuracy_github_style": per_duration_accuracy_github,
+    }
+    if include_per_item:
+        out["daily_omni_eval_items"] = items
+    return out
+
+
+def print_daily_omni_accuracy_summary(metrics: dict[str, Any]) -> None:
+    """Pretty-print accuracy block (stdout)."""
+    acc = metrics.get("daily_omni_accuracy")
+    acc_fail = metrics.get("daily_omni_accuracy_incl_http_fail")
+    if acc is None and acc_fail is None and metrics.get("daily_omni_evaluated", 0) == 0:
+        return
+    print("{s:{c}^{n}}".format(s=" Daily-Omni accuracy (MCQ) ", n=50, c="="))
+    ok = int(metrics.get("daily_omni_evaluated_ok", 0) or 0)
+    cor = int(metrics.get("daily_omni_correct", 0) or 0)
+    if ok > 0 and acc is not None:
+        print(f"Overall Accuracy: {cor}/{ok} = {acc:.2%}")
+    elif int(metrics.get("daily_omni_evaluated", 0) or 0) > 0:
+        print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)")
+    print(
+        "{:<40} {:<10}".format(
+            "Submitted (gold present):",
+            metrics.get("daily_omni_evaluated", 0),
+        )
+    )
+    print(
+        "{:<40} {:<10}".format(
+            "Successful HTTP (GitHub denom.):",
+            metrics.get("daily_omni_evaluated_ok", 0),
+        )
+    )
+    print("{:<40} {:<10}".format("Correct:", metrics.get("daily_omni_correct", 0)))
+    if acc is not None:
+        print("{:<40} {:<10.4f}".format("Accuracy (ratio, same as above):", acc))
+    if acc_fail is not None and metrics.get("daily_omni_request_failed", 0):
+        print(
+            "{:<40} {:<10.4f}".format(
+                "Accuracy (incl. HTTP as wrong):",
+                acc_fail,
+            )
+        )
+    print("{:<40} {:<10}".format("Skipped (no gold):", metrics.get("daily_omni_no_gold", 0)))
+    print(
+        "{:<40} {:<10}".format(
+            "HTTP failed (excl. from GitHub acc.):",
+            metrics.get("daily_omni_request_failed", 0),
+        )
+    )
+    print(
+        "{:<40} {:<10}".format(
+            "Parsed OK but no A–D found:",
+            metrics.get("daily_omni_parse_failed", 0),
+        )
+    )
+    pt = metrics.get("daily_omni_per_task") or {}
+    pta = metrics.get("daily_omni_per_task_accuracy_github_style") or {}
+    if pta:
+        print("\n--- Accuracy by QA Type ---")
+        for name in sorted(pta.keys()):
+            a = pta[name]
+            st = pt.get(name) or {}
+            tok = int(st.get("total_ok", 0) or 0)
+            cok = int(st.get("correct_ok", 0) or 0)
+            if tok and a is not None:
+                print(f"{name}: {cok}/{tok} = {a:.2%}")
+            else:
+                print(f"{name}: 0/0 = N/A")
+
+    pc = metrics.get("daily_omni_per_category") or {}
+    ptc = metrics.get("daily_omni_per_category_accuracy_github_style") or {}
+    if ptc:
+        print("\n--- Accuracy by Video Category ---")
+        for name in sorted(ptc.keys()):
+            a = ptc[name]
+            st = pc.get(name) or {}
+            tok = int(st.get("total_ok", 0) or 0)
+            cok = int(st.get("correct_ok", 0) or 0)
+            if tok and a is not None:
+                print(f"{name}: {cok}/{tok} = {a:.2%}")
+            else:
+                print(f"{name}: 0/0 = N/A")
+
+    pdf = metrics.get("daily_omni_per_duration_accuracy_github_style") or {}
+    if pdf:
+        print("\n--- Accuracy by Video Duration ---")
+        for name in DAILY_OMNI_DURATION_KEYS:
+            a = pdf.get(name)
+            st = (metrics.get("daily_omni_per_duration") or {}).get(name) or {}
+            tok = int(st.get("total_ok", 0) or 0)
+            cor = int(st.get("correct_ok", 0) or 0)
+            if tok and a is not None:
+                print(f"{name} Duration: {cor}/{tok} = {a:.2%}")
+            else:
+                print(f"{name} Duration: 0/0 = N/A")
+    print("=" * 50)

vllm_omni/benchmarks/data_modules/daily_omni_text_audio.py

@@ -0,0 +1,255 @@
+"""Daily-Omni: optional consistency check between text stream and generated speech.
+
+The benchmark MCQ accuracy uses ``generated_text`` only. When the omni server also
+streams ``modality=audio`` (TTS), this module can transcribe the concatenated WAV
+with Whisper and compare the inferred option letter to the one parsed from text.
+
+Requires ``openai-whisper`` (``pip install openai-whisper``). Enable via env
+``DAILY_OMNI_TEXT_AUDIO_CONSISTENCY=1`` or CLI ``--daily-omni-text-audio-consistency``.
+
+Whisper model name defaults to ``tiny`` (override with ``DAILY_OMNI_WHISPER_MODEL``).
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import threading
+from typing import Any
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniSampleRequest
+from vllm_omni.benchmarks.data_modules.daily_omni_eval import extract_predicted_choice
+
+logger = logging.getLogger(__name__)
+
+_whisper_model = None
+_whisper_model_name: str | None = None
+_whisper_lock = threading.Lock()
+
+
+def env_text_audio_check_enabled() -> bool:
+    return os.environ.get("DAILY_OMNI_TEXT_AUDIO_CONSISTENCY", "").lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+
+
+def extract_choice_from_asr_transcript(transcript: str) -> str | None:
+    """Parse A–D from ASR text; extends :func:`extract_predicted_choice` with spoken Chinese phrases."""
+    c = extract_predicted_choice(transcript)
+    if c:
+        return c
+    t = transcript or ""
+    for pat in (
+        r"(?i)选项\s*([ABCD])\b",
+        r"(?i)选\s*([ABCD])\b",
+        r"(?i)答案\s*是\s*([ABCD])\b",
+        r"(?i)答案\s*([ABCD])\b",
+    ):
+        m = re.search(pat, t)
+        if m:
+            return m.group(1).upper()
+    return None
+
+
+def _get_whisper_model(model_name: str):
+    global _whisper_model, _whisper_model_name
+    with _whisper_lock:
+        if _whisper_model is None or _whisper_model_name != model_name:
+            import whisper
+
+            logger.warning(
+                "Loading Whisper model %r for Daily-Omni text/audio consistency (one-time)...",
+                model_name,
+            )
+            _whisper_model = whisper.load_model(model_name)
+            _whisper_model_name = model_name
+        return _whisper_model
+
+
+def transcribe_wav_bytes(
+    wav_bytes: bytes,
+    *,
+    language: str | None = None,
+    model_name: str | None = None,
+) -> tuple[str | None, str | None]:
+    """Transcribe WAV bytes. Returns ``(transcript, error)`` — one of them is set.
+
+    Args:
+        wav_bytes: RIFF WAV file bytes.
+        language: Optional Whisper language code (e.g. ``en``, ``zh``); improves accuracy/latency.
+        model_name: Override model id; else ``DAILY_OMNI_WHISPER_MODEL`` or ``tiny``.
+    """
+    if not wav_bytes:
+        return None, "empty_wav"
+    if model_name is None or not str(model_name).strip():
+        model_name = os.environ.get("DAILY_OMNI_WHISPER_MODEL") or "tiny"
+    model_name = str(model_name).strip() or "tiny"
+    path: str | None = None
+    try:
+        import tempfile
+
+        model = _get_whisper_model(model_name)
+        fd, path = tempfile.mkstemp(suffix=".wav")
+        with os.fdopen(fd, "wb") as fp:
+            fp.write(wav_bytes)
+        kwargs: dict = {}
+        if language:
+            kwargs["language"] = language
+        result = model.transcribe(path, **kwargs)
+        text = (result.get("text") or "").strip()
+        return (text if text else None), None
+    except ImportError:
+        return None, "openai-whisper is not installed (pip install openai-whisper)"
+    except Exception as e:
+        return None, str(e)[:500]
+    finally:
+        if path:
+            try:
+                os.unlink(path)
+            except OSError:
+                pass
+
+
+def compute_daily_omni_text_audio_consistency_metrics(
+    input_requests: list[Any],
+    outputs: list[Any],
+    *,
+    include_per_item: bool = False,
+) -> dict[str, Any] | None:
+    """Compare option letter from ``generated_text`` vs Whisper transcript of output audio.
+
+    Only considers requests where ``outputs[i]`` has ``generated_audio_wav_bytes`` set
+    (populated by the omni benchmark when TA check is enabled).
+    """
+    if not input_requests or len(input_requests) != len(outputs):
+        return None
+    if not all(isinstance(r, DailyOmniSampleRequest) for r in input_requests):
+        return None
+
+    ta_no_wav = 0
+    ta_asr_failed = 0
+    ta_text_unparsed = 0
+    ta_audio_unparsed = 0
+    ta_consistent = 0
+    ta_mismatch = 0
+    ta_both_parsed = 0
+    items: list[dict[str, Any]] = []
+
+    for req, out in zip(input_requests, outputs, strict=True):
+        assert isinstance(req, DailyOmniSampleRequest)
+        rid = req.request_id
+        if not getattr(out, "success", False):
+            if include_per_item:
+                items.append(
+                    {
+                        "request_id": rid,
+                        "skipped": True,
+                        "reason": "request_not_success",
+                    }
+                )
+            continue
+
+        wav = getattr(out, "generated_audio_wav_bytes", None)
+        if not wav:
+            ta_no_wav += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "request_id": rid,
+                        "skipped": False,
+                        "reason": "no_output_audio",
+                        "text_choice": extract_predicted_choice(getattr(out, "generated_text", "") or ""),
+                    }
+                )
+            continue
+
+        transcript, asr_err = transcribe_wav_bytes(wav)
+        if asr_err:
+            ta_asr_failed += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "request_id": rid,
+                        "asr_error": asr_err,
+                        "text_choice": extract_predicted_choice(getattr(out, "generated_text", "") or ""),
+                    }
+                )
+            continue
+
+        text_choice = extract_predicted_choice(getattr(out, "generated_text", "") or "")
+        audio_choice = extract_choice_from_asr_transcript(transcript or "")
+
+        if text_choice is None:
+            ta_text_unparsed += 1
+        if audio_choice is None:
+            ta_audio_unparsed += 1
+
+        if text_choice is not None and audio_choice is not None:
+            ta_both_parsed += 1
+            if text_choice == audio_choice:
+                ta_consistent += 1
+            else:
+                ta_mismatch += 1
+
+        if include_per_item:
+            consistent: bool | None
+            if text_choice is None or audio_choice is None:
+                consistent = None
+            else:
+                consistent = text_choice == audio_choice
+            items.append(
+                {
+                    "request_id": rid,
+                    "text_choice": text_choice,
+                    "audio_choice": audio_choice,
+                    "asr_transcript": (transcript or "")[:500],
+                    "text_audio_consistent": consistent,
+                }
+            )
+
+    comparable = ta_consistent + ta_mismatch
+    rate = (ta_consistent / comparable) if comparable else None
+
+    out: dict[str, Any] = {
+        "daily_omni_ta_enabled": True,
+        "daily_omni_ta_no_output_audio": ta_no_wav,
+        "daily_omni_ta_asr_failed": ta_asr_failed,
+        "daily_omni_ta_text_unparsed": ta_text_unparsed,
+        "daily_omni_ta_audio_unparsed": ta_audio_unparsed,
+        "daily_omni_ta_both_parsed": ta_both_parsed,
+        "daily_omni_ta_consistent": ta_consistent,
+        "daily_omni_ta_mismatch": ta_mismatch,
+        "daily_omni_ta_consistency_rate": rate,
+    }
+    if include_per_item:
+        out["daily_omni_ta_items"] = items
+    return out
+
+
+def print_daily_omni_text_audio_summary(metrics: dict[str, Any]) -> None:
+    if not metrics.get("daily_omni_ta_enabled"):
+        return
+    print("{s:{c}^{n}}".format(s=" Daily-Omni text vs audio (ASR) ", n=50, c="="))
+    print("{:<40} {:<10}".format("No output audio captured:", metrics.get("daily_omni_ta_no_output_audio", 0)))
+    print("{:<40} {:<10}".format("ASR failed:", metrics.get("daily_omni_ta_asr_failed", 0)))
+    print("{:<40} {:<10}".format("Both text+audio letter parsed:", metrics.get("daily_omni_ta_both_parsed", 0)))
+    print("{:<40} {:<10}".format("Consistent (same letter):", metrics.get("daily_omni_ta_consistent", 0)))
+    print("{:<40} {:<10}".format("Mismatch:", metrics.get("daily_omni_ta_mismatch", 0)))
+    r = metrics.get("daily_omni_ta_consistency_rate")
+    if r is not None:
+        print("{:<40} {:<10.4f}".format("Consistency rate (of both parsed):", r))
+    print(
+        "{:<40} {:<10}".format(
+            "Text unparsed (among w/ audio):",
+            metrics.get("daily_omni_ta_text_unparsed", 0),
+        )
+    )
+    print(
+        "{:<40} {:<10}".format(
+            "Audio unparsed (among w/ audio):",
+            metrics.get("daily_omni_ta_audio_unparsed", 0),
+        )
+    )

vllm_omni/benchmarks/data_modules/seed_tts_dataset.py

@@ -0,0 +1,272 @@
+"""Seed-TTS zero-shot evaluation-style prompts for ``vllm bench serve``.
+
+Loads rows from the `meta.lst` format used in `BytedanceSpeech/seed-tts-eval`_ (or any
+HuggingFace dataset repo with the same layout)::
+
+    utt_id|prompt_transcript|prompt_wav_relative_path|text_to_synthesize
+
+Each benchmark request supplies target text plus ``ref_text`` / ``ref_audio`` (Qwen3-TTS ``Base`` /
+voice clone), merged into the JSON body. By default ``ref_audio`` is an inline ``data:`` URL so
+the server does not need ``--allowed-local-media-path``. Use ``--seed-tts-file-ref-audio`` for
+``file://`` (smaller bodies; requires that flag). Use ``--backend openai-audio-speech``
+(``/v1/audio/speech``) or ``--backend openai-chat-omni`` (``/v1/chat/completions`` with the same
+fields on the body plus a Qwen3-Omni-style ``system`` message and the target text as ``user`` content).
+
+.. _BytedanceSpeech/seed-tts-eval: https://github.com/BytedanceSpeech/seed-tts-eval
+"""
+
+from __future__ import annotations
+
+import base64
+import logging
+import random
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from vllm.benchmarks.datasets import BenchmarkDataset, SampleRequest
+from vllm.tokenizers import TokenizerLike
+from vllm.tokenizers.hf import get_cached_tokenizer
+
+logger = logging.getLogger(__name__)
+
+# Matches Qwen3-Omni serving examples (``openai_chat_completion_client_for_multimodal_generation`` /
+# ``qwen3_omni/gradio_demo``) plus explicit TTS / voice-clone instructions for chat completions.
+SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT = (
+    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
+    "capable of perceiving auditory and visual inputs, as well as generating text and speech.\n"
+    "For this request you act as a text-to-speech engine with zero-shot voice cloning: "
+    "the API provides reference audio and its transcript (ref_audio, ref_text) and task_type Base. "
+    "The user message is the exact text you must speak. "
+    "Synthesize natural speech in the same language as that user text, "
+    "matching the timbre, prosody, and speaking style of the reference audio while reading the new content clearly."
+)
+
+
+@dataclass
+class SeedTTSSampleRequest(SampleRequest):
+    """``SampleRequest`` with per-row fields merged into ``/v1/audio/speech`` JSON."""
+
+    #: Shallow-merged into ``RequestFuncInput.extra_body`` (ref_audio, ref_text, task_type, …).
+    seed_tts_speech_extra: dict[str, Any] | None = None
+    seed_tts_utterance_id: str = ""
+    seed_tts_locale: str = ""
+    #: For ``openai-chat-omni``: becomes the chat ``system`` message (Qwen3-Omni + TTS behavior).
+    seed_tts_system_prompt: str = ""
+    #: Local path to reference prompt WAV (for SIM vs. synthesized PCM in ``seed_tts_eval``).
+    seed_tts_ref_wav_path: str = ""
+
+
+@dataclass
+class _SeedTTSRow:
+    utterance_id: str
+    ref_text: str
+    prompt_wav_rel: str
+    target_text: str
+
+
+def _parse_meta_line(line: str) -> _SeedTTSRow | None:
+    line = line.strip()
+    if not line or line.startswith("#"):
+        return None
+    parts = line.split("|")
+    if len(parts) < 4:
+        logger.warning("Skipping malformed meta.lst line (need 4 '|'-fields): %r", line[:120])
+        return None
+    utt_id, ref_text, wav_rel, target = parts[0], parts[1], parts[2], parts[3]
+    if not target.strip():
+        return None
+    return _SeedTTSRow(
+        utterance_id=utt_id.strip(),
+        ref_text=ref_text.strip(),
+        prompt_wav_rel=wav_rel.strip(),
+        target_text=target.strip(),
+    )
+
+
+def _load_meta_rows(meta_file: Path) -> list[_SeedTTSRow]:
+    text = meta_file.read_text(encoding="utf-8")
+    rows: list[_SeedTTSRow] = []
+    for line in text.splitlines():
+        r = _parse_meta_line(line)
+        if r is not None:
+            rows.append(r)
+    return rows
+
+
+def resolve_seed_tts_root(dataset_path: str | None, *, explicit_root: str | None) -> Path:
+    """Return directory containing ``{locale}/meta.lst`` and ``{locale}/prompt-wavs/``."""
+    if explicit_root:
+        root = Path(explicit_root).expanduser().resolve()
+        if not root.is_dir():
+            raise FileNotFoundError(f"--seed-tts-root is not a directory: {root}")
+        return root
+
+    if not dataset_path:
+        raise ValueError("Seed-TTS requires --dataset-path (HF repo id or local root) or --seed-tts-root.")
+
+    p = Path(dataset_path).expanduser()
+    if p.exists() and p.is_dir():
+        return p.resolve()
+
+    repo_id = dataset_path.strip()
+    try:
+        from huggingface_hub import snapshot_download
+    except ImportError as e:
+        raise ImportError(
+            "Install huggingface_hub to download Seed-TTS from the Hub, or clone the dataset "
+            "locally and pass --dataset-path / --seed-tts-root to that directory."
+        ) from e
+    cache = snapshot_download(repo_id=repo_id, repo_type="dataset")
+    return Path(cache).resolve()
+
+
+def _ref_audio_payload(wav_path: Path, *, inline: bool) -> str:
+    if inline:
+        raw = wav_path.read_bytes()
+        b64 = base64.b64encode(raw).decode("ascii")
+        return f"data:audio/wav;base64,{b64}"
+    return wav_path.expanduser().resolve().as_uri()
+
+
+class SeedTTSDataset(BenchmarkDataset):
+    """Seed-TTS-style zero-shot TTS rows for throughput/latency benchmarking.
+
+    Args:
+        dataset_path: HuggingFace dataset repo id (``org/dataset``) or local directory with
+            ``en/meta.lst`` (and ``zh/meta.lst`` if using zh).
+        locale: ``en`` or ``zh`` — which subfolder under the root to read.
+        inline_ref_audio: If True (default), embed prompt WAV as ``data:audio/wav;base64,...``
+            so Qwen3-TTS / ``/v1/audio/speech`` works without server
+            ``--allowed-local-media-path``. If False, use ``file://`` (smaller
+            requests; server must set ``--allowed-local-media-path`` to the dataset root).
+        seed_tts_root: Optional override for the root directory (same layout as HF dataset).
+        system_prompt: Optional override for the chat system message when using
+            ``--backend openai-chat-omni``; defaults to :data:`SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT`.
+    """
+
+    IS_MULTIMODAL = False
+    DEFAULT_OUTPUT_LEN = 2048
+
+    def __init__(
+        self,
+        dataset_path: str,
+        random_seed: int = 0,
+        locale: str = "en",
+        inline_ref_audio: bool = True,
+        seed_tts_root: str | None = None,
+        system_prompt: str | None = None,
+        disable_shuffle: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        if locale not in ("en", "zh"):
+            raise ValueError("locale must be 'en' or 'zh'")
+        self.locale = locale
+        self.inline_ref_audio = inline_ref_audio
+        self._explicit_root = seed_tts_root
+        sp = (system_prompt or "").strip()
+        self._system_prompt = sp if sp else SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
+        super().__init__(
+            dataset_path=dataset_path,
+            random_seed=random_seed,
+            disable_shuffle=disable_shuffle,
+            **kwargs,
+        )
+        self._root = resolve_seed_tts_root(self.dataset_path, explicit_root=self._explicit_root)
+        self._rows: list[_SeedTTSRow] = []
+        self.load_data()
+
+    def load_data(self) -> None:
+        meta = self._root / self.locale / "meta.lst"
+        if not meta.is_file():
+            raise FileNotFoundError(
+                f"Seed-TTS meta not found: {meta}. "
+                f"Expected layout from seed-tts-eval (e.g. {self._root}/{self.locale}/meta.lst)."
+            )
+        self._rows = _load_meta_rows(meta)
+        if not self._rows:
+            raise ValueError(f"No valid rows in {meta}")
+        if not self.disable_shuffle:
+            rng = random.Random(self.random_seed)
+            rng.shuffle(self._rows)
+        self.data = self._rows
+        logger.info(
+            "Loaded Seed-TTS: root=%s locale=%s rows=%d inline_ref_audio=%s",
+            self._root,
+            self.locale,
+            len(self._rows),
+            self.inline_ref_audio,
+        )
+
+    def sample(
+        self,
+        tokenizer: TokenizerLike,
+        num_requests: int,
+        output_len: int | None = None,
+        request_id_prefix: str = "",
+        no_oversample: bool = False,
+        **kwargs: Any,
+    ) -> list[SampleRequest]:
+        if output_len is None:
+            output_len = self.DEFAULT_OUTPUT_LEN
+
+        tok = get_cached_tokenizer(tokenizer)
+        out: list[SampleRequest] = []
+        for i, row in enumerate(self._rows):
+            if len(out) >= num_requests:
+                break
+            wav_path = (self._root / self.locale / row.prompt_wav_rel).resolve()
+            if not wav_path.is_file():
+                logger.warning("Missing prompt wav for %s: %s", row.utterance_id, wav_path)
+                continue
+
+            target = row.target_text
+            prompt_len = len(tok.encode(f"{self._system_prompt}\n{target}"))
+            lang = "English" if self.locale == "en" else "Chinese"
+            ref_uri = _ref_audio_payload(wav_path, inline=self.inline_ref_audio)
+            speech_extra: dict[str, Any] = {
+                "ref_audio": ref_uri,
+                "ref_text": row.ref_text,
+                "task_type": "Base",
+                "language": lang,
+                "max_new_tokens": output_len,
+            }
+
+            out.append(
+                SeedTTSSampleRequest(
+                    prompt=target,
+                    prompt_len=prompt_len,
+                    expected_output_len=output_len,
+                    multi_modal_data=None,
+                    request_id=f"{request_id_prefix}{i}",
+                    seed_tts_speech_extra=speech_extra,
+                    seed_tts_utterance_id=row.utterance_id,
+                    seed_tts_locale=self.locale,
+                    seed_tts_system_prompt=self._system_prompt,
+                    seed_tts_ref_wav_path=str(wav_path),
+                )
+            )
+
+        logger.info("Seed-TTS: built %d requests (asked %d)", len(out), num_requests)
+        self.maybe_oversample_requests(out, num_requests, request_id_prefix, no_oversample)
+        return out
+
+
+def load_seed_tts_dataset(
+    dataset_path: str,
+    random_seed: int = 0,
+    locale: str = "en",
+    inline_ref_audio: bool = True,
+    seed_tts_root: str | None = None,
+    system_prompt: str | None = None,
+    **kwargs: Any,
+) -> SeedTTSDataset:
+    return SeedTTSDataset(
+        dataset_path=dataset_path,
+        random_seed=random_seed,
+        locale=locale,
+        inline_ref_audio=inline_ref_audio,
+        seed_tts_root=seed_tts_root,
+        system_prompt=system_prompt,
+        **kwargs,
+    )

vllm_omni/benchmarks/data_modules/seed_tts_eval.py

@@ -0,0 +1,729 @@
+"""Seed-TTS WER aligned with Bytedance ``seed-tts-eval`` / ``run_wer.py``.
+
+Matches the published protocol (see Hugging Face dataset card and
+https://github.com/BytedanceSpeech/seed-tts-eval):
+
+- **EN**: ``openai/whisper-large-v3`` via ``transformers``, audio resampled to **16 kHz**
+  (same as ``run_wer.py``).
+- **ZH**: ``funasr`` **paraformer-zh**, hypothesis converted with **zhconv** to zh-cn.
+- **WER**: ``jiwer`` after punctuation stripping (``zhon.hanzi.punctuation`` + ``string.punctuation``,
+  preserving ``'``) and EN lowercasing / ZH per-character spacing. Supports jiwer 3.x
+  (``compute_measures``) and 4.x (``process_words``).
+
+- **SIM** (speaker similarity proxy): cosine similarity of L2-normalized mean-pooled **WavLM**
+  embeddings (reference prompt WAV vs. synthesized PCM), 16 kHz. Official ``cal_sim.sh`` uses
+  UniSpeech ``verification_pair_list_v2.py`` with a **fine-tuned** WavLM SV checkpoint — set
+  ``SEED_TTS_WAVLM_MODEL`` to another HF id if you need closer parity. Disable with
+  ``SEED_TTS_SIM_EVAL=0``. Optional: ``SEED_TTS_SIM_DEVICE`` (e.g. ``cpu``) to avoid GPU
+  issues when Whisper already uses CUDA; ``SEED_TTS_WAVLM_MIN_SAMPLES`` pads very short
+  waveforms so the WavLM CNN front-end does not fail.
+
+- **UTMOS** (predicted MOS from TorchScript): default ``balacoon/utmos`` → ``utmos.jit``
+  (Sarulab-style demo export). Uses ``torch`` + ``huggingface_hub`` only. Aggregate metrics
+  are over **all requests with captured PCM** (independent of ASR/WER). Non-finite scores are
+  dropped and counted as failures. Override repo/file via ``SEED_TTS_UTMOS_HF_REPO`` /
+  ``SEED_TTS_UTMOS_JIT_FILE``. **Device**: defaults to **CPU** when ``SEED_TTS_UTMOS_DEVICE``
+  is unset; set ``SEED_TTS_UTMOS_DEVICE=cuda:0`` (or ``cuda:1`` etc.) to run on GPU. The JIT
+  model is loaded directly onto the target device via ``map_location`` to avoid cross-device
+  issues (some PyTorch builds/Windows have problems moving TorchScript modules after load).
+  Forward uses **float32** waveform in ``[-1, 1]`` (same as the WER resampled array) so
+  tensor dtypes match JIT weights; using int16 triggers
+  ``RuntimeError: input type and weight type should be same`` on common exports. Disable
+  with ``SEED_TTS_UTMOS_EVAL=0``.
+
+Enable with ``SEED_TTS_WER_EVAL=1`` or ``--seed-tts-wer-eval``. Install optional deps::
+
+    pip install 'vllm-omni[seed-tts-eval]'
+
+Env: ``SEED_TTS_EVAL_DEVICE`` (e.g. ``cuda:0``, ``cpu``); ``SEED_TTS_HF_WHISPER_MODEL``
+defaults to ``openai/whisper-large-v3`` (override for debugging only).
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import math
+import os
+import statistics
+import string
+import tempfile
+import threading
+import wave
+from typing import Any
+
+import numpy as np
+from vllm.benchmarks.datasets import SampleRequest
+
+from vllm_omni.benchmarks.data_modules.seed_tts_dataset import SeedTTSSampleRequest
+
+logger = logging.getLogger(__name__)
+
+# Mirrors seed-tts-eval/run_wer.py
+OFFICIAL_WHISPER_HF_ID = "openai/whisper-large-v3"
+PARAFORMER_MODEL_ID = "paraformer-zh"
+
+_lock = threading.Lock()
+_device: str | None = None
+_en_processor = None
+_en_model = None
+_zh_paraformer = None
+_wavlm_model = None
+_wavlm_processor = None
+_wavlm_device: str | None = None
+_utmos_jit_model = None
+_utmos_jit_device: str | None = None
+_utmos_jit_load_failed = False
+_utmos_forward_warned = False
+
+
+def pcm_s16le_mono_to_wav_bytes(pcm: bytes, *, sample_rate: int = 24000) -> bytes:
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(pcm)
+    return buf.getvalue()
+
+
+def _get_eval_device() -> str:
+    explicit = os.environ.get("SEED_TTS_EVAL_DEVICE", "").strip()
+    if explicit:
+        return explicit
+    try:
+        import torch
+
+        return "cuda:0" if torch.cuda.is_available() else "cpu"
+    except ImportError:
+        return "cpu"
+
+
+def _punctuation_all() -> str:
+    from zhon.hanzi import punctuation
+
+    return punctuation + string.punctuation
+
+
+def _jiwer_wer(reference: str, hypothesis: str) -> float:
+    """Word-level WER; strings are normalized like ``run_wer.process_one``.
+
+    jiwer 4.x removed ``compute_measures`` (``ImportError``); fall back to ``process_words``.
+    """
+    try:
+        from jiwer import compute_measures
+
+        return float(compute_measures(reference, hypothesis)["wer"])
+    except ImportError:
+        import jiwer
+
+        out = jiwer.process_words(reference, hypothesis)
+        return float(out.wer)
+
+
+def process_one_official(hypo: str, truth: str, lang: str) -> tuple[float, str, str]:
+    """Same normalization + ``jiwer`` call as ``run_wer.process_one`` (hypo=ASR, truth=reference)."""
+    raw_truth = truth
+    raw_hypo = hypo
+    truth_n = truth
+    hypo_n = hypo
+    for x in _punctuation_all():
+        if x == "'":
+            continue
+        truth_n = truth_n.replace(x, "")
+        hypo_n = hypo_n.replace(x, "")
+    truth_n = truth_n.replace(" ", " ")
+    hypo_n = hypo_n.replace(" ", " ")
+    if lang == "zh":
+        truth_n = " ".join([x for x in truth_n])
+        hypo_n = " ".join([x for x in hypo_n])
+    elif lang == "en":
+        truth_n = truth_n.lower()
+        hypo_n = hypo_n.lower()
+    else:
+        raise ValueError(f"unsupported lang {lang!r}")
+    wer = _jiwer_wer(truth_n, hypo_n)
+    return wer, raw_truth, raw_hypo
+
+
+def _pcm_s16le_to_f32_16k(pcm: bytes, pcm_sample_rate: int = 24000) -> np.ndarray:
+    import scipy.signal
+
+    if not pcm:
+        return np.zeros(0, dtype=np.float32)
+    raw = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0
+    target_len = int(len(raw) * 16000 / pcm_sample_rate)
+    if target_len <= 0:
+        return np.zeros(0, dtype=np.float32)
+    return scipy.signal.resample(raw, target_len).astype(np.float32)
+
+
+def _eval_submetric_enabled(env_name: str, *, default: bool = True) -> bool:
+    raw = os.environ.get(env_name, "").strip().lower()
+    if raw in ("0", "false", "no", "off"):
+        return False
+    if raw in ("1", "true", "yes", "on"):
+        return True
+    return default
+
+
+def _audio_path_to_f32_16k(path: str) -> np.ndarray:
+    import scipy.signal
+    import soundfile as sf
+
+    data, sr = sf.read(path, dtype="float32", always_2d=True)
+    mono = np.mean(data, axis=1).astype(np.float32)
+    if int(sr) == 16000:
+        return mono
+    target_len = max(1, int(len(mono) * 16000 / int(sr)))
+    return scipy.signal.resample(mono, target_len).astype(np.float32)
+
+
+def _ensure_wavlm_sim() -> None:
+    global _wavlm_model, _wavlm_processor, _wavlm_device
+    with _lock:
+        if _wavlm_model is not None:
+            return
+        from transformers import AutoFeatureExtractor, AutoModel
+
+        mid = os.environ.get("SEED_TTS_WAVLM_MODEL", "microsoft/wavlm-base-plus").strip() or "microsoft/wavlm-base-plus"
+        _wavlm_device = os.environ.get("SEED_TTS_SIM_DEVICE", "").strip() or _get_eval_device()
+        logger.warning(
+            "Loading WavLM %r on %s for Seed-TTS SIM (embedding cosine; not identical to "
+            "seed-tts-eval UniSpeech SV checkpoint).",
+            mid,
+            _wavlm_device,
+        )
+        _wavlm_processor = AutoFeatureExtractor.from_pretrained(mid)
+        _wavlm_model = AutoModel.from_pretrained(mid).to(_wavlm_device)
+        _wavlm_model.eval()
+
+
+def _wavlm_prepare_waveform(wav: np.ndarray) -> np.ndarray:
+    """Trim, pad to a minimum length WavLM/Wav2Vec2 CNN stack accepts, float32 mono."""
+    max_sec = float(os.environ.get("SEED_TTS_WAVLM_MAX_SECONDS", "30"))
+    cap = int(max_sec * 16000)
+    w = np.asarray(wav, dtype=np.float32).reshape(-1)
+    if len(w) == 0:
+        return w
+    if len(w) > cap:
+        w = w[:cap].copy()
+    # Very short clips make the strided conv front-end fail (shape / empty time dim).
+    min_samples = int(os.environ.get("SEED_TTS_WAVLM_MIN_SAMPLES", "4000"))
+    if len(w) < min_samples:
+        w = np.pad(w, (0, min_samples - len(w)), mode="constant")
+    return w
+
+
+def _wavlm_mean_embedding_f32_16k(wav: np.ndarray) -> np.ndarray | None:
+    import torch
+
+    _ensure_wavlm_sim()
+    w = _wavlm_prepare_waveform(wav)
+    if len(w) == 0:
+        return None
+    assert _wavlm_processor is not None and _wavlm_model is not None and _wavlm_device is not None
+    # Single utterance: avoid padding=True (adds zeros that distort mean pooling). Still pass
+    # attention_mask when the extractor provides it (sample-level; do not mix with hidden length).
+    try:
+        inputs = _wavlm_processor(
+            w,
+            sampling_rate=16000,
+            return_tensors="pt",
+            padding=False,
+            return_attention_mask=True,
+        )
+    except TypeError:
+        inputs = _wavlm_processor(
+            w,
+            sampling_rate=16000,
+            return_tensors="pt",
+            padding=False,
+        )
+    iv = inputs["input_values"].to(_wavlm_device)
+    am = inputs.get("attention_mask")
+    if am is not None:
+        am = am.to(_wavlm_device)
+    with torch.inference_mode():
+        out = _wavlm_model(iv, attention_mask=am)
+        h = out.last_hidden_state
+        v = h.mean(dim=1).squeeze(0).float().cpu().numpy()
+    n = float(np.linalg.norm(v))
+    if not np.isfinite(n) or n < 1e-8:
+        return None
+    return (v / n).astype(np.float32)
+
+
+def _cosine_similarity_unit_vectors(a: np.ndarray, b: np.ndarray) -> float:
+    return float(np.dot(a, b))
+
+
+def _ensure_utmos_jit_model() -> Any | None:
+    """Load UTMOS as TorchScript (``balacoon/utmos`` style): no ``import utmos`` / fairseq."""
+    global _utmos_jit_model, _utmos_jit_device, _utmos_jit_load_failed
+    with _lock:
+        if _utmos_jit_load_failed:
+            return None
+        if _utmos_jit_model is not None:
+            return _utmos_jit_model
+        try:
+            import torch
+            from huggingface_hub import hf_hub_download
+
+            repo = os.environ.get("SEED_TTS_UTMOS_HF_REPO", "balacoon/utmos").strip() or "balacoon/utmos"
+            fname = os.environ.get("SEED_TTS_UTMOS_JIT_FILE", "utmos.jit").strip() or "utmos.jit"
+            logger.warning(
+                "Loading UTMOS TorchScript from Hugging Face %r file %r (one-time download/cache)...",
+                repo,
+                fname,
+            )
+            path = hf_hub_download(repo_id=repo, filename=fname, repo_type="model")
+
+            # TODO The model weights in UTMOS must be loaded in cuda:0; otherwise, the model execution will fail.
+            want = "cuda:0"
+            if want.startswith("cuda") and torch.cuda.is_available():
+                idx = want.split(":")[-1] if ":" in want else "0"
+                target_dev = f"cuda:{idx}"
+            else:
+                target_dev = "cpu"
+
+            try:
+                m = torch.jit.load(path, map_location=target_dev)
+                m.eval()
+                _utmos_jit_device = target_dev
+            except Exception as load_e:
+                if target_dev.startswith("cuda"):
+                    logger.warning(
+                        "UTMOS JIT load on %s failed (%s), retrying on CPU...",
+                        target_dev,
+                        load_e,
+                    )
+                    m = torch.jit.load(path, map_location="cpu")
+                    m.eval()
+                    _utmos_jit_device = "cpu"
+                else:
+                    raise
+            _utmos_jit_model = m
+        except Exception as e:
+            logger.warning(
+                "UTMOS JIT unavailable (install torch + huggingface_hub; check HF access): %s",
+                e,
+            )
+            _utmos_jit_load_failed = True
+            return None
+    return _utmos_jit_model
+
+
+def _utmos_predict_f32_16k(wav_f32: np.ndarray) -> float | None:
+    """MOS from JIT model; input is float32 mono @ 16 kHz in ``[-1, 1]`` (WER pipeline).
+
+    ``balacoon/utmos`` demos sometimes use int16 numpy, but the exported ``.jit`` weights are
+    float32; passing int16 tensors causes: "RuntimeError: ... input type and weight type
+    should be same".
+    """
+    import torch
+
+    if len(wav_f32) == 0:
+        return None
+    model = _ensure_utmos_jit_model()
+    if model is None:
+        return None
+    # Infer model's device from its first parameter/buffer to guarantee input sits with weights.
+    try:
+        model_dev = next(model.parameters()).device
+    except StopIteration:
+        try:
+            model_dev = next(model.buffers()).device
+        except StopIteration:
+            model_dev = torch.device("cpu")
+    w = np.ascontiguousarray(wav_f32, dtype=np.float32)
+    x = torch.from_numpy(w).unsqueeze(0).to(device=model_dev, dtype=torch.float32)
+    with torch.no_grad():
+        out = model(x)
+    val = float(out.reshape(-1)[0].item())
+    if not math.isfinite(val):
+        return None
+    return val
+
+
+def _ensure_en_asr() -> None:
+    global _en_processor, _en_model, _device
+    with _lock:
+        if _en_processor is not None:
+            return
+        from transformers import WhisperForConditionalGeneration, WhisperProcessor
+
+        _device = _get_eval_device()
+        mid = os.environ.get("SEED_TTS_HF_WHISPER_MODEL", OFFICIAL_WHISPER_HF_ID).strip() or OFFICIAL_WHISPER_HF_ID
+        logger.warning(
+            "Loading Seed-TTS eval Whisper HF model %r on %s (one-time, seed-tts-eval protocol)...",
+            mid,
+            _device,
+        )
+        _en_processor = WhisperProcessor.from_pretrained(mid)
+        _en_model = WhisperForConditionalGeneration.from_pretrained(mid).to(_device)
+        _en_model.eval()
+
+
+def _ensure_zh_asr() -> None:
+    global _zh_paraformer, _device
+    with _lock:
+        if _zh_paraformer is not None:
+            return
+        from funasr import AutoModel
+
+        _device = _get_eval_device()
+        logger.warning(
+            "Loading Seed-TTS eval Paraformer %r on %s (one-time, seed-tts-eval protocol)...",
+            PARAFORMER_MODEL_ID,
+            _device,
+        )
+        try:
+            _zh_paraformer = AutoModel(model=PARAFORMER_MODEL_ID, device=_device)
+        except TypeError:
+            _zh_paraformer = AutoModel(model=PARAFORMER_MODEL_ID)
+
+
+def _transcribe_en_f32_16k(wav_f32: np.ndarray) -> str:
+    import torch
+
+    _ensure_en_asr()
+    if len(wav_f32) == 0:
+        return ""
+    with _lock:
+        assert _en_processor is not None and _en_model is not None and _device is not None
+        inputs = _en_processor(wav_f32, sampling_rate=16000, return_tensors="pt")
+        input_features = inputs.input_features.to(_device)
+        with torch.no_grad():
+            try:
+                forced = _en_processor.get_decoder_prompt_ids(language="english", task="transcribe")
+                predicted_ids = _en_model.generate(input_features, forced_decoder_ids=forced)
+            except Exception:
+                predicted_ids = _en_model.generate(
+                    input_features,
+                    language="english",
+                    task="transcribe",
+                )
+        text = _en_processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]
+    return (text or "").strip()
+
+
+def _transcribe_zh_wav_path(wav_path: str) -> str:
+    import zhconv
+
+    _ensure_zh_asr()
+    with _lock:
+        assert _zh_paraformer is not None
+        res = _zh_paraformer.generate(input=wav_path, batch_size_s=300)
+    transcription = res[0]["text"] if res else ""
+    return zhconv.convert(transcription, "zh-cn").strip()
+
+
+def _missing_deps_message(lang: str) -> str | None:
+    try:
+        import jiwer  # noqa: F401
+        from zhon.hanzi import punctuation  # noqa: F401
+    except ImportError as e:
+        return f"Seed-TTS WER eval needs jiwer and zhon ({e!s}). Install: pip install 'vllm-omni[seed-tts-eval]'"
+    try:
+        import scipy.signal  # noqa: F401
+        import soundfile  # noqa: F401
+    except ImportError as e:
+        return f"Seed-TTS WER eval needs scipy and soundfile ({e!s})."
+    if lang == "en":
+        try:
+            import torch  # noqa: F401
+            from transformers import WhisperForConditionalGeneration  # noqa: F401
+        except ImportError as e:
+            return f"English WER needs torch and transformers ({e!s}). Install: pip install 'vllm-omni[seed-tts-eval]'"
+    else:
+        try:
+            import zhconv  # noqa: F401
+            from funasr import AutoModel  # noqa: F401
+        except ImportError as e:
+            return f"Chinese WER needs funasr and zhconv ({e!s}). Install: pip install 'vllm-omni[seed-tts-eval]'"
+    return None
+
+
+def compute_seed_tts_wer_metrics(
+    input_requests: list[SampleRequest],
+    outputs: list[Any],
+    *,
+    include_per_item: bool = False,
+) -> dict[str, Any] | None:
+    """If all requests are :class:`SeedTTSSampleRequest`, run seed-tts-eval-style WER."""
+    global _utmos_forward_warned
+    if not input_requests or len(input_requests) != len(outputs):
+        return None
+    if not all(isinstance(r, SeedTTSSampleRequest) for r in input_requests):
+        return None
+
+    first = input_requests[0]
+    assert isinstance(first, SeedTTSSampleRequest)
+    lang = "zh" if (first.seed_tts_locale or "en").lower().startswith("zh") else "en"
+
+    setup_err = _missing_deps_message(lang)
+    if setup_err:
+        logger.error("%s", setup_err)
+        return {
+            "seed_tts_eval_setup_error": setup_err,
+            "seed_tts_eval_protocol": "seed-tts-eval",
+            "seed_tts_content_evaluated": 0,
+            "seed_tts_content_error_mean": None,
+            "seed_tts_content_error_median": None,
+            "seed_tts_request_failed": 0,
+            "seed_tts_no_pcm": 0,
+            "seed_tts_asr_failed": 0,
+            "seed_tts_content_metric": "wer",
+        }
+
+    import soundfile as sf
+
+    errs: list[float] = []
+    items: list[dict[str, Any]] = []
+    asr_failed = 0
+    no_pcm = 0
+    request_failed = 0
+    sim_values: list[float] = []
+    utmos_values: list[float] = []
+    sim_failed = 0
+    sim_skipped_no_ref = 0
+    utmos_failed = 0
+    utmos_on = _eval_submetric_enabled("SEED_TTS_UTMOS_EVAL", default=True)
+
+    for req, out in zip(input_requests, outputs, strict=True):
+        assert isinstance(req, SeedTTSSampleRequest)
+        ref = req.prompt
+        locale = req.seed_tts_locale or "en"
+        row_lang = "zh" if locale.lower().startswith("zh") else "en"
+        utmos_v: float | None = None
+
+        if not out.success:
+            request_failed += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "utterance_id": req.seed_tts_utterance_id,
+                        "locale": locale,
+                        "error": "request_failed",
+                        "detail": (out.error or "")[:500],
+                    }
+                )
+            continue
+
+        pcm = getattr(out, "tts_output_pcm_bytes", None)
+        if not pcm:
+            no_pcm += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "utterance_id": req.seed_tts_utterance_id,
+                        "locale": locale,
+                        "error": "no_pcm",
+                    }
+                )
+            continue
+
+        wav_16k = _pcm_s16le_to_f32_16k(pcm)
+        if len(wav_16k) == 0:
+            asr_failed += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "utterance_id": req.seed_tts_utterance_id,
+                        "locale": locale,
+                        "error": "empty_audio",
+                    }
+                )
+            continue
+
+        # UTMOS scores synthesized audio only; do not gate on ASR/WER (those can fail independently).
+        if utmos_on:
+            try:
+                utmos_v = _utmos_predict_f32_16k(wav_16k)
+                if utmos_v is not None:
+                    utmos_values.append(utmos_v)
+                elif not _utmos_jit_load_failed:
+                    utmos_failed += 1
+            except Exception:
+                if not _utmos_forward_warned:
+                    _utmos_forward_warned = True
+                    logger.warning(
+                        "UTMOS JIT forward failed (first utterance=%s; set logging DEBUG for "
+                        "full trace). Check sample rate (16 kHz), input shape, or "
+                        "SEED_TTS_UTMOS_DEVICE.",
+                        req.seed_tts_utterance_id,
+                        exc_info=True,
+                    )
+                else:
+                    logger.debug(
+                        "UTMOS forward failed for %s",
+                        req.seed_tts_utterance_id,
+                        exc_info=True,
+                    )
+                utmos_failed += 1
+
+        try:
+            if row_lang == "en":
+                hyp = _transcribe_en_f32_16k(wav_16k)
+            else:
+                fd, tmp_wav = tempfile.mkstemp(suffix=".wav")
+                os.close(fd)
+                try:
+                    sf.write(tmp_wav, wav_16k, 16000, subtype="PCM_16")
+                    hyp = _transcribe_zh_wav_path(tmp_wav)
+                finally:
+                    try:
+                        os.unlink(tmp_wav)
+                    except OSError:
+                        pass
+        except Exception as e:
+            logger.exception("Seed-TTS ASR failed for %s", req.seed_tts_utterance_id)
+            asr_failed += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "utterance_id": req.seed_tts_utterance_id,
+                        "locale": locale,
+                        "error": "asr_exception",
+                        "detail": str(e)[:500],
+                    }
+                )
+            continue
+
+        if not hyp:
+            asr_failed += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "utterance_id": req.seed_tts_utterance_id,
+                        "locale": locale,
+                        "error": "empty_asr",
+                    }
+                )
+            continue
+
+        try:
+            wer, raw_truth, raw_hypo = process_one_official(hyp, ref, row_lang)
+        except Exception as e:
+            logger.warning("jiwer/normalize failed for %s: %s", req.seed_tts_utterance_id, e)
+            asr_failed += 1
+            if include_per_item:
+                items.append(
+                    {
+                        "utterance_id": req.seed_tts_utterance_id,
+                        "locale": locale,
+                        "error": "wer_compute_failed",
+                        "detail": str(e)[:500],
+                    }
+                )
+            continue
+
+        errs.append(wer)
+        sim_v: float | None = None
+
+        if _eval_submetric_enabled("SEED_TTS_SIM_EVAL", default=True):
+            ref_path = getattr(req, "seed_tts_ref_wav_path", "") or ""
+            if ref_path and os.path.isfile(ref_path):
+                try:
+                    ref_wav = _audio_path_to_f32_16k(ref_path)
+                    e_ref = _wavlm_mean_embedding_f32_16k(ref_wav)
+                    e_hyp = _wavlm_mean_embedding_f32_16k(wav_16k)
+                    if e_ref is not None and e_hyp is not None:
+                        sim_v = _cosine_similarity_unit_vectors(e_ref, e_hyp)
+                        sim_values.append(sim_v)
+                except Exception as e:
+                    logger.warning(
+                        "SIM embedding failed for utterance=%s: %s: %s",
+                        req.seed_tts_utterance_id,
+                        type(e).__name__,
+                        e,
+                    )
+                    sim_failed += 1
+            else:
+                sim_skipped_no_ref += 1
+
+        if include_per_item:
+            row: dict[str, Any] = {
+                "utterance_id": req.seed_tts_utterance_id,
+                "locale": locale,
+                "wer": wer,
+                "reference_raw": raw_truth,
+                "asr_raw": raw_hypo,
+            }
+            if sim_v is not None:
+                row["sim"] = sim_v
+            if utmos_v is not None:
+                row["utmos"] = utmos_v
+            items.append(row)
+
+    result: dict[str, Any] = {
+        "seed_tts_eval_protocol": "seed-tts-eval",
+        "seed_tts_content_evaluated": len(errs),
+        "seed_tts_content_error_mean": statistics.fmean(errs) if errs else None,
+        "seed_tts_content_error_median": statistics.median(errs) if errs else None,
+        "seed_tts_request_failed": request_failed,
+        "seed_tts_no_pcm": no_pcm,
+        "seed_tts_asr_failed": asr_failed,
+        "seed_tts_content_metric": "wer",
+        "seed_tts_sim_evaluated": len(sim_values),
+        "seed_tts_sim_mean": statistics.fmean(sim_values) if sim_values else None,
+        "seed_tts_sim_median": statistics.median(sim_values) if sim_values else None,
+        "seed_tts_sim_failed": sim_failed,
+        "seed_tts_sim_skipped_no_ref": sim_skipped_no_ref,
+        "seed_tts_utmos_evaluated": len(utmos_values),
+        "seed_tts_utmos_mean": statistics.fmean(utmos_values) if utmos_values else None,
+        "seed_tts_utmos_median": statistics.median(utmos_values) if utmos_values else None,
+        "seed_tts_utmos_failed": utmos_failed,
+    }
+    if include_per_item:
+        result["seed_tts_wer_eval_items"] = items
+    return result
+
+
+def print_seed_tts_wer_summary(metrics: dict[str, Any]) -> None:
+    setup = metrics.get("seed_tts_eval_setup_error")
+    if setup:
+        print("{s:{c}^{n}}".format(s=" Seed-TTS eval (seed-tts-eval protocol) ", n=50, c="="))
+        print(setup)
+        return
+
+    ev = int(metrics.get("seed_tts_content_evaluated", 0) or 0)
+    rf = int(metrics.get("seed_tts_request_failed", 0) or 0)
+    npc = int(metrics.get("seed_tts_no_pcm", 0) or 0)
+    af = int(metrics.get("seed_tts_asr_failed", 0) or 0)
+    sim_ev = int(metrics.get("seed_tts_sim_evaluated", 0) or 0)
+    ut_ev = int(metrics.get("seed_tts_utmos_evaluated", 0) or 0)
+    if ev == 0 and rf == 0 and npc == 0 and af == 0 and sim_ev == 0 and ut_ev == 0:
+        return
+    print("{s:{c}^{n}}".format(s=" Seed-TTS eval (seed-tts-eval protocol) ", n=50, c="="))
+    print("{:<40} {:<10}".format("Evaluated (WER, lower is better):", ev))
+    mean = metrics.get("seed_tts_content_error_mean")
+    if mean is not None:
+        print("{:<40} {:<10.4f}".format("Mean WER:", float(mean)))
+    med = metrics.get("seed_tts_content_error_median")
+    if med is not None:
+        print("{:<40} {:<10.4f}".format("Median WER:", float(med)))
+    print("{:<40} {:<10}".format("Request failed:", metrics.get("seed_tts_request_failed", 0)))
+    print("{:<40} {:<10}".format("No PCM captured:", metrics.get("seed_tts_no_pcm", 0)))
+    print("{:<40} {:<10}".format("ASR / WER failed:", metrics.get("seed_tts_asr_failed", 0)))
+    if sim_ev or metrics.get("seed_tts_sim_skipped_no_ref") or metrics.get("seed_tts_sim_failed"):
+        print("{:<40} {:<10}".format("SIM evaluated (higher ~ closer):", sim_ev))
+        sm = metrics.get("seed_tts_sim_mean")
+        if sm is not None:
+            print("{:<40} {:<10.4f}".format("Mean SIM:", float(sm)))
+        s_med = metrics.get("seed_tts_sim_median")
+        if s_med is not None:
+            print("{:<40} {:<10.4f}".format("Median SIM:", float(s_med)))
+        print("{:<40} {:<10}".format("SIM skipped (no ref path):", metrics.get("seed_tts_sim_skipped_no_ref", 0)))
+        print("{:<40} {:<10}".format("SIM embedding errors:", metrics.get("seed_tts_sim_failed", 0)))
+    if ut_ev or metrics.get("seed_tts_utmos_failed"):
+        print("{:<40} {:<10}".format("UTMOS evaluated (JIT MOS, higher better):", ut_ev))
+        um = metrics.get("seed_tts_utmos_mean")
+        if um is not None:
+            print("{:<40} {:<10.4f}".format("Mean UTMOS:", float(um)))
+        u_med = metrics.get("seed_tts_utmos_median")
+        if u_med is not None:
+            print("{:<40} {:<10.4f}".format("Median UTMOS:", float(u_med)))
+        print("{:<40} {:<10}".format("UTMOS errors:", metrics.get("seed_tts_utmos_failed", 0)))
+    print("=" * 50)

vllm_omni/benchmarks/patch/__init__.py

@@ -0,0 +1,3 @@
+"""Omni benchmark monkey-patches (side effects in ``patch.patch``)."""
+
+from . import patch as _patch_module  # noqa: F401

vllm_omni/benchmarks/patch/patch.py

@@ -6,6 +6,7 @@ import json
 import os
 import random
 import ssl
+import sys
 import time
 import traceback
 from collections.abc import Iterable
@@ -33,15 +34,245 @@ from vllm.logger import init_logger
 from vllm.tokenizers import TokenizerLike
 
 logger = init_logger(__name__)
+
+from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest
 from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
+from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
+    SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
+    SeedTTSDataset,
+    SeedTTSSampleRequest,
+)
 
 get_samples_old = datasets.get_samples
 
+_DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni"
+
+
+def _seed_tts_capture_pcm_for_wer() -> bool:
+    return os.environ.get("SEED_TTS_WER_EVAL", "").lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+
+
+def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict | None:
+    """Shallow-merge ``extra_body`` dicts; deep-merge ``mm_processor_kwargs`` if both set."""
+    if not base and not overlay:
+        return None
+    out = dict(base or {})
+    if not overlay:
+        return out
+    for k, v in overlay.items():
+        if k == "mm_processor_kwargs" and isinstance(v, dict):
+            prev = out.get("mm_processor_kwargs")
+            merged_kw = {**(prev if isinstance(prev, dict) else {}), **v}
+            out["mm_processor_kwargs"] = merged_kw
+        else:
+            out[k] = v
+    return out
+
+
+def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
+    """Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
+    if not isinstance(sample, DailyOmniSampleRequest):
+        return
+    rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
+    if sample.omni_chat_messages is not None:
+        setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
+    else:
+        setattr(rfi, "mm_position", sample.omni_chat_mm_position)
+
+
+def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
+    """Merge Seed-TTS per-row TTS fields (ref_audio, ref_text, task_type, …) into ``extra_body``.
+
+    Used by both ``/v1/audio/speech`` and ``/v1/chat/completions`` (flattened into JSON body).
+    For ``openai-chat-omni``, also sets ``omni_chat_messages`` (system + user) so Qwen3-Omni
+    follows the same role layout as official TTS / multimodal demos. ``/v1/audio/speech`` ignores
+    ``messages`` and only uses ``input`` + body fields.
+    Flags ``openai-chat-omni`` to request audio output and optionally export PCM for WER.
+    """
+    if not isinstance(sample, SeedTTSSampleRequest):
+        return
+    ex = sample.seed_tts_speech_extra
+    if not ex:
+        return
+    base = dict(rfi.extra_body) if rfi.extra_body else {}
+    base.update(ex)
+    rfi.extra_body = base
+    # Used by request funcs to force streaming TTS behavior and to export PCM when WER is on.
+    setattr(rfi, "seed_tts_row", True)
+    sys_prompt = (sample.seed_tts_system_prompt or "").strip() or SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
+    setattr(
+        rfi,
+        "omni_chat_messages",
+        [
+            {"role": "system", "content": [{"type": "text", "text": sys_prompt}]},
+            {"role": "user", "content": [{"type": "text", "text": sample.prompt}]},
+        ],
+    )
+
+
+def _daily_omni_repo_from_args(args) -> str | None:
+    """Resolve HuggingFace repo id for Daily-Omni from CLI args.
+
+    vLLM allows ``--dataset-path`` to be a local path while the real HF id is
+    passed via ``--hf-name``. Upstream ``get_samples`` for ``hf`` only matches
+    a fixed elif-chain and never discovers Omni's loader, so we must detect
+    Daily-Omni here using either field.
+    """
+    dp = getattr(args, "dataset_path", None)
+    hn = getattr(args, "hf_name", None)
+    if dp in DailyOmniDataset.SUPPORTED_DATASET_PATHS:
+        return dp
+    if hn in DailyOmniDataset.SUPPORTED_DATASET_PATHS:
+        return hn
+    return None
+
 
 def get_samples(args, tokenizer):
-    if args.backend not in ["openai-chat-omni", "openai-audio-speech"]:
+    # Daily-Omni: explicit dataset name, or hf + matching path/hf-name
+    is_daily_omni = args.dataset_name == "daily-omni" or (
+        args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
+    )
+    is_seed_tts = args.dataset_name == "seed-tts"
+
+    # Check if we need to handle omni-related backends/datasets
+    is_omni_backend = args.backend in ["openai-chat-omni", "openai-audio-speech", "daily-omni"]
+    is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
+
+    if not is_omni_backend and not is_omni_dataset:
+        # Not an omni-related request, delegate to original implementation
         return get_samples_old(args, tokenizer)
-    elif args.dataset_name == "random-mm":
+
+    # Handle Daily-Omni dataset
+    if is_daily_omni:
+        # Support:
+        #   --dataset-name daily-omni [--dataset-path liarliar/Daily-Omni]
+        #   --dataset-name daily-omni --daily-omni-qa-json /path/to/qa.json  (offline QA)
+        #   --dataset-name hf --dataset-path liarliar/Daily-Omni
+        #   --dataset-name hf --hf-name liarliar/Daily-Omni  (dataset-path may be local)
+
+        # Validate backend supports multimodal (video)
+        if args.backend not in ["openai-chat-omni", "daily-omni"]:
+            raise ValueError(
+                f"Daily-Omni dataset requires a multimodal backend that supports video. "
+                f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
+            )
+
+        # Determine video directory if specified (for local video files)
+        video_dir = getattr(args, "daily_omni_video_dir", None)
+
+        # Get HF split (default to "train"; unused when loading from local qa.json)
+        dataset_split = getattr(args, "hf_split", None) or "train"
+
+        qa_json = getattr(args, "daily_omni_qa_json", None)
+        if isinstance(qa_json, str):
+            qa_json = qa_json.strip() or None
+
+        if qa_json is not None:
+            logger.info(
+                "Loading Daily-Omni dataset: qa_json=%s, video_dir=%s (Hub not used for QA)",
+                qa_json,
+                video_dir,
+            )
+            dataset = DailyOmniDataset(
+                qa_json_path=qa_json,
+                dataset_path=None,
+                dataset_split=dataset_split,
+                random_seed=args.seed,
+                video_dir=video_dir,
+                input_mode=getattr(args, "daily_omni_input_mode", "all"),
+                inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
+                trust_remote_code=getattr(args, "trust_remote_code", False),
+                disable_shuffle=getattr(args, "disable_shuffle", False),
+            )
+        else:
+            repo_id = _daily_omni_repo_from_args(args)
+            if args.dataset_name == "daily-omni":
+                if repo_id is None:
+                    repo_id = _DEFAULT_DAILY_OMNI_REPO
+            elif repo_id is None:
+                raise ValueError(
+                    "Daily-Omni with --dataset-name hf requires "
+                    f"--dataset-path {_DEFAULT_DAILY_OMNI_REPO} or "
+                    f"--hf-name {_DEFAULT_DAILY_OMNI_REPO}."
+                )
+
+            logger.info(
+                "Loading Daily-Omni dataset: hf_repo=%s, split=%s, video_dir=%s",
+                repo_id,
+                dataset_split,
+                video_dir,
+            )
+
+            dataset = DailyOmniDataset(
+                dataset_path=repo_id,
+                dataset_split=dataset_split,
+                dataset_subset=getattr(args, "hf_subset", None),
+                random_seed=args.seed,
+                video_dir=video_dir,
+                input_mode=getattr(args, "daily_omni_input_mode", "all"),
+                inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
+                trust_remote_code=getattr(args, "trust_remote_code", False),
+                no_stream=getattr(args, "no_stream", False),
+                disable_shuffle=getattr(args, "disable_shuffle", False),
+            )
+
+        out_len = getattr(args, "output_len", None)
+        if out_len is None:
+            out_len = getattr(args, "hf_output_len", None)
+        if out_len is None:
+            out_len = DailyOmniDataset.DEFAULT_OUTPUT_LEN
+
+        input_requests = dataset.sample(
+            tokenizer=tokenizer,
+            num_requests=args.num_prompts,
+            output_len=out_len,
+            request_id_prefix=args.request_id_prefix,
+            no_oversample=args.no_oversample,
+        )
+        return input_requests
+
+    if is_seed_tts:
+        if args.backend not in ("openai-audio-speech", "openai-chat-omni"):
+            raise ValueError(
+                "Seed-TTS requires --backend openai-audio-speech (POST /v1/audio/speech) or "
+                "--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text). "
+                f"Got backend={args.backend!r}."
+            )
+        repo_id = getattr(args, "dataset_path", None) or getattr(args, "hf_name", None)
+        if not repo_id:
+            raise ValueError(
+                "Seed-TTS requires --dataset-path (HF dataset repo id or local directory) or "
+                "--hf-name for the Hub dataset id."
+            )
+
+        dataset = SeedTTSDataset(
+            dataset_path=repo_id,
+            random_seed=args.seed,
+            locale=getattr(args, "seed_tts_locale", "en"),
+            inline_ref_audio=not getattr(args, "seed_tts_file_ref_audio", False),
+            seed_tts_root=getattr(args, "seed_tts_root", None),
+            system_prompt=getattr(args, "seed_tts_system_prompt", None),
+            disable_shuffle=getattr(args, "disable_shuffle", False),
+        )
+        out_len = getattr(args, "output_len", None)
+        if out_len is None:
+            out_len = getattr(args, "hf_output_len", None)
+        if out_len is None:
+            out_len = SeedTTSDataset.DEFAULT_OUTPUT_LEN
+        return dataset.sample(
+            tokenizer=tokenizer,
+            num_requests=args.num_prompts,
+            output_len=out_len,
+            request_id_prefix=args.request_id_prefix,
+            no_oversample=args.no_oversample,
+        )
+
+    # Handle random-mm dataset (Omni's synthetic multimodal dataset)
+    if args.dataset_name == "random-mm":
         dataset = OmniRandomMultiModalDataset(random_seed=args.seed, dataset_path=args.dataset_path)
         input_requests = dataset.sample(
             tokenizer=tokenizer,
@@ -64,6 +295,10 @@ def get_samples(args, tokenizer):
 
 datasets.get_samples = get_samples
 
+_serve_mod = sys.modules.get("vllm.benchmarks.serve")
+if _serve_mod is not None:
+    _serve_mod.get_samples = get_samples
+
 
 @dataclass
 class MixRequestFuncOutput(RequestFuncOutput):
@@ -72,6 +307,9 @@ class MixRequestFuncOutput(RequestFuncOutput):
     audio_frames: int = 0
     audio_rtf: float = 0.0
     text_latency: float = 0.0
+    #: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
+    #: resampled export after ``openai-chat-omni`` audio deltas.
+    tts_output_pcm_bytes: bytes | None = None
 
 
 async def async_request_openai_chat_omni_completions(
@@ -83,13 +321,17 @@ async def async_request_openai_chat_omni_completions(
     api_url = request_func_input.api_url
     _validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
 
-    content = _get_chat_content(request_func_input, mm_position=mm_position)
+    omni_messages = getattr(request_func_input, "omni_chat_messages", None)
+    if omni_messages is not None:
+        messages_payload = omni_messages
+    else:
+        effective_mm_position = getattr(request_func_input, "mm_position", mm_position)
+        content = _get_chat_content(request_func_input, mm_position=effective_mm_position)
+        messages_payload = [{"role": "user", "content": content}]
 
     payload = {
         "model": request_func_input.model_name if request_func_input.model_name else request_func_input.model,
-        "messages": [
-            {"role": "user", "content": content},
-        ],
+        "messages": messages_payload,
         "temperature": 0.0,
         "max_tokens": request_func_input.output_len,
         "stream": True,
@@ -98,6 +340,10 @@ async def async_request_openai_chat_omni_completions(
         },
     }
     _update_payload_common(payload, request_func_input)
+    # Seed-TTS via chat: voice-clone fields live on the body; ensure audio is streamed.
+    if getattr(request_func_input, "seed_tts_row", False):
+        if payload.get("modalities") is None:
+            payload["modalities"] = ["text", "audio"]
 
     response_format = payload.get("response_format", "wav")
     if response_format == "pcm":
@@ -167,7 +413,10 @@ async def async_request_openai_chat_omni_completions(
                                 data = json.loads(chunk)
                                 if choices := data.get("choices"):
                                     modality = data.get("modality")
-                                    content = choices[0]["delta"].get("content")
+                                    delta = choices[0].get("delta") or {}
+                                    content = delta.get("content")
+                                    if not content and isinstance(delta.get("audio"), dict):
+                                        content = delta["audio"].get("data")
                                     if modality == "text":
                                         # First token
                                         if ttft == 0.0:
@@ -182,7 +431,7 @@ async def async_request_openai_chat_omni_completions(
                                         if output.audio_ttfp == 0.0:
                                             output.audio_ttfp = timestamp - st
                                         audio_generate_time = timestamp - st
-                                        if content != "":
+                                        if content:
                                             audio_bytes = base64.b64decode(content)
                                             seg = AudioSegment.from_file(io.BytesIO(audio_bytes))
                                             if seg is not None:
@@ -214,6 +463,12 @@ async def async_request_openai_chat_omni_completions(
                         else:
                             output.audio_rtf = 0
                             logger.warning("Audio duration is zero")
+                        if _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False):
+                            try:
+                                seg = generated_audio.set_frame_rate(24000).set_channels(1).set_sample_width(2)
+                                output.tts_output_pcm_bytes = bytes(seg.raw_data)
+                            except Exception as ex:
+                                logger.warning("seed_tts WER PCM export failed: %s", ex)
                     output.success = True
                 else:
                     output.error = response.reason or ""
@@ -268,6 +523,10 @@ async def async_request_openai_audio_speech(
         "response_format": "pcm",
     }
     _update_payload_common(payload, request_func_input)
+    # Seed-TTS + WER: ``--extra-body`` may set stream=false / other formats; speech must stream PCM.
+    if getattr(request_func_input, "seed_tts_row", False) and _seed_tts_capture_pcm_for_wer():
+        payload["stream"] = True
+        payload["response_format"] = "pcm"
 
     headers = {
         "Content-Type": "application/json",
@@ -286,6 +545,8 @@ async def async_request_openai_audio_speech(
     st = time.perf_counter()
     output.start_time = st
     total_pcm_bytes = 0
+    capture_wer_pcm = _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False)
+    pcm_capture = bytearray() if capture_wer_pcm else None
     try:
         async with session.post(url=api_url, json=payload, headers=headers) as response:
             if response.status == 200:
@@ -297,6 +558,8 @@ async def async_request_openai_audio_speech(
                         output.audio_ttfp = timestamp - st
                         output.ttft = output.audio_ttfp
                     total_pcm_bytes += len(chunk)
+                    if pcm_capture is not None:
+                        pcm_capture.extend(chunk)
 
                 end_time = time.perf_counter()
                 output.latency = end_time - st
@@ -309,6 +572,16 @@ async def async_request_openai_audio_speech(
                 else:
                     output.audio_rtf = 0
                     logger.warning("Audio duration is zero")
+                if pcm_capture is not None and pcm_capture:
+                    output.tts_output_pcm_bytes = bytes(pcm_capture)
+                elif capture_wer_pcm:
+                    ct = response.headers.get("Content-Type", "")
+                    logger.warning(
+                        "Seed-TTS WER: HTTP 200 but no PCM bytes (Content-Type=%r, url=%s). "
+                        "Check stream=true and response_format=pcm on the server.",
+                        ct,
+                        api_url,
+                    )
                 output.success = True
             else:
                 output.error = response.reason or ""
@@ -331,6 +604,12 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
 if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
     OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
 
+# Daily-Omni backend for audio-visual reasoning benchmark
+# Reuses openai-chat-omni completions for video+text understanding
+ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
+if "daily-omni" not in OPENAI_COMPATIBLE_BACKENDS:
+    OPENAI_COMPATIBLE_BACKENDS.append("daily-omni")
+
 # ruff: noqa: E402
 # Prevent import order from causing patch failures
 from vllm.benchmarks import serve
@@ -422,6 +701,8 @@ async def benchmark(
         extra_headers=extra_headers,
         extra_body=extra_body,
     )
+    _attach_daily_omni_to_request_func_input(input_requests[0], test_input)
+    _attach_seed_tts_to_request_func_input(input_requests[0], test_input)
 
     if ready_check_timeout_sec > 0:
         test_output = await wait_for_endpoint(
@@ -484,6 +765,8 @@ async def benchmark(
             extra_headers=extra_headers,
             extra_body=extra_body,
         )
+        _attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
+        _attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
         profile_output = await request_func(request_func_input=profile_input, session=session)
         if profile_output.success:
             print("Profiler started")
@@ -564,6 +847,8 @@ async def benchmark(
             extra_body=extra_body,
             request_id=request_id,
         )
+        _attach_daily_omni_to_request_func_input(request, request_func_input)
+        _attach_seed_tts_to_request_func_input(request, request_func_input)
         tasks.append(
             asyncio.create_task(limited_request_func(request_func_input=request_func_input, session=session, pbar=pbar))
         )
@@ -631,6 +916,37 @@ async def benchmark(
             "errors": [output.error for output in outputs],
         }
 
+    from vllm_omni.benchmarks.data_modules.daily_omni_eval import (
+        compute_daily_omni_accuracy_metrics,
+        print_daily_omni_accuracy_summary,
+    )
+
+    _save_items = os.environ.get("DAILY_OMNI_SAVE_EVAL_ITEMS", "").lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+    _daily_acc = compute_daily_omni_accuracy_metrics(input_requests, outputs, include_per_item=_save_items)
+    if _daily_acc is not None:
+        result.update(_daily_acc)
+        print_daily_omni_accuracy_summary(_daily_acc)
+
+    if _seed_tts_capture_pcm_for_wer():
+        from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
+            compute_seed_tts_wer_metrics,
+            print_seed_tts_wer_summary,
+        )
+
+        _save_wer = os.environ.get("SEED_TTS_WER_SAVE_ITEMS", "").lower() in (
+            "1",
+            "true",
+            "yes",
+        )
+        _wer_m = compute_seed_tts_wer_metrics(input_requests, outputs, include_per_item=_save_wer)
+        if _wer_m is not None:
+            result.update(_wer_m)
+            print_seed_tts_wer_summary(_wer_m)
+
     if rps_change_events:
         result["rps_change_events"] = rps_change_events
 

vllm_omni/benchmarks/serve.py

@@ -1,9 +1,21 @@
 import argparse
 import asyncio
+import os
 from typing import Any
 
 from vllm.benchmarks.serve import main_async
 
+# Import patch to register daily-omni dataset and omni backends
+# This monkey-patches vllm.benchmarks.datasets.get_samples before it's used
+# Must be imported before any vllm.benchmarks module usage
+import vllm_omni.benchmarks.patch.patch  # noqa: F401
+
 
 def main(args: argparse.Namespace) -> dict[str, Any]:
+    if getattr(args, "seed_tts_wer_eval", False):
+        os.environ["SEED_TTS_WER_EVAL"] = "1"
+    if getattr(args, "seed_tts_wer_save_items", False):
+        os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
+    if getattr(args, "daily_omni_save_eval_items", False):
+        os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
     return asyncio.run(main_async(args))

vllm_omni/entrypoints/cli/benchmark/serve.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 
 from vllm.benchmarks.serve import add_cli_args
 
@@ -6,15 +7,149 @@ from vllm_omni.benchmarks.serve import main
 from vllm_omni.entrypoints.cli.benchmark.base import OmniBenchmarkSubcommandBase
 
 
+def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
+    """Add CLI arguments specific to Daily-Omni dataset.
+
+    This function should be called by the CLI entrypoint to add additional
+    arguments for daily-omni benchmark support.
+
+    Args:
+        parser: The ArgumentParser instance to extend
+    """
+    # Daily-Omni specific arguments
+    daily_omni_group = parser.add_argument_group("Daily-Omni Dataset Options")
+
+    daily_omni_group.add_argument(
+        "--daily-omni-qa-json",
+        type=str,
+        default=None,
+        help="Path to local upstream qa.json. When set, QA rows are read from this file and "
+        "the HuggingFace dataset is not loaded (no network). Use with --daily-omni-video-dir "
+        "for fully offline runs. --dataset-path / Hub split flags are then ignored for QA loading.",
+    )
+    daily_omni_group.add_argument(
+        "--daily-omni-video-dir",
+        type=str,
+        default=None,
+        help="Root directory of extracted Daily-Omni videos (contents of Videos.tar: "
+        "each video_id in its own subdir with {video_id}_video.mp4). "
+        "When using file URLs, you MUST start the vLLM server with "
+        "--allowed-local-media-path set to this same directory (or a parent), "
+        "otherwise requests fail with 'Cannot load local files without "
+        "--allowed-local-media-path'.",
+    )
+    daily_omni_group.add_argument(
+        "--daily-omni-inline-local-video",
+        action="store_true",
+        default=False,
+        help="For local videos only: embed MP4 as base64 data URLs in benchmark "
+        "requests so the server does not need --allowed-local-media-path. "
+        "Increases request size and client memory; use for small --num-prompts. "
+        "When using --daily-omni-input-mode audio or all, local WAV files are "
+        "embedded the same way.",
+    )
+    daily_omni_group.add_argument(
+        "--daily-omni-input-mode",
+        type=str,
+        choices=["all", "visual", "audio"],
+        default="all",
+        help="Daily-Omni input protocol (mirrors upstream Lliar-liar/Daily-Omni "
+        "--input_mode). 'visual': video only (default). 'audio': WAV only, "
+        "requires {video_id}/{video_id}_audio.wav under --daily-omni-video-dir. "
+        "'all': video + WAV together. Sets mm_processor_kwargs.use_audio_in_video=false "
+        "and matches official separate video/audio streams.",
+    )
+    daily_omni_group.add_argument(
+        "--daily-omni-save-eval-items",
+        action="store_true",
+        default=False,
+        help="Include per-request Daily-Omni accuracy rows (gold/predicted/correct) "
+        "in the saved JSON under key daily_omni_eval_items. "
+        "Alternatively set env DAILY_OMNI_SAVE_EVAL_ITEMS=1.",
+    )
+
+    # Note: --dataset-name daily-omni via get_samples patch; use either Hub (--dataset-path
+    # liarliar/Daily-Omni) or local --daily-omni-qa-json (offline).
+
+
+def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
+    """CLI for Seed-TTS zero-shot TTS benchmark (``--dataset-name seed-tts``)."""
+    g = parser.add_argument_group("Seed-TTS Dataset Options")
+    g.add_argument(
+        "--seed-tts-locale",
+        type=str,
+        choices=["en", "zh"],
+        default="en",
+        help="Which Seed-TTS split to load: en/meta.lst or zh/meta.lst under the dataset root.",
+    )
+    g.add_argument(
+        "--seed-tts-root",
+        type=str,
+        default=None,
+        help="Override root directory that contains en/ and zh/ (meta.lst + prompt-wavs). "
+        "If set, --dataset-path can still name the HF repo for logging; this path is used for files.",
+    )
+    g.add_argument(
+        "--seed-tts-file-ref-audio",
+        action="store_true",
+        default=False,
+        help="Send ref_audio as file:// URIs (smaller HTTP bodies). Requires the API server "
+        "to be started with --allowed-local-media-path covering the Seed-TTS dataset root. "
+        "Default is inline data:audio/wav;base64 so Qwen3-TTS works without that flag.",
+    )
+    g.add_argument(
+        "--seed-tts-inline-ref-audio",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+    g.add_argument(
+        "--seed-tts-system-prompt",
+        type=str,
+        default=None,
+        help="Override chat system message for --backend openai-chat-omni (Qwen3-Omni TTS). "
+        "Default follows official Qwen3-Omni identity + zero-shot voice-clone instructions.",
+    )
+    g.add_argument(
+        "--seed-tts-wer-eval",
+        action="store_true",
+        default=False,
+        help="Keep synthesized audio as 24 kHz mono PCM for WER (works with "
+        "--backend openai-audio-speech or openai-chat-omni). Scoring follows "
+        "BytedanceSpeech/seed-tts-eval (Whisper-large-v3 / Paraformer-zh + jiwer). "
+        "Sets SEED_TTS_WER_EVAL=1. Install: pip install 'vllm-omni[seed-tts-eval]'. "
+        "Optional: SEED_TTS_EVAL_DEVICE, SEED_TTS_HF_WHISPER_MODEL.",
+    )
+    g.add_argument(
+        "--seed-tts-wer-save-items",
+        action="store_true",
+        default=False,
+        help="Include per-utterance ASR rows in the saved JSON under key seed_tts_wer_eval_items. "
+        "Or set SEED_TTS_WER_SAVE_ITEMS=1.",
+    )
+
+
 class OmniBenchmarkServingSubcommand(OmniBenchmarkSubcommandBase):
     """The `serve` subcommand for vllm bench."""
 
     name = "serve"
-    help = "Benchmark the online serving throughput."
+    help = "Benchmark the online serving throughput. Supports Daily-Omni and Seed-TTS datasets."
 
     @classmethod
     def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
         add_cli_args(parser)
+
+        # Add Daily-Omni specific arguments
+        add_daily_omni_cli_args(parser)
+        add_seed_tts_cli_args(parser)
+
+        for action in parser._actions:
+            if action.dest == "dataset_name" and action.choices is not None:
+                extra = [c for c in ("daily-omni", "seed-tts") if c not in action.choices]
+                if extra:
+                    action.choices = list(action.choices) + extra
+
+        # Update help messages for omni-specific features
         for action in parser._actions:
             if action.dest == "percentile_metrics":
                 action.help = (
@@ -48,4 +183,10 @@ class OmniBenchmarkServingSubcommand(OmniBenchmarkSubcommandBase):
 
     @staticmethod
     def cmd(args: argparse.Namespace) -> None:
+        if getattr(args, "daily_omni_save_eval_items", False):
+            os.environ["DAILY_OMNI_SAVE_EVAL_ITEMS"] = "1"
+        if getattr(args, "seed_tts_wer_eval", False):
+            os.environ["SEED_TTS_WER_EVAL"] = "1"
+        if getattr(args, "seed_tts_wer_save_items", False):
+            os.environ["SEED_TTS_WER_SAVE_ITEMS"] = "1"
         main(args)