vllm-omni/vllm_omni/entrypoints/cli/benchmark/cli_args.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

"""vLLM-Omni extensions for the ``vllm bench serve`` CLI.

Core functions:
    add_omni_args: Register all Omni-specific argument groups by calling the
        feature-specific ``add_*_cli_args`` helpers in this module.
    extend_omni_choices: Add Omni datasets and backends to choices defined by
        the upstream vLLM parser, including its shadow parser.
    update_omni_help: Extend upstream help text with Omni-specific behavior.
    preprocess_serve_args: Apply transformations that require parsed values
        before the serving benchmark starts.

``OmniBenchmarkServingSubcommand.add_cli_args`` invokes the first three after
upstream vLLM registers its arguments. New feature arguments should normally be
added to the corresponding ``add_*_cli_args`` helper, or to a new helper called
by ``add_omni_args``.
"""

import argparse
import math
from pathlib import Path

_DEFAULT_OMNIINTERACT_NUM_PROMPTS = 3


def _positive_finite_float(value: str) -> float:
    parsed = float(value)
    if not math.isfinite(parsed) or parsed <= 0:
        raise argparse.ArgumentTypeError(f"must be a finite positive number, got {value!r}")
    return parsed


def _existing_file(value: str) -> str:
    path = Path(value).expanduser()
    if not path.is_file():
        raise argparse.ArgumentTypeError(f"file does not exist: {value!r}")
    return str(path)


def add_omniinteract_cli_args(parser: argparse.ArgumentParser) -> None:
    from vllm_omni.benchmarks.data_modules.omniinteract_dataset import (
        DEFAULT_MAX_VIDEO_DURATION_S,
        OMNIINTERACT_SUBSETS,
    )

    group = parser.add_argument_group("OmniInteract Benchmark Options")
    group.add_argument(
        "--omniinteract-subsets", nargs="+", choices=OMNIINTERACT_SUBSETS, default=list(OMNIINTERACT_SUBSETS)
    )
    group.add_argument(
        "--omniinteract-timeout-s", type=_positive_finite_float, default=900.0, help="Complete session timeout."
    )
    group.add_argument(
        "--omniinteract-media-timeout-s",
        type=_positive_finite_float,
        default=600.0,
        help="Per-command media timeout.",
    )
    group.add_argument(
        "--omniinteract-max-video-duration-s",
        type=_positive_finite_float,
        default=DEFAULT_MAX_VIDEO_DURATION_S,
        help="Reject media longer than this safety limit before decoding.",
    )
    group.add_argument(
        "--omniinteract-ref-audio", type=_existing_file, help="Reference WAV for native-duplex audio output."
    )
    group.add_argument(
        "--omniinteract-require-response", action="store_true", help="Fail LISTEN-only functional E2E cases."
    )
    group.add_argument(
        "--omniinteract-output-dir",
        type=Path,
        default=Path("omniinteract-output"),
        help="Directory for case and evaluator artifacts.",
    )


def add_multi_stage_cli_args(parser: argparse.ArgumentParser) -> None:
    """Add CLI arguments for vLLM-Omni multi-stage benchmarks."""
    group = parser.add_argument_group("vLLM-Omni Multi-stage Benchmark Options")
    group.add_argument(
        "--print-stage",
        action="store_true",
        default=False,
        help=(
            "Print per-stage benchmark metrics for --omni serving when stage metrics are returned by the server. "
            "Disabled by default. The latency sections follow --percentile-metrics by modality: "
            "ttft/tpot/itl control text stages, ttfc/tpoc/icl control internal stream stages, "
            "and tpop controls both text TPOP and internal stream TPOP."
        ),
    )


def add_diffusion_cli_args(parser: argparse.ArgumentParser) -> None:
    """Add CLI arguments for diffusion model benchmarks."""
    group = parser.add_argument_group("Diffusion Models Options")
    group.add_argument(
        "--image-edits-bot-task",
        dest="bot_task",
        type=str,
        default="think",
        help=(
            "Default bot_task form field for --backend openai-image-edits-omni "
            "(/v1/images/edits). "
            'Use --extra-body \'{"bot_task":"..."}\' to override per run.'
        ),
    )


def add_daily_omni_cli_args(parser: argparse.ArgumentParser) -> None:
    """Add CLI arguments specific to the Daily-Omni dataset."""
    group = parser.add_argument_group("Daily-Omni Dataset Options")
    group.add_argument(
        "--daily-omni-qa-json",
        type=str,
        default=None,
        help="Path to local upstream qa.json. When set, QA rows are read from this file and "
        "the HuggingFace dataset is not loaded (no network). Use with --daily-omni-video-dir "
        "for fully offline runs. --dataset-path / Hub split flags are then ignored for QA loading.",
    )
    group.add_argument(
        "--daily-omni-video-dir",
        type=str,
        default=None,
        help="Root directory of extracted Daily-Omni videos (contents of Videos.tar: "
        "each video_id in its own subdir with {video_id}_video.mp4). "
        "If omitted, Videos.tar is downloaded from the Hugging Face dataset repo on first multimodal "
        "request. "
        "When using file URLs, you MUST start the vLLM server with "
        "--allowed-local-media-path set to this same directory (or a parent), "
        "otherwise requests fail with 'Cannot load local files without "
        "--allowed-local-media-path'.",
    )
    group.add_argument(
        "--daily-omni-inline-local-video",
        action="store_true",
        default=False,
        help="For local videos only: embed MP4 as base64 data URLs in benchmark "
        "requests so the server does not need --allowed-local-media-path. "
        "Increases request size and client memory; use for small --num-prompts. "
        "When using --daily-omni-input-mode audio or all, local WAV files are "
        "embedded the same way.",
    )
    group.add_argument(
        "--daily-omni-input-mode",
        type=str,
        choices=["all", "visual", "audio"],
        default="all",
        help="Daily-Omni input protocol (mirrors upstream Lliar-liar/Daily-Omni "
        "--input_mode). 'visual': video only (default). 'audio': WAV only, "
        "requires {video_id}/{video_id}_audio.wav under --daily-omni-video-dir. "
        "'all': video + WAV together. Sets mm_processor_kwargs.use_audio_in_video=false "
        "and matches official separate video/audio streams.",
    )
    group.add_argument(
        "--daily-omni-pack-mode",
        type=str,
        choices=["qwen", "minicpm-interleave"],
        default="qwen",
        help="How to pack multimodal parts into OpenAI chat messages. "
        "'qwen' (default): one video_url + one audio_url (Daily-Omni/Qwen protocol). "
        "'minicpm-interleave': MiniCPM-o official recipe — 1fps frames interleaved with "
        "matching 1s audio segments as image_url/audio_url pairs (needed to approach "
        "OpenBMB ~80%% Daily-Omni; requires local Videos extract). By default this mode "
        "writes JPEG/WAV segments under "
        "<daily-omni-video-dir>/.minicpm_daily_omni_interleave and sends file:// URLs, "
        "so start the server with --allowed-local-media-path <daily-omni-video-dir> "
        "(or a parent). Alternatively pass --daily-omni-inline-local-video to embed "
        "segments as data URLs. For MiniCPM-o 4.5 string chat templates also start the "
        "server with --interleave-mm-strings.",
    )
    group.add_argument(
        "--daily-omni-save-eval-items",
        action="store_true",
        default=False,
        help="Include per-request Daily-Omni accuracy rows (gold/predicted/correct) "
        "in the saved JSON under key daily_omni_eval_items. "
        "Alternatively set env DAILY_OMNI_SAVE_EVAL_ITEMS=1.",
    )


def add_seed_tts_cli_args(parser: argparse.ArgumentParser) -> None:
    """Add CLI arguments for Seed-TTS benchmarks."""
    group = parser.add_argument_group("Seed-TTS Dataset Options")
    group.add_argument(
        "--seed-tts-locale",
        type=str,
        choices=["en", "zh"],
        default="en",
        help="Which Seed-TTS split to load: en/meta.lst or zh/meta.lst under the dataset root.",
    )
    group.add_argument(
        "--seed-tts-turns-per-session",
        type=int,
        default=1,
        help="Group this many Seed-TTS target texts into one Realtime session. "
        "The first row's reference audio and transcript are reused for every turn.",
    )
    group.add_argument(
        "--seed-tts-root",
        type=str,
        default=None,
        help="Override root directory that contains en/ and zh/ (meta.lst + prompt-wavs). "
        "If set, --dataset-path can still name the HF repo for logging; this path is used for files.",
    )
    group.add_argument(
        "--seed-tts-file-ref-audio",
        action="store_true",
        default=False,
        help="Send ref_audio as file:// URIs (smaller HTTP bodies). Requires the API server "
        "to be started with --allowed-local-media-path covering the Seed-TTS dataset root. "
        "Default is inline data:audio/wav;base64 so Qwen3-TTS works without that flag.",
    )
    group.add_argument(
        "--seed-tts-inline-ref-audio",
        action="store_true",
        default=False,
        help=argparse.SUPPRESS,
    )
    group.add_argument(
        "--seed-tts-system-prompt",
        type=str,
        default=None,
        help="Override chat system message for --backend openai-chat-omni (Qwen3-Omni TTS). "
        "Default follows official Qwen3-Omni identity + zero-shot voice-clone instructions.",
    )
    group.add_argument(
        "--seed-tts-wer-eval",
        action="store_true",
        default=False,
        help="Keep synthesized audio as 24 kHz mono PCM for WER (works with "
        "--backend openai-audio-speech or openai-chat-omni). Scoring follows "
        "zhaochenyang20/seed-tts-eval (Whisper-large-v3 / Paraformer-zh + jiwer). "
        "Sets SEED_TTS_WER_EVAL=1. Install: pip install 'vllm-omni[dev]'. "
        "Optional: SEED_TTS_EVAL_DEVICE, SEED_TTS_HF_WHISPER_MODEL.",
    )
    group.add_argument(
        "--seed-tts-wer-save-items",
        action="store_true",
        default=False,
        help="Include per-utterance ASR rows in the saved JSON under key seed_tts_wer_eval_items. "
        "Or set SEED_TTS_WER_SAVE_ITEMS=1.",
    )


_OMNI_BENCH_DATASET_CHOICES = (
    "daily-omni",
    "omniinteract",
    "seed-tts",
    "seed-tts-text",
    "seed-tts-design",
    "ttsd",
    "sound-effect",
)


def extend_omni_choices(parser: argparse.ArgumentParser) -> None:
    """Extend upstream argument choices with Omni-specific values."""
    parsers = [parser]
    shadow = getattr(parser, "_shadow", None)
    if shadow is not None:
        parsers.append(shadow)

    for current_parser in parsers:
        for action in current_parser._actions:
            if action.dest == "dataset_name" and action.choices is not None:
                extra = [choice for choice in _OMNI_BENCH_DATASET_CHOICES if choice not in action.choices]
                if extra:
                    action.choices = list(action.choices) + extra
            if action.dest == "backend" and action.choices is not None:
                extra = [choice for choice in ("openai-image-edits-omni",) if choice not in action.choices]
                if extra:
                    action.choices = list(action.choices) + extra


def update_omni_help(parser: argparse.ArgumentParser) -> None:
    """Update upstream argument help text to describe Omni-specific behavior."""
    for action in parser._actions:
        if action.dest == "num_prompts":
            action.help = (
                f"{action.help} OmniInteract uses {_DEFAULT_OMNIINTERACT_NUM_PROMPTS} when this option is omitted; "
                "0 selects all available cases."
            )
        if action.dest == "percentile_metrics":
            action.help = (
                "Comma-separated list of selected metrics to report percentiles. "
                'For text metrics, "ttft", "tpot", and "itl" affect the global benchmark and text '
                'stage metrics. "tpop" also requests text TPOT/TPOP globally and per stage, and internal '
                'stream TPOP. "ttfc", "tpoc", and "icl" only affect internal stream stage metrics. '
                'Audio metrics include "audio_ttfp", "audio_rtf", "audio_duration", and "audio_underrun".'
            )
        if action.dest == "random_mm_limit_mm_per_prompt":
            action.help = (
                "Per-modality hard caps for items attached per request, e.g. "
                '\'{"image": 3, "video": 0, "audio": 1}\'. The sampled per-request item '
                "count is clamped to the sum of these limits. When a modality "
                "reaches its cap, its buckets are excluded and probabilities are "
                "renormalized."
            )
        if action.dest == "random_mm_bucket_config":
            action.help = (
                "The bucket config is a dictionary mapping a multimodal item"
                "sampling configuration to a probability."
                "Currently allows for 3 modalities: audio, images and videos. "
                "A bucket key is a tuple of (height, width, num_frames)"
                "The value is the probability of sampling that specific item. "
                "Example: "
                "--random-mm-bucket-config "
                "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10} "
                "First item: images with resolution 256x256 w.p. 0.5"
                "Second item: videos with resolution 720x1280 and 16 frames "
                "Third item: audios with 1s duration and 5 channels w.p. 0.1"
                "OBS.: If the probabilities do not sum to 1, they are normalized."
            )


def add_omni_args(parser: argparse.ArgumentParser) -> None:
    """Register all vLLM-Omni serving benchmark arguments."""
    add_daily_omni_cli_args(parser)
    add_omniinteract_cli_args(parser)
    add_seed_tts_cli_args(parser)
    add_multi_stage_cli_args(parser)
    add_diffusion_cli_args(parser)


def preprocess_serve_args(args: argparse.Namespace) -> None:
    """Apply serving benchmark CLI transformations after parsing."""
    if getattr(args, "dataset_name", None) == "omniinteract":
        if getattr(args, "backend", None) != "openai-realtime-duplex":
            raise ValueError("OmniInteract requires --backend openai-realtime-duplex")
        if getattr(args, "endpoint", None) != "/v1/realtime":
            raise ValueError("OmniInteract requires --endpoint /v1/realtime")
        if not getattr(args, "omniinteract_ref_audio", None):
            raise ValueError("OmniInteract requires --omniinteract-ref-audio")
        if getattr(args, "ignore_eos", False):
            raise ValueError("OmniInteract does not support --ignore-eos")
        if getattr(args, "profile", False):
            raise ValueError("OmniInteract does not support --profile")
        if getattr(args, "skip_tokenizer_init", False):
            raise ValueError("OmniInteract does not support --skip-tokenizer-init")
        if float(getattr(args, "probe_request_rate", 0.0) or 0.0) > 0:
            raise ValueError("OmniInteract does not support --probe-request-rate")
        if "num_prompts" not in getattr(args, "explicit_keys", ()):
            args.num_prompts = _DEFAULT_OMNIINTERACT_NUM_PROMPTS
        max_concurrency = getattr(args, "max_concurrency", None)
        if max_concurrency is None:
            args.max_concurrency = 1
        elif max_concurrency <= 0:
            raise ValueError("OmniInteract requires --max-concurrency to be positive")
    extra_body = dict(getattr(args, "extra_body", None) or {})
    bot_task = getattr(args, "bot_task", None)
    if getattr(args, "backend", None) == "openai-image-edits-omni" and bot_task is not None:
        extra_body.setdefault("bot_task", bot_task)
    args.extra_body = extra_body