[CI] add multimodal processing correctness tests for Omni models (#1445)
Signed-off-by: zhou zhuoxin <zhouzhuoxin1508@outlook.com> Co-authored-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: Ding Zuhao <e1583181@u.nus.edu>
This commit is contained in:
2 files changed
+506
No files matched your search
@@ -0,0 +1,179 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""
|
||||
Test metadata for Omni models.
|
||||
|
||||
Mirrors vllm's ``tests/models/registry.py`` (_HfExamplesInfo) but adapted for
|
||||
the Omni multi-stage architecture.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import pytest
|
||||
from packaging.version import Version
|
||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _OmniExamplesInfo:
|
||||
"""Metadata used by processing-correctness tests for Omni models."""
|
||||
|
||||
default: str
|
||||
"""The default HF model ID to use for testing this architecture."""
|
||||
|
||||
model_stage: str = "thinker"
|
||||
"""Stage type identifier, e.g. ``"thinker"`` or ``"talker"``."""
|
||||
|
||||
hf_config_name: str | None = None
|
||||
"""
|
||||
Name of the sub-config inside the top-level HF config
|
||||
(e.g. ``"thinker_config"``). ``None`` means use the root config.
|
||||
"""
|
||||
|
||||
has_multimodal_processor: bool = False
|
||||
"""Whether the architecture registers a multimodal processor."""
|
||||
|
||||
min_transformers_version: str | None = None
|
||||
"""Minimum ``transformers`` version required to load this model."""
|
||||
|
||||
max_transformers_version: str | None = None
|
||||
"""Maximum ``transformers`` version that this model supports."""
|
||||
|
||||
is_available_online: bool = True
|
||||
"""Set to ``False`` if the model is no longer hosted on HF Hub."""
|
||||
|
||||
trust_remote_code: bool = False
|
||||
"""Whether ``trust_remote_code=True`` is needed to load the model."""
|
||||
|
||||
max_model_len: int | None = None
|
||||
"""
|
||||
An explicit ``max_model_len`` override. Useful when the default value
|
||||
from the HF config is too large for CI.
|
||||
"""
|
||||
|
||||
# ----- helper methods (same API as _HfExamplesInfo) --------------------
|
||||
|
||||
def check_transformers_version(
|
||||
self,
|
||||
*,
|
||||
on_fail: Literal["error", "skip", "return"],
|
||||
) -> str | None:
|
||||
"""Skip / error / return a message when the installed
|
||||
``transformers`` does not satisfy the version constraints."""
|
||||
if self.min_transformers_version is None and self.max_transformers_version is None:
|
||||
return None
|
||||
|
||||
current = TRANSFORMERS_VERSION
|
||||
cur_base = Version(current).base_version
|
||||
min_ver = self.min_transformers_version
|
||||
max_ver = self.max_transformers_version
|
||||
|
||||
msg = f"`transformers=={current}` installed, but `transformers"
|
||||
if min_ver and Version(cur_base) < Version(min_ver):
|
||||
msg += f">={min_ver}` is required to run this model."
|
||||
elif max_ver and Version(cur_base) > Version(max_ver):
|
||||
msg += f"<={max_ver}` is required to run this model."
|
||||
else:
|
||||
return None
|
||||
|
||||
if on_fail == "error":
|
||||
raise RuntimeError(msg)
|
||||
elif on_fail == "skip":
|
||||
pytest.skip(msg)
|
||||
return msg
|
||||
|
||||
def check_available_online(
|
||||
self,
|
||||
*,
|
||||
on_fail: Literal["error", "skip"],
|
||||
) -> None:
|
||||
"""Skip / error when the model is not available online."""
|
||||
if not self.is_available_online:
|
||||
msg = "Model is not available online"
|
||||
if on_fail == "error":
|
||||
raise RuntimeError(msg)
|
||||
else:
|
||||
pytest.skip(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Example model registry for tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_OMNI_EXAMPLE_MODELS: dict[str, _OmniExamplesInfo] = {
|
||||
# ---- Qwen2.5-Omni ----
|
||||
"Qwen2_5OmniForConditionalGeneration": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
model_stage="thinker",
|
||||
has_multimodal_processor=True,
|
||||
),
|
||||
"Qwen2_5OmniThinkerModel": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
model_stage="thinker",
|
||||
hf_config_name="thinker_config",
|
||||
has_multimodal_processor=True,
|
||||
),
|
||||
"Qwen2_5OmniTalkerModel": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
model_stage="talker",
|
||||
hf_config_name="talker_config",
|
||||
has_multimodal_processor=True,
|
||||
),
|
||||
"Qwen2_5OmniToken2WavModel": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
model_stage="token2wav",
|
||||
),
|
||||
"Qwen2_5OmniToken2WavDiTModel": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
model_stage="token2wav",
|
||||
),
|
||||
# Internal sub-model key hardcoded by Qwen2_5OmniTalkerForConditionalGeneration
|
||||
# to load its language model component (see qwen2_5_omni_talker.py).
|
||||
"Qwen2ForCausalLM_old": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen2.5-Omni-7B",
|
||||
model_stage="thinker",
|
||||
),
|
||||
# ---- Qwen3-Omni MoE ----
|
||||
"Qwen3OmniMoeForConditionalGeneration": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
model_stage="thinker",
|
||||
has_multimodal_processor=True,
|
||||
),
|
||||
"Qwen3OmniMoeThinkerForConditionalGeneration": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
model_stage="thinker",
|
||||
hf_config_name="thinker_config",
|
||||
has_multimodal_processor=True,
|
||||
),
|
||||
"Qwen3OmniMoeTalkerForConditionalGeneration": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
model_stage="talker",
|
||||
hf_config_name="talker_config",
|
||||
),
|
||||
"Qwen3OmniMoeCode2Wav": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
model_stage="code2wav",
|
||||
),
|
||||
# ---- Qwen3-TTS ----
|
||||
"Qwen3TTSForConditionalGeneration": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-TTS",
|
||||
model_stage="talker",
|
||||
),
|
||||
"Qwen3TTSTalkerForConditionalGeneration": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-TTS",
|
||||
model_stage="talker",
|
||||
),
|
||||
"Qwen3TTSCode2Wav": _OmniExamplesInfo(
|
||||
default="Qwen/Qwen3-TTS",
|
||||
model_stage="code2wav",
|
||||
),
|
||||
}
|
||||
|
||||
# Only architectures that register a multimodal processor are relevant for
|
||||
# processing-correctness tests.
|
||||
_MULTIMODAL_OMNI_EXAMPLE_MODELS: dict[str, _OmniExamplesInfo] = {
|
||||
arch: info for arch, info in _OMNI_EXAMPLE_MODELS.items() if info.has_multimodal_processor
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""
|
||||
Test multimodal processing correctness for Omni models.
|
||||
|
||||
Tests that:
|
||||
1. Cached vs non-cached processor outputs are identical.
|
||||
2. Text prompt vs token prompt inputs produce identical results after processing.
|
||||
|
||||
Adapted from vllm/tests/models/multimodal/processing/test_common.py
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
|
||||
from tests.model_executor.models.registry import (
|
||||
_MULTIMODAL_OMNI_EXAMPLE_MODELS,
|
||||
_OmniExamplesInfo,
|
||||
)
|
||||
from vllm_omni.config import OmniModelConfig
|
||||
|
||||
|
||||
def random_image(rng: np.random.RandomState, min_wh: int, max_wh: int):
|
||||
w, h = rng.randint(min_wh, max_wh, size=(2,))
|
||||
arr = rng.randint(0, 255, size=(w, h, 3), dtype=np.uint8)
|
||||
return Image.fromarray(arr)
|
||||
|
||||
|
||||
def random_video(
|
||||
rng: np.random.RandomState,
|
||||
min_frames: int,
|
||||
max_frames: int,
|
||||
min_wh: int,
|
||||
max_wh: int,
|
||||
):
|
||||
num_frames = rng.randint(min_frames, max_frames)
|
||||
w, h = rng.randint(min_wh, max_wh, size=(2,))
|
||||
return rng.randint(0, 255, size=(num_frames, w, h, 3), dtype=np.uint8)
|
||||
|
||||
|
||||
def random_audio(
|
||||
rng: np.random.RandomState,
|
||||
min_len: int,
|
||||
max_len: int,
|
||||
sr: int,
|
||||
):
|
||||
audio_len = rng.randint(min_len, max_len)
|
||||
return rng.rand(audio_len), sr
|
||||
|
||||
|
||||
_IGNORE_MM_KEYS: dict[str, set[str]] = {}
|
||||
|
||||
|
||||
def _get_model_archs_to_test() -> list[str]:
|
||||
"""Return architecture strings for parametrization."""
|
||||
return list(_MULTIMODAL_OMNI_EXAMPLE_MODELS.keys())
|
||||
|
||||
|
||||
def get_text_token_prompts(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
):
|
||||
"""Return ``(text_prompt, token_prompt)`` for the given multimodal data."""
|
||||
dummy_inputs = processor.dummy_inputs
|
||||
tokenizer: TokenizerLike = processor.info.get_tokenizer()
|
||||
model_config = processor.info.ctx.model_config
|
||||
|
||||
parsed_data = processor.info.parse_mm_data(mm_data)
|
||||
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
|
||||
|
||||
inputs = dummy_inputs.get_dummy_processor_inputs(
|
||||
model_config.max_model_len,
|
||||
mm_counts,
|
||||
mm_options={},
|
||||
)
|
||||
|
||||
text_prompt: str | None
|
||||
token_prompt: list[int]
|
||||
if isinstance(inputs.prompt, list):
|
||||
text_prompt = None
|
||||
token_prompt = inputs.prompt
|
||||
elif isinstance(inputs.prompt, str):
|
||||
text_prompt = inputs.prompt
|
||||
token_prompt = tokenizer.encode(
|
||||
text_prompt,
|
||||
**processor.info.get_default_tok_params().get_encode_kwargs(),
|
||||
)
|
||||
else:
|
||||
raise TypeError(type(inputs.prompt))
|
||||
|
||||
return text_prompt, token_prompt
|
||||
|
||||
|
||||
def _build_model_config(model_arch: str, info: _OmniExamplesInfo) -> OmniModelConfig:
|
||||
"""Create an ``OmniModelConfig`` suitable for processor testing."""
|
||||
kwargs: dict = dict(
|
||||
model=info.default,
|
||||
tokenizer=info.default,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=info.trust_remote_code,
|
||||
model_arch=model_arch,
|
||||
model_stage=info.model_stage,
|
||||
max_model_len=info.max_model_len,
|
||||
enforce_eager=True,
|
||||
dtype="auto",
|
||||
)
|
||||
if info.hf_config_name is not None:
|
||||
kwargs["hf_config_name"] = info.hf_config_name
|
||||
|
||||
model_config = OmniModelConfig(**kwargs)
|
||||
# Ensure cache is effectively unlimited to prevent eviction during test
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 2048
|
||||
return model_config
|
||||
|
||||
|
||||
def _test_processing_correctness(
|
||||
model_arch: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
"""Run the full processing-correctness test for one Omni model."""
|
||||
info = _MULTIMODAL_OMNI_EXAMPLE_MODELS[model_arch]
|
||||
info.check_transformers_version(on_fail="skip")
|
||||
info.check_available_online(on_fail="skip")
|
||||
|
||||
model_config = _build_model_config(model_arch, info)
|
||||
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
factories = model_cls._processor_factory
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
|
||||
processing_info = factories.info(ctx)
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
limit_mm_per_prompt_ints = {
|
||||
modality: 3 if limit is None else limit for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
|
||||
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
|
||||
if modality == "video":
|
||||
return VideoDummyOptions(count=count)
|
||||
if modality == "image":
|
||||
return ImageDummyOptions(count=count)
|
||||
if modality == "audio":
|
||||
return AudioDummyOptions(count=count)
|
||||
return BaseDummyOptions(count=count)
|
||||
|
||||
model_config.get_multimodal_config().limit_per_prompt = {
|
||||
modality: _to_dummy_options(modality, count) for modality, count in limit_mm_per_prompt_ints.items()
|
||||
}
|
||||
|
||||
baseline_processor = factories.build_processor(ctx, cache=None)
|
||||
cached_processor = factories.build_processor(ctx, cache=cache)
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
input_to_hit = {
|
||||
"image": Image.new("RGB", size=(128, 128)),
|
||||
"video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
|
||||
"audio": (np.zeros((512,)), 16000),
|
||||
}
|
||||
input_factory = {
|
||||
"image": partial(random_image, rng, min_wh=128, max_wh=256),
|
||||
"video": partial(random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256),
|
||||
"audio": partial(random_audio, rng, min_len=512, max_len=1024, sr=16000),
|
||||
}
|
||||
|
||||
for batch_idx in range(num_batches):
|
||||
mm_data = {
|
||||
k: [
|
||||
(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
|
||||
for _ in range(rng.randint(limit + 1))
|
||||
]
|
||||
for k, limit in limit_mm_per_prompt_ints.items()
|
||||
}
|
||||
|
||||
# Drop unnecessary keys and test single -> multi conversion
|
||||
if rng.rand() < simplify_rate:
|
||||
for k in list(mm_data.keys()):
|
||||
if not mm_data[k]:
|
||||
del mm_data[k]
|
||||
elif len(mm_data[k]) == 1:
|
||||
mm_data[k] = mm_data[k][0]
|
||||
|
||||
_test_processing_correctness_one(
|
||||
model_config,
|
||||
mm_data,
|
||||
baseline_processor,
|
||||
cached_processor,
|
||||
batch_idx,
|
||||
)
|
||||
|
||||
|
||||
def _test_processing_correctness_one(
|
||||
model_config: OmniModelConfig,
|
||||
mm_data: MultiModalDataDict,
|
||||
baseline_processor: BaseMultiModalProcessor,
|
||||
cached_processor: BaseMultiModalProcessor,
|
||||
batch_idx: int,
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
|
||||
mm_items = baseline_processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
baseline_tokenized_result = baseline_processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
cached_tokenized_result = cached_processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_tokenized_result,
|
||||
cached_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=f"Failed ({batch_idx=}, {token_prompt=}, {mm_data=})",
|
||||
)
|
||||
|
||||
if text_prompt is not None:
|
||||
baseline_text_result = baseline_processor(
|
||||
text_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
cached_text_result = cached_processor(
|
||||
text_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_text_result,
|
||||
cached_text_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=f"Failed ({batch_idx=}, {text_prompt=}, {mm_data=})",
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_text_result,
|
||||
baseline_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=f"Failed ({batch_idx=}, {text_prompt=}, {token_prompt=}, {mm_data=})",
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
cached_text_result,
|
||||
cached_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=f"Failed ({batch_idx=}, {text_prompt=}, {token_prompt=}, {mm_data=})",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("model_arch", _get_model_archs_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_omni_processing_correctness(
|
||||
model_arch: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
"""
|
||||
For each Omni model verify that:
|
||||
- cached and non-cached processor outputs are identical.
|
||||
- text-prompt and token-prompt inputs yield the same result.
|
||||
"""
|
||||
_test_processing_correctness(
|
||||
model_arch,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
)
|
||||
|
||||
|
||||
def _assert_inputs_equal(
|
||||
a: MultiModalInputs,
|
||||
b: MultiModalInputs,
|
||||
*,
|
||||
ignore_mm_keys: set[str] | None = None,
|
||||
msg: str = "",
|
||||
):
|
||||
if ignore_mm_keys is None:
|
||||
ignore_mm_keys = set()
|
||||
|
||||
ignore_prompt_keys = ("prompt", "mm_kwargs")
|
||||
a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}
|
||||
b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}
|
||||
|
||||
assert a_rest == b_rest, msg
|
||||
|
||||
a_data = a["mm_kwargs"].get_data()
|
||||
b_data = b["mm_kwargs"].get_data()
|
||||
|
||||
for key in ignore_mm_keys:
|
||||
a_data.pop(key, None)
|
||||
b_data.pop(key, None)
|
||||
|
||||
assert batched_tensors_equal(a_data, b_data), msg
|
||||
Reference in new issue
Block a user