[Rebase] Rebase to vLLM 0.29.0 (#7230)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
333f14e97e
commit
aff7d64948
120 files changed
+2958
-1690
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for GLM-Image AR model: DataParser, processor, and M-RoPE."""
|
||||
|
||||
import importlib.util
|
||||
@@ -91,6 +91,41 @@ GlmImageRotaryEmbedding = _ar_mod.GlmImageRotaryEmbedding
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_images", [0, 1, 2])
|
||||
@pytest.mark.parametrize("preformatted", [False, True])
|
||||
def test_generation_prompt_ids_preserve_hf_target_scaffold(num_images, preformatted):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessorInputs
|
||||
|
||||
processor = object.__new__(GlmImageMultiModalProcessor)
|
||||
# Only the public token attributes of GlmImageProcessor are needed; the
|
||||
# expected suffix below is what HF's own formatter produces for 512x768.
|
||||
hf_processor = SimpleNamespace(grid_bos_token="<sop>", grid_eos_token="<eop>", bos_token="<image_start>")
|
||||
tokenizer = SimpleNamespace(encode=lambda text, **kwargs: list(text.encode()))
|
||||
processor.info = SimpleNamespace(
|
||||
get_tokenizer=lambda: tokenizer,
|
||||
get_hf_processor=lambda: hf_processor,
|
||||
get_hf_config=lambda: _make_hf_config(),
|
||||
)
|
||||
items = MagicMock(spec=MultiModalDataItems)
|
||||
items.get_all_counts.return_value = {"image": num_images}
|
||||
suffix = "<sop>16 24<eop>" + ("<sop>13 19<eop>" if not num_images else "") + "<image_start>"
|
||||
text_ids = list(b"a red bird")
|
||||
expected = [167855] * num_images + text_ids + list(suffix.encode())
|
||||
inputs = ProcessorInputs(
|
||||
prompt=expected if preformatted else text_ids,
|
||||
mm_data_items=items,
|
||||
hf_processor_mm_kwargs={"target_h": 512, "target_w": 768},
|
||||
)
|
||||
with patch.object(BaseMultiModalProcessor, "apply", side_effect=lambda inputs, timing: inputs) as upstream:
|
||||
result = processor.apply(inputs, MagicMock())
|
||||
assert result.prompt == expected
|
||||
assert inputs.prompt == (expected if preformatted else text_ids)
|
||||
upstream.assert_called_once()
|
||||
|
||||
|
||||
def test_glm_image_ar_omits_hidden_pooler_payload():
|
||||
assert GlmImageForConditionalGeneration.omni_pooler_payload_include_hidden is False
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for the MiniCPM-o 4.5 ``audio_embeds`` input path.
|
||||
|
||||
Two defects made the OpenAI ``audio_embeds`` content part unusable with this model:
|
||||
@@ -71,6 +71,12 @@ class _IdentityTokenizer:
|
||||
def decode(self, tokens: str) -> str:
|
||||
return tokens
|
||||
|
||||
def convert_tokens_to_ids(self, token: str) -> int:
|
||||
# vLLM 0.29 prompt updates select placeholder positions by token id,
|
||||
# so the processor resolves ``<unk>`` once; the value is irrelevant here.
|
||||
assert token == "<unk>"
|
||||
return 0
|
||||
|
||||
|
||||
def _processing_info() -> MiniCPMO45OmniLLMProcessingInfo:
|
||||
processor = _FakeProcessor()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for MiniCPM-o 4.5 >30s audio preprocessing.
|
||||
|
||||
Covers:
|
||||
@@ -43,6 +43,34 @@ _FEAT_DIM = 128
|
||||
_CHUNK_FRAMES = 3000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [(2, 5), (4, 5)])
|
||||
def test_prompt_batching_delegates_upstream_without_tts_kwargs(version):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
info = SimpleNamespace(
|
||||
get_model_version=lambda: version,
|
||||
get_hf_processor=MagicMock(return_value=object()),
|
||||
ctx=SimpleNamespace(call_hf_processor=MagicMock()),
|
||||
)
|
||||
info.ctx.call_hf_processor.side_effect = (
|
||||
[{"features": [1]}, {"features": [2]}] if version == (2, 5) else [{"features": [1, 2]}]
|
||||
)
|
||||
kwargs = {"use_tts": True, "sampling_rate": 16000}
|
||||
result = MiniCPMO45OmniLLMMultiModalProcessor._call_hf_processor_on_prompts(
|
||||
SimpleNamespace(info=info), ["one", "two"], {"audios": [10, 20]}, kwargs, out_keys={"features"}
|
||||
)
|
||||
assert result == {"features": [1, 2]}
|
||||
assert kwargs == {"use_tts": True, "sampling_rate": 16000}
|
||||
for call in info.ctx.call_hf_processor.call_args_list:
|
||||
assert call.args[2] == {"sampling_rate": 16000}
|
||||
payloads = [call.args[1] for call in info.ctx.call_hf_processor.call_args_list]
|
||||
assert payloads == (
|
||||
[{"text": "one", "audios": 10}, {"text": "two", "audios": 20}]
|
||||
if version == (2, 5)
|
||||
else [{"text": ["one", "two"], "audios": [10, 20]}]
|
||||
)
|
||||
|
||||
|
||||
class TestMiniCPMOFieldConfig:
|
||||
def test_short_audios_stay_batched(self) -> None:
|
||||
# two <=30s audios: one feature entry + one 1-element lens per audio
|
||||
@@ -99,10 +127,10 @@ def _make_processor(
|
||||
processor.info = SimpleNamespace(audio_pattern="(<audio>./</audio>)")
|
||||
processor.data_parser = MiniCPMOMultiModalDataParser(target_sr=_SAMPLE_RATE)
|
||||
|
||||
def fake_base_call_hf_processor(prompts, mm_data, mm_kwargs, tok_kwargs, *, out_keys):
|
||||
def fake_call_hf_processor_on_prompts(prompts, mm_data, mm_kwargs, *, out_keys):
|
||||
return {key: fake_hf_outputs[key] for key in out_keys}
|
||||
|
||||
processor._base_call_hf_processor = fake_base_call_hf_processor
|
||||
processor._call_hf_processor_on_prompts = fake_call_hf_processor_on_prompts
|
||||
return processor
|
||||
|
||||
|
||||
@@ -120,7 +148,7 @@ class TestProcessAudiosUnpadding:
|
||||
}
|
||||
processor = _make_processor(fake_outputs)
|
||||
|
||||
result = processor.process_audios({"audios": [_audio_seconds(45)]}, {}, {})
|
||||
result = processor.process_audios({"audios": [_audio_seconds(45)]}, {})
|
||||
|
||||
features = result["audio_features"]
|
||||
assert [feat.shape[-1] for feat in features] == chunk_lens
|
||||
@@ -138,7 +166,7 @@ class TestProcessAudiosUnpadding:
|
||||
}
|
||||
processor = _make_processor(fake_outputs)
|
||||
|
||||
result = processor.process_audios({"audios": [_audio_seconds(42), _audio_seconds(38)]}, {}, {})
|
||||
result = processor.process_audios({"audios": [_audio_seconds(42), _audio_seconds(38)]}, {})
|
||||
|
||||
widths = [feat.shape[-1] for feat in result["audio_features"]]
|
||||
assert widths == [_CHUNK_FRAMES, 1200, _CHUNK_FRAMES, 800]
|
||||
@@ -151,7 +179,7 @@ class TestProcessAudiosUnpadding:
|
||||
}
|
||||
processor = _make_processor(fake_outputs)
|
||||
|
||||
result = processor.process_audios({"audios": [_audio_seconds(9.8)]}, {}, {})
|
||||
result = processor.process_audios({"audios": [_audio_seconds(9.8)]}, {})
|
||||
|
||||
features = result["audio_features"]
|
||||
assert len(features) == 1
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""OmniVoice processor: the prompt is tokenized exactly once per request.
|
||||
|
||||
Upstream ``_cached_apply_hf_processor`` calls ``_apply_hf_processor_main`` for
|
||||
every request, including text-only requests and requests whose reference audio
|
||||
is already cached (empty ``mm_items``). These tests pin that only ``apply``
|
||||
touches the text tokenizer and that the media path only encodes the audio.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import ProcessorInputs
|
||||
|
||||
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
|
||||
from vllm_omni.model_executor.models.omnivoice.omnivoice import OmniVoiceMultiModalProcessor
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_PROMPT_IDS = [5, 6, 7]
|
||||
|
||||
|
||||
def _make_processor(monkeypatch: pytest.MonkeyPatch) -> OmniVoiceMultiModalProcessor:
|
||||
processor = object.__new__(OmniVoiceMultiModalProcessor)
|
||||
tokenizer = SimpleNamespace(decode=Mock(return_value="hello world"))
|
||||
processor.info = SimpleNamespace(
|
||||
ctx=SimpleNamespace(
|
||||
get_hf_config=lambda *_args, **_kwargs: SimpleNamespace(sample_rate=24000),
|
||||
model_config=SimpleNamespace(model="fake-model"),
|
||||
),
|
||||
get_tokenizer=lambda: tokenizer,
|
||||
)
|
||||
processor.text_tokenizer = Mock(return_value=SimpleNamespace(input_ids=torch.tensor([_PROMPT_IDS])))
|
||||
processor.audio_tokenizer = SimpleNamespace(encode=Mock(return_value=torch.zeros(8, 4, dtype=torch.long)))
|
||||
processor.feature_extractor = None
|
||||
monkeypatch.setattr(processor, "_ensure_cached_runtime_components", lambda *_args, **_kwargs: None)
|
||||
return processor
|
||||
|
||||
|
||||
def _items(counts: dict[str, int], mm_data: dict[str, object]) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
get_all_counts=lambda: counts,
|
||||
select=lambda _keys: mm_data,
|
||||
)
|
||||
|
||||
|
||||
def test_apply_tokenizes_the_prompt_once(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
processor = _make_processor(monkeypatch)
|
||||
captured: dict[str, ProcessorInputs] = {}
|
||||
|
||||
def fake_parent_apply(self, inputs, timing_ctx):
|
||||
captured["inputs"] = inputs
|
||||
return "sentinel"
|
||||
|
||||
monkeypatch.setattr(OmniMultiModalProcessor, "apply", fake_parent_apply)
|
||||
inputs = ProcessorInputs(
|
||||
prompt=[1, 2],
|
||||
mm_data_items=MultiModalDataItems({}),
|
||||
hf_processor_mm_kwargs={"ref_text": "reference"},
|
||||
)
|
||||
|
||||
assert processor.apply(inputs, timing_ctx=None) == "sentinel"
|
||||
assert captured["inputs"].prompt == _PROMPT_IDS
|
||||
assert processor.text_tokenizer.call_count == 1
|
||||
(text_prompt,), _ = processor.text_tokenizer.call_args
|
||||
assert text_prompt.endswith("<|text_start|>reference hello world<|text_end|>")
|
||||
assert captured["inputs"].hf_processor_mm_kwargs["ref_text"] == "reference"
|
||||
|
||||
|
||||
def test_media_path_encodes_audio_without_retokenizing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
processor = _make_processor(monkeypatch)
|
||||
audio = np.zeros(2400, dtype=np.float32)
|
||||
monkeypatch.setattr(processor, "_get_hf_mm_data", lambda _items: ({"audios": [audio]}, {}))
|
||||
|
||||
out = processor._apply_hf_processor_main(_items({"audio": 1}, {}), {"ref_text": "reference"})
|
||||
|
||||
assert tuple(out["ref_audio_tokens"].shape) == (8, 4)
|
||||
assert list(out["ref_audio_len"]) == [4]
|
||||
assert "input_ids" not in out
|
||||
assert processor.audio_tokenizer.encode.call_count == 1
|
||||
assert processor.text_tokenizer.call_count == 0
|
||||
|
||||
|
||||
def test_media_path_with_no_audio_touches_no_tokenizer(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Text-only requests and fully cached audio both reach this path with empty items."""
|
||||
processor = _make_processor(monkeypatch)
|
||||
monkeypatch.setattr(processor, "_get_hf_mm_data", lambda _items: ({}, {}))
|
||||
|
||||
out = processor._apply_hf_processor_main(_items({}, {}), {"ref_text": "reference"})
|
||||
|
||||
assert len(out.keys()) == 0
|
||||
assert processor.text_tokenizer.call_count == 0
|
||||
assert processor.audio_tokenizer.encode.call_count == 0
|
||||
|
||||
|
||||
def test_legacy_call_hf_processor_is_gone() -> None:
|
||||
assert not hasattr(OmniVoiceMultiModalProcessor, "_call_hf_processor")
|
||||
@@ -266,8 +266,6 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
|
||||
inputs = MagicMock()
|
||||
inputs.mm_data_items = mm_data_items
|
||||
inputs.hf_processor_mm_kwargs = {"use_audio_in_video": True}
|
||||
inputs.tokenization_kwargs = {}
|
||||
inputs.prompt = [1, 2, 3]
|
||||
inputs.get_mm_hashes = MagicMock(return_value={"video": ["v0"], "audio": ["a0"]})
|
||||
|
||||
timing_ctx = MagicMock()
|
||||
@@ -307,7 +305,7 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
|
||||
)
|
||||
fake_self._paired_cache_keys = Qwen2_5OmniThinkerMultiModalProcessor._paired_cache_keys
|
||||
fake_self.info.parse_mm_data = MagicMock(return_value=SimpleNamespace(name="missing"))
|
||||
fake_self._apply_hf_processor_main = MagicMock(return_value=([9], {"video_grid_thw": object()}, False))
|
||||
fake_self._apply_hf_processor_main = MagicMock(return_value={"video_grid_thw": object()})
|
||||
fake_self._get_mm_fields_config = MagicMock(return_value={})
|
||||
fake_self._get_mm_prompt_updates = MagicMock(return_value={"video": [[object()]], "audio": [[object()]]})
|
||||
fake_self._merge_mm_kwargs = capture_merge
|
||||
@@ -316,13 +314,12 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
|
||||
"vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.MultiModalKwargsItems.from_hf_inputs",
|
||||
return_value={"video": [object()], "audio": [object()]},
|
||||
):
|
||||
prompt_ids, mm_info, _ = Qwen2_5OmniThinkerMultiModalProcessor._cached_apply_hf_processor(
|
||||
mm_info = Qwen2_5OmniThinkerMultiModalProcessor._cached_apply_hf_processor(
|
||||
fake_self,
|
||||
inputs,
|
||||
timing_ctx,
|
||||
)
|
||||
|
||||
assert prompt_ids == [9]
|
||||
# Both modalities were reprocessed as one unit after pair-miss expansion.
|
||||
assert fake_self.info.parse_mm_data.call_args.args[0] == {
|
||||
"video": ["video-0"],
|
||||
@@ -623,10 +620,17 @@ def test_derive_audio_from_video_placeholders_only_pairs_true_videos():
|
||||
|
||||
|
||||
def test_qwen3_processor_inherits_vllm_omni_per_video_helpers():
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor as UpstreamQwen3Processor,
|
||||
)
|
||||
|
||||
assert issubclass(
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
)
|
||||
assert Qwen3OmniMoeThinkerMultiModalProcessor._get_hf_mm_data is UpstreamQwen3Processor._get_hf_mm_data
|
||||
mro = Qwen3OmniMoeThinkerMultiModalProcessor.__mro__
|
||||
assert mro.index(Qwen2_5OmniThinkerMultiModalProcessor) < mro.index(UpstreamQwen3Processor)
|
||||
|
||||
|
||||
def test_qwen3_prompt_updates_prefer_hf_second_per_grid_ts_from_outputs():
|
||||
|
||||
@@ -113,7 +113,9 @@ def _save_peft_adapter(tmp_path: Path) -> Path:
|
||||
return adapter_dir
|
||||
|
||||
|
||||
def test_peft_expert_lora_loads_into_thinker(tmp_path: Path):
|
||||
def test_peft_expert_lora_loads_into_thinker(tmp_path: Path, monkeypatch):
|
||||
# This CPU harness tests weight mapping, not CUDA pinned-memory allocation.
|
||||
monkeypatch.setattr("vllm.lora.lora_model.PIN_MEMORY", False)
|
||||
adapter_dir = _save_peft_adapter(tmp_path)
|
||||
peft_helper = PEFTHelper.from_local_dir(str(adapter_dir), max_position_embeddings=128)
|
||||
lora_model = LoRAModel.from_local_checkpoint(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
@@ -157,7 +157,10 @@ def _load_weights_noop(model: Qwen3TTSCode2Wav) -> set[str]:
|
||||
def __init__(self, *_: object, **__: object):
|
||||
pass
|
||||
|
||||
def load_weights(self, _weights: object) -> set[str]:
|
||||
def load_weights(self, _weights: object, *, mapper: object | None = None) -> set[str]:
|
||||
# vLLM 0.29 dropped skip_prefixes from AutoWeightsLoader; the model
|
||||
# now passes the stage exclusions as a WeightsMapper instead.
|
||||
del mapper
|
||||
return {"decoder.fake_weight"}
|
||||
|
||||
with (
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _TinyModel(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.kept = nn.Linear(1, 1, bias=False)
|
||||
self.skipped = nn.Linear(1, 1, bias=False)
|
||||
self.rotary_embed = nn.Module()
|
||||
self.rotary_embed.register_parameter("inv_freq", nn.Parameter(torch.zeros(1)))
|
||||
|
||||
|
||||
def test_upstream_mapper_preserves_stage_exclusions_after_renaming() -> None:
|
||||
model = _TinyModel()
|
||||
model.skipped.weight.data.zero_()
|
||||
|
||||
loaded = AutoWeightsLoader(model).load_weights(
|
||||
[
|
||||
("checkpoint.kept.weight", torch.ones(1, 1)),
|
||||
("checkpoint.skipped.weight", torch.full((1, 1), 2.0)),
|
||||
("rotary_embed.inv_freq", torch.ones(1)),
|
||||
],
|
||||
mapper=WeightsMapper(orig_to_new_prefix={"checkpoint.": ""})
|
||||
| WeightsMapper(orig_to_new_prefix={"skipped.": None}, orig_to_new_substr={"rotary_embed.inv_freq": None}),
|
||||
)
|
||||
|
||||
assert loaded == {"kept.weight"}
|
||||
torch.testing.assert_close(model.kept.weight, torch.ones(1, 1))
|
||||
torch.testing.assert_close(model.skipped.weight, torch.zeros(1, 1))
|
||||
torch.testing.assert_close(model.rotary_embed.inv_freq, torch.zeros(1))
|
||||
Reference in new issue
Block a user