[Rebase] Rebase to vLLM 0.29.0 (#7230)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Signed-off-by: Zhou Taichang <tzhouam@connect.ust.hk>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Zhou TaichangandClaude Opus 5 authored and GitHub committed 2026-09-10 12:08:28 +08:00
1 parent 333f14e97e
commit aff7d64948
120 files changed
+2958 -1690

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for GLM-Image AR model: DataParser, processor, and M-RoPE."""
import importlib.util
@@ -91,6 +91,41 @@ GlmImageRotaryEmbedding = _ar_mod.GlmImageRotaryEmbedding
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize("num_images", [0, 1, 2])
@pytest.mark.parametrize("preformatted", [False, True])
def test_generation_prompt_ids_preserve_hf_target_scaffold(num_images, preformatted):
from types import SimpleNamespace
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessorInputs
processor = object.__new__(GlmImageMultiModalProcessor)
# Only the public token attributes of GlmImageProcessor are needed; the
# expected suffix below is what HF's own formatter produces for 512x768.
hf_processor = SimpleNamespace(grid_bos_token="<sop>", grid_eos_token="<eop>", bos_token="<image_start>")
tokenizer = SimpleNamespace(encode=lambda text, **kwargs: list(text.encode()))
processor.info = SimpleNamespace(
get_tokenizer=lambda: tokenizer,
get_hf_processor=lambda: hf_processor,
get_hf_config=lambda: _make_hf_config(),
)
items = MagicMock(spec=MultiModalDataItems)
items.get_all_counts.return_value = {"image": num_images}
suffix = "<sop>16 24<eop>" + ("<sop>13 19<eop>" if not num_images else "") + "<image_start>"
text_ids = list(b"a red bird")
expected = [167855] * num_images + text_ids + list(suffix.encode())
inputs = ProcessorInputs(
prompt=expected if preformatted else text_ids,
mm_data_items=items,
hf_processor_mm_kwargs={"target_h": 512, "target_w": 768},
)
with patch.object(BaseMultiModalProcessor, "apply", side_effect=lambda inputs, timing: inputs) as upstream:
result = processor.apply(inputs, MagicMock())
assert result.prompt == expected
assert inputs.prompt == (expected if preformatted else text_ids)
upstream.assert_called_once()
def test_glm_image_ar_omits_hidden_pooler_payload():
assert GlmImageForConditionalGeneration.omni_pooler_payload_include_hidden is False
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for the MiniCPM-o 4.5 ``audio_embeds`` input path.
Two defects made the OpenAI ``audio_embeds`` content part unusable with this model:
@@ -71,6 +71,12 @@ class _IdentityTokenizer:
def decode(self, tokens: str) -> str:
return tokens
def convert_tokens_to_ids(self, token: str) -> int:
# vLLM 0.29 prompt updates select placeholder positions by token id,
# so the processor resolves ``<unk>`` once; the value is irrelevant here.
assert token == "<unk>"
return 0
def _processing_info() -> MiniCPMO45OmniLLMProcessingInfo:
processor = _FakeProcessor()
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for MiniCPM-o 4.5 >30s audio preprocessing.
Covers:
@@ -43,6 +43,34 @@ _FEAT_DIM = 128
_CHUNK_FRAMES = 3000
@pytest.mark.parametrize("version", [(2, 5), (4, 5)])
def test_prompt_batching_delegates_upstream_without_tts_kwargs(version):
from unittest.mock import MagicMock
info = SimpleNamespace(
get_model_version=lambda: version,
get_hf_processor=MagicMock(return_value=object()),
ctx=SimpleNamespace(call_hf_processor=MagicMock()),
)
info.ctx.call_hf_processor.side_effect = (
[{"features": [1]}, {"features": [2]}] if version == (2, 5) else [{"features": [1, 2]}]
)
kwargs = {"use_tts": True, "sampling_rate": 16000}
result = MiniCPMO45OmniLLMMultiModalProcessor._call_hf_processor_on_prompts(
SimpleNamespace(info=info), ["one", "two"], {"audios": [10, 20]}, kwargs, out_keys={"features"}
)
assert result == {"features": [1, 2]}
assert kwargs == {"use_tts": True, "sampling_rate": 16000}
for call in info.ctx.call_hf_processor.call_args_list:
assert call.args[2] == {"sampling_rate": 16000}
payloads = [call.args[1] for call in info.ctx.call_hf_processor.call_args_list]
assert payloads == (
[{"text": "one", "audios": 10}, {"text": "two", "audios": 20}]
if version == (2, 5)
else [{"text": ["one", "two"], "audios": [10, 20]}]
)
class TestMiniCPMOFieldConfig:
def test_short_audios_stay_batched(self) -> None:
# two <=30s audios: one feature entry + one 1-element lens per audio
@@ -99,10 +127,10 @@ def _make_processor(
processor.info = SimpleNamespace(audio_pattern="(<audio>./</audio>)")
processor.data_parser = MiniCPMOMultiModalDataParser(target_sr=_SAMPLE_RATE)
def fake_base_call_hf_processor(prompts, mm_data, mm_kwargs, tok_kwargs, *, out_keys):
def fake_call_hf_processor_on_prompts(prompts, mm_data, mm_kwargs, *, out_keys):
return {key: fake_hf_outputs[key] for key in out_keys}
processor._base_call_hf_processor = fake_base_call_hf_processor
processor._call_hf_processor_on_prompts = fake_call_hf_processor_on_prompts
return processor
@@ -120,7 +148,7 @@ class TestProcessAudiosUnpadding:
}
processor = _make_processor(fake_outputs)
result = processor.process_audios({"audios": [_audio_seconds(45)]}, {}, {})
result = processor.process_audios({"audios": [_audio_seconds(45)]}, {})
features = result["audio_features"]
assert [feat.shape[-1] for feat in features] == chunk_lens
@@ -138,7 +166,7 @@ class TestProcessAudiosUnpadding:
}
processor = _make_processor(fake_outputs)
result = processor.process_audios({"audios": [_audio_seconds(42), _audio_seconds(38)]}, {}, {})
result = processor.process_audios({"audios": [_audio_seconds(42), _audio_seconds(38)]}, {})
widths = [feat.shape[-1] for feat in result["audio_features"]]
assert widths == [_CHUNK_FRAMES, 1200, _CHUNK_FRAMES, 800]
@@ -151,7 +179,7 @@ class TestProcessAudiosUnpadding:
}
processor = _make_processor(fake_outputs)
result = processor.process_audios({"audios": [_audio_seconds(9.8)]}, {}, {})
result = processor.process_audios({"audios": [_audio_seconds(9.8)]}, {})
features = result["audio_features"]
assert len(features) == 1
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""OmniVoice processor: the prompt is tokenized exactly once per request.
Upstream ``_cached_apply_hf_processor`` calls ``_apply_hf_processor_main`` for
every request, including text-only requests and requests whose reference audio
is already cached (empty ``mm_items``). These tests pin that only ``apply``
touches the text tokenizer and that the media path only encodes the audio.
"""
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
import torch
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import ProcessorInputs
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
from vllm_omni.model_executor.models.omnivoice.omnivoice import OmniVoiceMultiModalProcessor
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_PROMPT_IDS = [5, 6, 7]
def _make_processor(monkeypatch: pytest.MonkeyPatch) -> OmniVoiceMultiModalProcessor:
processor = object.__new__(OmniVoiceMultiModalProcessor)
tokenizer = SimpleNamespace(decode=Mock(return_value="hello world"))
processor.info = SimpleNamespace(
ctx=SimpleNamespace(
get_hf_config=lambda *_args, **_kwargs: SimpleNamespace(sample_rate=24000),
model_config=SimpleNamespace(model="fake-model"),
),
get_tokenizer=lambda: tokenizer,
)
processor.text_tokenizer = Mock(return_value=SimpleNamespace(input_ids=torch.tensor([_PROMPT_IDS])))
processor.audio_tokenizer = SimpleNamespace(encode=Mock(return_value=torch.zeros(8, 4, dtype=torch.long)))
processor.feature_extractor = None
monkeypatch.setattr(processor, "_ensure_cached_runtime_components", lambda *_args, **_kwargs: None)
return processor
def _items(counts: dict[str, int], mm_data: dict[str, object]) -> SimpleNamespace:
return SimpleNamespace(
get_all_counts=lambda: counts,
select=lambda _keys: mm_data,
)
def test_apply_tokenizes_the_prompt_once(monkeypatch: pytest.MonkeyPatch) -> None:
processor = _make_processor(monkeypatch)
captured: dict[str, ProcessorInputs] = {}
def fake_parent_apply(self, inputs, timing_ctx):
captured["inputs"] = inputs
return "sentinel"
monkeypatch.setattr(OmniMultiModalProcessor, "apply", fake_parent_apply)
inputs = ProcessorInputs(
prompt=[1, 2],
mm_data_items=MultiModalDataItems({}),
hf_processor_mm_kwargs={"ref_text": "reference"},
)
assert processor.apply(inputs, timing_ctx=None) == "sentinel"
assert captured["inputs"].prompt == _PROMPT_IDS
assert processor.text_tokenizer.call_count == 1
(text_prompt,), _ = processor.text_tokenizer.call_args
assert text_prompt.endswith("<|text_start|>reference hello world<|text_end|>")
assert captured["inputs"].hf_processor_mm_kwargs["ref_text"] == "reference"
def test_media_path_encodes_audio_without_retokenizing(monkeypatch: pytest.MonkeyPatch) -> None:
processor = _make_processor(monkeypatch)
audio = np.zeros(2400, dtype=np.float32)
monkeypatch.setattr(processor, "_get_hf_mm_data", lambda _items: ({"audios": [audio]}, {}))
out = processor._apply_hf_processor_main(_items({"audio": 1}, {}), {"ref_text": "reference"})
assert tuple(out["ref_audio_tokens"].shape) == (8, 4)
assert list(out["ref_audio_len"]) == [4]
assert "input_ids" not in out
assert processor.audio_tokenizer.encode.call_count == 1
assert processor.text_tokenizer.call_count == 0
def test_media_path_with_no_audio_touches_no_tokenizer(monkeypatch: pytest.MonkeyPatch) -> None:
"""Text-only requests and fully cached audio both reach this path with empty items."""
processor = _make_processor(monkeypatch)
monkeypatch.setattr(processor, "_get_hf_mm_data", lambda _items: ({}, {}))
out = processor._apply_hf_processor_main(_items({}, {}), {"ref_text": "reference"})
assert len(out.keys()) == 0
assert processor.text_tokenizer.call_count == 0
assert processor.audio_tokenizer.encode.call_count == 0
def test_legacy_call_hf_processor_is_gone() -> None:
assert not hasattr(OmniVoiceMultiModalProcessor, "_call_hf_processor")
@@ -266,8 +266,6 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
inputs = MagicMock()
inputs.mm_data_items = mm_data_items
inputs.hf_processor_mm_kwargs = {"use_audio_in_video": True}
inputs.tokenization_kwargs = {}
inputs.prompt = [1, 2, 3]
inputs.get_mm_hashes = MagicMock(return_value={"video": ["v0"], "audio": ["a0"]})
timing_ctx = MagicMock()
@@ -307,7 +305,7 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
)
fake_self._paired_cache_keys = Qwen2_5OmniThinkerMultiModalProcessor._paired_cache_keys
fake_self.info.parse_mm_data = MagicMock(return_value=SimpleNamespace(name="missing"))
fake_self._apply_hf_processor_main = MagicMock(return_value=([9], {"video_grid_thw": object()}, False))
fake_self._apply_hf_processor_main = MagicMock(return_value={"video_grid_thw": object()})
fake_self._get_mm_fields_config = MagicMock(return_value={})
fake_self._get_mm_prompt_updates = MagicMock(return_value={"video": [[object()]], "audio": [[object()]]})
fake_self._merge_mm_kwargs = capture_merge
@@ -316,13 +314,12 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
"vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.MultiModalKwargsItems.from_hf_inputs",
return_value={"video": [object()], "audio": [object()]},
):
prompt_ids, mm_info, _ = Qwen2_5OmniThinkerMultiModalProcessor._cached_apply_hf_processor(
mm_info = Qwen2_5OmniThinkerMultiModalProcessor._cached_apply_hf_processor(
fake_self,
inputs,
timing_ctx,
)
assert prompt_ids == [9]
# Both modalities were reprocessed as one unit after pair-miss expansion.
assert fake_self.info.parse_mm_data.call_args.args[0] == {
"video": ["video-0"],
@@ -623,10 +620,17 @@ def test_derive_audio_from_video_placeholders_only_pairs_true_videos():
def test_qwen3_processor_inherits_vllm_omni_per_video_helpers():
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerMultiModalProcessor as UpstreamQwen3Processor,
)
assert issubclass(
Qwen3OmniMoeThinkerMultiModalProcessor,
Qwen2_5OmniThinkerMultiModalProcessor,
)
assert Qwen3OmniMoeThinkerMultiModalProcessor._get_hf_mm_data is UpstreamQwen3Processor._get_hf_mm_data
mro = Qwen3OmniMoeThinkerMultiModalProcessor.__mro__
assert mro.index(Qwen2_5OmniThinkerMultiModalProcessor) < mro.index(UpstreamQwen3Processor)
def test_qwen3_prompt_updates_prefer_hf_second_per_grid_ts_from_outputs():
@@ -113,7 +113,9 @@ def _save_peft_adapter(tmp_path: Path) -> Path:
return adapter_dir
def test_peft_expert_lora_loads_into_thinker(tmp_path: Path):
def test_peft_expert_lora_loads_into_thinker(tmp_path: Path, monkeypatch):
# This CPU harness tests weight mapping, not CUDA pinned-memory allocation.
monkeypatch.setattr("vllm.lora.lora_model.PIN_MEMORY", False)
adapter_dir = _save_peft_adapter(tmp_path)
peft_helper = PEFTHelper.from_local_dir(str(adapter_dir), max_position_embeddings=128)
lora_model = LoRAModel.from_local_checkpoint(
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
from unittest.mock import patch
@@ -157,7 +157,10 @@ def _load_weights_noop(model: Qwen3TTSCode2Wav) -> set[str]:
def __init__(self, *_: object, **__: object):
pass
def load_weights(self, _weights: object) -> set[str]:
def load_weights(self, _weights: object, *, mapper: object | None = None) -> set[str]:
# vLLM 0.29 dropped skip_prefixes from AutoWeightsLoader; the model
# now passes the stage exclusions as a WeightsMapper instead.
del mapper
return {"decoder.fake_weight"}
with (
@@ -0,0 +1,38 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from torch import nn
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _TinyModel(nn.Module):
def __init__(self) -> None:
super().__init__()
self.kept = nn.Linear(1, 1, bias=False)
self.skipped = nn.Linear(1, 1, bias=False)
self.rotary_embed = nn.Module()
self.rotary_embed.register_parameter("inv_freq", nn.Parameter(torch.zeros(1)))
def test_upstream_mapper_preserves_stage_exclusions_after_renaming() -> None:
model = _TinyModel()
model.skipped.weight.data.zero_()
loaded = AutoWeightsLoader(model).load_weights(
[
("checkpoint.kept.weight", torch.ones(1, 1)),
("checkpoint.skipped.weight", torch.full((1, 1), 2.0)),
("rotary_embed.inv_freq", torch.ones(1)),
],
mapper=WeightsMapper(orig_to_new_prefix={"checkpoint.": ""})
| WeightsMapper(orig_to_new_prefix={"skipped.": None}, orig_to_new_substr={"rotary_embed.inv_freq": None}),
)
assert loaded == {"kept.weight"}
torch.testing.assert_close(model.kept.weight, torch.ones(1, 1))
torch.testing.assert_close(model.skipped.weight, torch.zeros(1, 1))
torch.testing.assert_close(model.rotary_embed.inv_freq, torch.zeros(1))