[Rebase] Rebase to vllm 0.26.0 (#5443)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Signed-off-by: tzhouam <tzhouam@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: tzhouam <tzhouam@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-07-27 23:35:49 +08:00
1 parent fb89ab43ea
commit cbab1d4d8f
41 files changed
+334 -183

No files matched your search

+29 -24
View File
@@ -100,8 +100,9 @@ def _decode_kv_cache(
head_size: int = 128,
dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
# vLLM >=0.23.0 KV cache layout: key/value live on dim 1.
return torch.zeros((num_blocks, 2, block_size, num_kv_heads, head_size), dtype=dtype)
# vLLM >=0.23.0 KV cache layout: (num_blocks, num_kv_heads, block_size,
# 2*head_size) with K and V packed in the last dimension.
return torch.zeros((num_blocks, num_kv_heads, block_size, 2 * head_size), dtype=dtype)
def test_fish_kvcache_enabled_by_default(monkeypatch):
@@ -221,10 +222,11 @@ def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size):
def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
# vLLM >=0.23.0 lays the KV cache out as (num_blocks, 2, block_size,
# num_kv_heads, head_size). The backend must unbind on dim 1 so each of
# key/value comes back as the 4-D (num_blocks, block_size, num_kv_heads,
# head_size) tensor the decode kernel expects.
# vLLM >=0.23.0 lays out the KV cache as (num_blocks, num_kv_heads,
# block_size, 2*head_size) with K and V packed in the last dimension.
# The backend must split on the content dim so each of key/value comes
# back as the 4-D (num_blocks, block_size, num_kv_heads, head_size)
# tensor the decode kernel expects.
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None)
@@ -232,7 +234,8 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
num_blocks, block_size, num_kv_heads, head_size = 8, 16, 8, 128
cache_shape = _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size)
assert cache_shape == (num_blocks, 2, block_size, num_kv_heads, head_size)
# Upstream now returns a 4-D packed shape: (B, H, N, 2*D).
assert cache_shape == (num_blocks, num_kv_heads, block_size, 2 * head_size)
captured = {}
@@ -262,20 +265,21 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkeypatch):
# End-to-end with the real vLLM cache layout and the real (unmocked) guard.
#
# The point of the unbind(1) fix: unbind(0) raised "too many values to
# unpack" on the 5-D vLLM 0.23.0 layout, so the request crashed outright.
# unbind(1) makes the request *work* again -- it unpacks cleanly and the
# backend gracefully falls back to the original attention with correct
# output. This test guards that fix and must NOT depend on the fast path
# firing.
# The point of the transpose+split fix: unbind(1) on the old 5-D layout
# raised "too many values to unpack" on the vLLM 0.23.0 layout, and the
# upstream now packs K/V into the last dimension (4-D shape). The
# transpose(1, 2) + split correctly unpacks into separate key/value tensors
# and the backend gracefully falls back to the original attention with
# correct output. This test guards that fix and must NOT depend on the fast
# path firing.
#
# It also documents a known limitation (tracked separately): unbind(1) on
# the interleaved (num_blocks, 2, ...) layout yields *non-contiguous*
# key/value views, which the guard rejects on the is_contiguous() check.
# So the dimensions are compatible (4-D, block_size=16, head_size=128) --
# the only thing keeping the fast path from engaging is contiguity, not
# shape. Making the fast path actually fire would require a .contiguous()
# copy and is out of scope here.
# It also documents a known limitation (tracked separately):
# transpose(1, 2) on the interleaved (B, H, N, 2*D) layout yields
# *non-contiguous* key/value views, which the guard rejects on the
# is_contiguous() check. So the dimensions are compatible (4-D,
# block_size=16, head_size=128) -- the only thing keeping the fast path
# from engaging is contiguity, not shape. Making the fast path actually
# fire would require a .contiguous() copy and is out of scope here.
fish_kvcache_backend.reset_fish_kvcache_attn_stats()
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
@@ -300,13 +304,14 @@ def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkey
output = torch.zeros_like(query)
kv_cache = torch.zeros(cache_shape, dtype=torch.float16)
# Root cause record: unbind(1) gives the right 4-D shape but a non-contiguous view.
key_cache, value_cache = kv_cache.unbind(1)
# Root cause record: transpose(1,2) + split gives the right 4-D shape but
# a non-contiguous view.
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_size, dim=-1)
assert tuple(key_cache.shape) == (num_blocks, block_size, num_kv_heads, head_size)
assert not key_cache.is_contiguous()
# unbind(1) no longer crashes (unbind(0) would have raised here) and the
# request completes via the correct fallback path.
# transpose+split no longer crashes (unbind(1) would have raised here) and
# the request completes via the correct fallback path.
result = model.layers[0].self_attn.attn.impl.forward(None, query, None, None, kv_cache, _metadata(), output)
assert result is output
@@ -73,6 +73,9 @@ def _run_resumable_segment_stop(
sched._update_request_with_output.side_effect = stop_request
sched._handle_stopped_request.return_value = session_finished
# vLLM 0.26 returns (kv_xfer_params, ec_xfer_params); an unconfigured
# MagicMock iterates empty and fails to unpack at the call site.
sched._free_request.return_value = (None, None)
sched.chunk_transfer_adapter = None
sched.running = [session]
sched.waiting_for_transfer_free = set()
@@ -137,6 +140,23 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None:
assert output.is_segment_finished is False
def test_update_from_output_settles_in_flight_tokens() -> None:
"""vLLM 0.26: schedule() increments num_in_flight_tokens per scheduled
token; update_from_output must decrement it symmetrically. If the
decrement is dropped the counter grows monotonically and both readers
(allocate_slots, _connector_finished) clamp
max(0, num_computed_tokens - num_in_flight_tokens) to zero forever,
silently freezing sliding-window block freeing.
"""
session = _make_request()
session.status = RequestStatus.RUNNING
session.num_in_flight_tokens = 1 # as left by schedule() for this step
_run_resumable_segment_stop(session)
assert session.num_in_flight_tokens == 0
def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None:
"""A decode step that neither stops nor carries an inter-stage payload.
@@ -76,8 +76,9 @@ def test_finish_requests_purges_finished_request_from_running(
)
def fake_finish_requests(self, request_ids, finished_status):
# super() returning the abort tuple but leaving self.running untouched
return [(finished.request_id, 0)]
# vLLM 0.26 super() returns the finished Request objects but here
# leaves self.running untouched to simulate the missed removal.
return [finished]
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
@@ -88,7 +89,7 @@ def test_finish_requests_purges_finished_request_from_running(
)
assert scheduler.running == []
assert result == [(finished.request_id, 0)]
assert result == [finished]
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
@@ -111,7 +112,7 @@ def test_finish_requests_purges_untracked_request_from_running(
)
def fake_finish_requests(self, request_ids, finished_status):
return [(untracked.request_id, 0)]
return [untracked]
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
@@ -122,7 +123,7 @@ def test_finish_requests_purges_untracked_request_from_running(
)
assert scheduler.running == []
assert result == [(untracked.request_id, 0)]
assert result == [untracked]
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
@@ -136,7 +137,7 @@ def test_finish_requests_leaves_healthy_running_intact(
is conservative and won't aggressively drop healthy entries.
"""
alive = _StubRequest("req-alive", RequestStatus.RUNNING)
leaving_id = "req-leaving"
leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED)
scheduler = _make_scheduler(
scheduler_cls,
requests={alive.request_id: alive},
@@ -145,13 +146,13 @@ def test_finish_requests_leaves_healthy_running_intact(
)
def fake_finish_requests(self, request_ids, finished_status):
return [(leaving_id, 0)]
return [leaving]
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
scheduler_cls.finish_requests(
scheduler,
[leaving_id],
[leaving.request_id],
RequestStatus.FINISHED_STOPPED,
)
@@ -53,7 +53,11 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
def fake_finish_requests(self, request_ids, finished_status):
assert request_ids == ["req-a", "req-b"]
assert finished_status == RequestStatus.FINISHED_STOPPED
return [("req-a", 0), ("req-b", 1)]
# vLLM 0.26 returns the finished Request objects, not (req_id, client_index) tuples.
return [
SimpleNamespace(request_id="req-a", client_index=0),
SimpleNamespace(request_id="req-b", client_index=1),
]
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
@@ -63,7 +67,7 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
RequestStatus.FINISHED_STOPPED,
)
assert result == [("req-a", 0), ("req-b", 1)]
assert [(r.request_id, r.client_index) for r in result] == [("req-a", 0), ("req-b", 1)]
assert coordinator.freed == ["req-a", "req-b"]
@@ -82,7 +86,7 @@ def test_ar_free_request_cleans_input_coordinator_on_normal_free() -> None:
result = OmniARScheduler._free_request(scheduler, DummyRequest())
assert result is None
assert result == (None, None)
assert coordinator.freed == ["req-free"]
assert scheduler._omits_kv_transfer_cache == {}
assert scheduler._new_prompt_len_snapshot == {}
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import os
from types import SimpleNamespace
@@ -338,27 +339,43 @@ def test_transformer_load_weights_round_trip():
assert torch.allclose(reloaded[native_name], expected)
def test_transformer_load_weights_warns_for_unexpected_and_unloaded(caplog):
def test_transformer_load_weights_warns_for_unexpected_and_unloaded():
from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import (
BooguImageTransformer2DModel,
)
model = BooguImageTransformer2DModel(od_config=_tiny_od_config())
native_params = dict(model.named_parameters())
loaded_name = next(iter(native_params))
checkpoint_name = _native_to_checkpoint_name(loaded_name)
# vllm_omni's logger hierarchy hangs off vLLM's `vllm` logger, which sets
# propagate=False, so records never reach the root logger that pytest's
# caplog listens on. Capture at the emitting logger instead.
messages: list[str] = []
loaded = model.load_weights(
[
(checkpoint_name, torch.randn_like(native_params[loaded_name])),
("unexpected.weight", torch.ones(1)),
]
)
class _Capture(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
messages.append(record.getMessage())
log = logging.getLogger("vllm_omni.diffusion.models.boogu_image.boogu_image_transformer")
handler = _Capture(level=logging.WARNING)
log.addHandler(handler)
try:
model = BooguImageTransformer2DModel(od_config=_tiny_od_config())
native_params = dict(model.named_parameters())
loaded_name = next(iter(native_params))
checkpoint_name = _native_to_checkpoint_name(loaded_name)
loaded = model.load_weights(
[
(checkpoint_name, torch.randn_like(native_params[loaded_name])),
("unexpected.weight", torch.ones(1)),
]
)
finally:
log.removeHandler(handler)
text = "\n".join(messages)
assert loaded == {loaded_name}
assert "Skipping unexpected checkpoint weight unexpected.weight" in caplog.text
assert "Model parameters not loaded from checkpoint" in caplog.text
assert next(name for name in native_params if name != loaded_name) in caplog.text
assert "Skipping unexpected checkpoint weight unexpected.weight" in text
assert "Model parameters not loaded from checkpoint" in text
assert next(name for name in native_params if name != loaded_name) in text
def test_transformer_forward_t2i_shape():
@@ -780,6 +780,10 @@ def test_restore_queues_skips_requests_missing_from_scheduler_requests(build_ada
class _HashableRequest(SimpleNamespace):
"""SimpleNamespace that can be added to a set (needed by scheduler internals)."""
# vLLM 0.26: update_from_output settles this counter for every scheduled
# request; real Requests initialise it to 0 (vllm/v1/request.py).
num_in_flight_tokens = 0
def __hash__(self):
return hash(self.request_id)
@@ -833,7 +837,7 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc
scheduler.requests = {"req-s1": request}
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
scheduler._free_request = mocker.MagicMock(return_value=None)
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
scheduler.running = [request]
scheduler.waiting = mocker.MagicMock()
@@ -918,7 +922,7 @@ def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerF
scheduler._update_request_with_output = mocker.MagicMock(return_value=([], True))
scheduler._process_kv_transfer_trigger = mocker.MagicMock(return_value=False)
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
scheduler._free_request = mocker.MagicMock(return_value=None)
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
scheduler.running = [request]
scheduler.waiting = mocker.MagicMock()
@@ -1047,7 +1051,7 @@ def _build_deferred_finish_scheduler(mocker, *, running, pending_finish_reqs):
scheduler._pending_finish_reqs = list(pending_finish_reqs)
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
scheduler._free_request = mocker.MagicMock(return_value=None)
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
scheduler.running = list(running)
scheduler.waiting = mocker.MagicMock()
@@ -1103,7 +1107,7 @@ def test_deferred_finish_emits_finished_output(mocker: MockerFixture):
running=[request],
pending_finish_reqs=[request],
)
scheduler._free_request.return_value = {"mock": "kv_params"}
scheduler._free_request.return_value = ({"mock": "kv_params"}, None)
result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
@@ -330,6 +330,31 @@ def test_normalize_layer_kv_rejects_invalid_inputs(kv_config, common_constants,
assert normalized is None
def test_normalize_layer_kv_unpacks_packed_layout(common_constants):
"""The vLLM >= 0.23 packed layout (num_blocks, n_heads, block_size,
2*head_dim) is unpacked into (num_blocks, block_size, n_heads, head_dim)
key/value blocks when block_size is provided."""
block_size = common_constants["block_size"]
num_heads = common_constants["num_heads"]
head_dim = common_constants["head_dim"]
req_id = common_constants["req_id"]
num_blocks = 10
layer_kv = torch.randn(num_blocks, num_heads, block_size, 2 * head_dim)
# Without block_size the 4-D packed layout is not recognized.
assert normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0) is None
normalized = normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0, block_size=block_size)
assert normalized is not None
key_blocks, value_blocks = normalized
assert key_blocks.shape == (num_blocks, block_size, num_heads, head_dim)
assert value_blocks.shape == (num_blocks, block_size, num_heads, head_dim)
reference = layer_kv.transpose(1, 2)
assert torch.equal(key_blocks, reference[..., :head_dim])
assert torch.equal(value_blocks, reference[..., head_dim:])
def test_manager_reception(kv_config, mock_connector, common_constants):
"""Test reception and injection logic in OmniKVTransferManager."""
num_layers = common_constants["num_layers"]
@@ -201,6 +201,7 @@ def _assert_realtime_smoke(result: dict) -> None:
def _assert_realtime_accuracy(
result: dict,
whisper_model_size: str = "large-v3",
threshold: float = 0.8,
) -> None:
"""Assert that whisper transcription of audio output matches model text.
@@ -214,6 +215,12 @@ def _assert_realtime_accuracy(
variability even though audio generation was correct. large-v3
transcribes these clips reliably, so a failure here now points
at the model, not the ASR grader.
threshold: Minimum cosine similarity (with length penalty) required to
pass. Default 0.8. Do not lower per-callsite without data:
at 0.35 the assertion no longer detects real audio
regressions. If a variant genuinely needs a different gate
(e.g. whisper partial transcripts under async_chunk), propose
it in its own PR with measurements.
"""
final_text = (result["transcription_text"] or "").strip()
assert final_text, "Expected non-empty transcription (model text stream)"
@@ -223,8 +230,9 @@ def _assert_realtime_accuracy(
assert whisper_text, "Whisper returned empty string for synthesized output audio"
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
assert sim > 0.8, (
f"Output audio transcript should match model text (sim={sim:.3f}): "
assert sim > threshold, (
f"Output audio transcript should match model text (sim={sim:.3f}, "
f"threshold={threshold}): "
f"whisper={whisper_text!r}, model_text={final_text!r}"
)
+5 -22
View File
@@ -305,21 +305,11 @@ def mock_get_config(monkeypatch):
"""Auto-mock get_config and related model loading functions."""
import sys
fake_tokenizer = _ns()
fake_tokenizer.encode = lambda *args, **kwargs: [1, 2, 3]
fake_tokenizer.decode = lambda *args, **kwargs: "test"
def _mock_init_tokenizer_from_configs(model_config=None, **kwargs):
return fake_tokenizer
monkeypatch.setattr(
"vllm.transformers_utils.tokenizer.init_tokenizer_from_configs",
_mock_init_tokenizer_from_configs,
raising=False,
)
tokenizer_module_path = "vllm.transformers_utils.tokenizer"
if tokenizer_module_path in sys.modules:
setattr(sys.modules[tokenizer_module_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
# TODO(pd-unskip): the tokenizer mock that lived here targeted
# vllm.transformers_utils.tokenizer.init_tokenizer_from_configs, which was
# deleted in vLLM 0.26 (the raising=False setattr made it a silent no-op).
# When the module-level skip is lifted, mock whatever tokenizer seam the
# revived PD entrypoint code actually uses.
def _mock_length_from_prompt_token_ids_or_embeds(prompt_token_ids=None, prompt_embeds=None):
if prompt_token_ids is not None:
@@ -344,13 +334,6 @@ def mock_get_config(monkeypatch):
_mock_length_from_prompt_token_ids_or_embeds,
)
monkeypatch.setattr(
"vllm_omni.entrypoints.async_omni.init_tokenizer_from_configs", _mock_init_tokenizer_from_configs, raising=False
)
async_omni_path = "vllm_omni.entrypoints.async_omni"
if async_omni_path in sys.modules:
setattr(sys.modules[async_omni_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
fake_hf_config = _ns()
fake_hf_config.model_type = "qwen2_5_omni"
@@ -15,6 +15,7 @@ import functools
import pytest
import torch
from pytest_mock import MockerFixture
from vllm.multimodal.utils import set_mm_embedding_modality
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -117,13 +118,30 @@ class TestCheckInterleavedAudioVideo:
# ---------------------------------------------------------------------------
def make_mock_model(mocker: MockerFixture, hidden: int = 8):
@functools.lru_cache(maxsize=1)
def _omni_qwen2_5_thinker_cls():
"""vllm_omni's thinker override, loaded lazily like the upstream one."""
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
Qwen2_5OmniThinkerForConditionalGeneration,
)
return Qwen2_5OmniThinkerForConditionalGeneration
def make_mock_model(mocker: MockerFixture, hidden: int = 8, thinker_cls=None):
"""
Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
that has enough structure to run embed_input_ids.
``thinker_cls`` selects whose ``embed_input_ids`` the mock dispatches to.
It defaults to upstream's. Only the interleaved case passes omni's override:
omni's non-interleaved path falls through to a zero-arg ``super()`` call,
which raises TypeError when ``self`` is a Mock rather than an instance.
"""
m = _qwen2_5_omni_thinker_mod()
Qwen2_5OmniThinkerForConditionalGeneration = m.Qwen2_5OmniThinkerForConditionalGeneration
if thinker_cls is None:
thinker_cls = Qwen2_5OmniThinkerForConditionalGeneration
model = mocker.Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
@@ -155,9 +173,7 @@ def make_mock_model(mocker: MockerFixture, hidden: int = 8):
is_multimodal=is_multimodal,
)
model.embed_input_ids = lambda *a, **kw: m.Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids( # noqa: E501
model, *a, **kw
)
model.embed_input_ids = lambda *a, **kw: thinker_cls.embed_input_ids(model, *a, **kw)
model._super_embed_input_ids = fake_super_embed
@@ -171,11 +187,11 @@ def build_mm_embeds(audio_n, image_n, video_n, hidden, audio_val=10.0, image_val
"""
embs = []
if audio_n:
embs.append(torch.full((audio_n, hidden), audio_val))
embs.append(set_mm_embedding_modality(torch.full((audio_n, hidden), audio_val), "audio"))
if image_n:
embs.append(torch.full((image_n, hidden), image_val))
embs.append(set_mm_embedding_modality(torch.full((image_n, hidden), image_val), "image"))
if video_n:
embs.append(torch.full((video_n, hidden), video_val))
embs.append(set_mm_embedding_modality(torch.full((video_n, hidden), video_val), "video"))
return embs
@@ -259,6 +275,11 @@ class TestEmbedInputIds:
"""
Interleaved (use_audio_in_video): video chunks interleaved with audio.
Video embeddings must go to video positions, audio to audio positions.
Runs against **omni's** thinker override, not upstream's: this is the
only unit-level coverage of omni's call into
``merge_interleaved_embeddings``, whose upstream signature dropped
``num_video``/``num_audio`` in vLLM #46213.
"""
hidden = 8
audio_val, video_val = 10.0, 30.0
@@ -270,11 +291,11 @@ class TestEmbedInputIds:
audio_n = sum(audio_chunks) # 6
mm_embeds = [
torch.full((video_n, hidden), video_val),
torch.full((audio_n, hidden), audio_val),
set_mm_embedding_modality(torch.full((video_n, hidden), video_val), "video"),
set_mm_embedding_modality(torch.full((audio_n, hidden), audio_val), "audio"),
]
model, _ = make_mock_model(mocker, hidden)
model, _ = make_mock_model(mocker, hidden, thinker_cls=_omni_qwen2_5_thinker_cls())
result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
@@ -308,8 +329,8 @@ class TestMergeInterleavedEmbeddings:
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
torch.full((num_video, hidden), 30.0),
torch.full((num_audio, hidden), 10.0),
set_mm_embedding_modality(torch.full((num_video, hidden), 30.0), "video"),
set_mm_embedding_modality(torch.full((num_audio, hidden), 10.0), "audio"),
]
result = m.merge_interleaved_embeddings(
@@ -318,8 +339,6 @@ class TestMergeInterleavedEmbeddings:
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
video_pos = is_video.nonzero(as_tuple=True)[0]
@@ -84,6 +84,7 @@ class _CpuFusedMoE3DWithLoRA(FusedMoE3DWithLoRA):
self.tp_size = 1
self.tp_rank = 0
self._w13_slices = 1
self.enable_moe_shared_loras = False
def _save_peft_adapter(tmp_path: Path) -> Path:
+3
View File
@@ -69,6 +69,9 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
# vLLM 0.26: one entry per KV cache group; max_model_len=8 with
# block_sizes=[1] means one group of 8 blocks.
max_num_blocks_per_req=[8],
)
input_batch.add_request(
CachedRequestState(
@@ -4,7 +4,7 @@ from types import SimpleNamespace
import pytest
import torch
from vllm.config import CacheConfig
from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
import vllm_omni.worker.gpu_generation_model_runner as gen_runner_module
from vllm_omni.worker.gpu_generation_model_runner import (
+1 -1
View File
@@ -1245,7 +1245,7 @@ class TestRankAwareKVRouting(unittest.TestCase):
class TestAttachOmniConnectorOutput(unittest.TestCase):
def test_wraps_empty_model_runner_output_when_signals_exist(self):
from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
host = MixinHost()
host.get_omni_connector_output = lambda: OmniConnectorOutput(chunk_ready_req_ids={"req-1"})