[Rebase] Rebase to vllm 0.26.0 (#5443)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Signed-off-by: tzhouam <tzhouam@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: tzhouam <tzhouam@gmail.com>
This commit is contained in:
41 files changed
+334
-183
No files matched your search
@@ -100,8 +100,9 @@ def _decode_kv_cache(
|
||||
head_size: int = 128,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
# vLLM >=0.23.0 KV cache layout: key/value live on dim 1.
|
||||
return torch.zeros((num_blocks, 2, block_size, num_kv_heads, head_size), dtype=dtype)
|
||||
# vLLM >=0.23.0 KV cache layout: (num_blocks, num_kv_heads, block_size,
|
||||
# 2*head_size) with K and V packed in the last dimension.
|
||||
return torch.zeros((num_blocks, num_kv_heads, block_size, 2 * head_size), dtype=dtype)
|
||||
|
||||
|
||||
def test_fish_kvcache_enabled_by_default(monkeypatch):
|
||||
@@ -221,10 +222,11 @@ def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size):
|
||||
|
||||
|
||||
def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
# vLLM >=0.23.0 lays the KV cache out as (num_blocks, 2, block_size,
|
||||
# num_kv_heads, head_size). The backend must unbind on dim 1 so each of
|
||||
# key/value comes back as the 4-D (num_blocks, block_size, num_kv_heads,
|
||||
# head_size) tensor the decode kernel expects.
|
||||
# vLLM >=0.23.0 lays out the KV cache as (num_blocks, num_kv_heads,
|
||||
# block_size, 2*head_size) with K and V packed in the last dimension.
|
||||
# The backend must split on the content dim so each of key/value comes
|
||||
# back as the 4-D (num_blocks, block_size, num_kv_heads, head_size)
|
||||
# tensor the decode kernel expects.
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
|
||||
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
|
||||
monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None)
|
||||
@@ -232,7 +234,8 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
|
||||
num_blocks, block_size, num_kv_heads, head_size = 8, 16, 8, 128
|
||||
cache_shape = _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size)
|
||||
assert cache_shape == (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# Upstream now returns a 4-D packed shape: (B, H, N, 2*D).
|
||||
assert cache_shape == (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -262,20 +265,21 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkeypatch):
|
||||
# End-to-end with the real vLLM cache layout and the real (unmocked) guard.
|
||||
#
|
||||
# The point of the unbind(1) fix: unbind(0) raised "too many values to
|
||||
# unpack" on the 5-D vLLM 0.23.0 layout, so the request crashed outright.
|
||||
# unbind(1) makes the request *work* again -- it unpacks cleanly and the
|
||||
# backend gracefully falls back to the original attention with correct
|
||||
# output. This test guards that fix and must NOT depend on the fast path
|
||||
# firing.
|
||||
# The point of the transpose+split fix: unbind(1) on the old 5-D layout
|
||||
# raised "too many values to unpack" on the vLLM 0.23.0 layout, and the
|
||||
# upstream now packs K/V into the last dimension (4-D shape). The
|
||||
# transpose(1, 2) + split correctly unpacks into separate key/value tensors
|
||||
# and the backend gracefully falls back to the original attention with
|
||||
# correct output. This test guards that fix and must NOT depend on the fast
|
||||
# path firing.
|
||||
#
|
||||
# It also documents a known limitation (tracked separately): unbind(1) on
|
||||
# the interleaved (num_blocks, 2, ...) layout yields *non-contiguous*
|
||||
# key/value views, which the guard rejects on the is_contiguous() check.
|
||||
# So the dimensions are compatible (4-D, block_size=16, head_size=128) --
|
||||
# the only thing keeping the fast path from engaging is contiguity, not
|
||||
# shape. Making the fast path actually fire would require a .contiguous()
|
||||
# copy and is out of scope here.
|
||||
# It also documents a known limitation (tracked separately):
|
||||
# transpose(1, 2) on the interleaved (B, H, N, 2*D) layout yields
|
||||
# *non-contiguous* key/value views, which the guard rejects on the
|
||||
# is_contiguous() check. So the dimensions are compatible (4-D,
|
||||
# block_size=16, head_size=128) -- the only thing keeping the fast path
|
||||
# from engaging is contiguity, not shape. Making the fast path actually
|
||||
# fire would require a .contiguous() copy and is out of scope here.
|
||||
fish_kvcache_backend.reset_fish_kvcache_attn_stats()
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
|
||||
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
|
||||
@@ -300,13 +304,14 @@ def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkey
|
||||
output = torch.zeros_like(query)
|
||||
kv_cache = torch.zeros(cache_shape, dtype=torch.float16)
|
||||
|
||||
# Root cause record: unbind(1) gives the right 4-D shape but a non-contiguous view.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# Root cause record: transpose(1,2) + split gives the right 4-D shape but
|
||||
# a non-contiguous view.
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_size, dim=-1)
|
||||
assert tuple(key_cache.shape) == (num_blocks, block_size, num_kv_heads, head_size)
|
||||
assert not key_cache.is_contiguous()
|
||||
|
||||
# unbind(1) no longer crashes (unbind(0) would have raised here) and the
|
||||
# request completes via the correct fallback path.
|
||||
# transpose+split no longer crashes (unbind(1) would have raised here) and
|
||||
# the request completes via the correct fallback path.
|
||||
result = model.layers[0].self_attn.attn.impl.forward(None, query, None, None, kv_cache, _metadata(), output)
|
||||
|
||||
assert result is output
|
||||
|
||||
@@ -73,6 +73,9 @@ def _run_resumable_segment_stop(
|
||||
|
||||
sched._update_request_with_output.side_effect = stop_request
|
||||
sched._handle_stopped_request.return_value = session_finished
|
||||
# vLLM 0.26 returns (kv_xfer_params, ec_xfer_params); an unconfigured
|
||||
# MagicMock iterates empty and fails to unpack at the call site.
|
||||
sched._free_request.return_value = (None, None)
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.running = [session]
|
||||
sched.waiting_for_transfer_free = set()
|
||||
@@ -137,6 +140,23 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None:
|
||||
assert output.is_segment_finished is False
|
||||
|
||||
|
||||
def test_update_from_output_settles_in_flight_tokens() -> None:
|
||||
"""vLLM 0.26: schedule() increments num_in_flight_tokens per scheduled
|
||||
token; update_from_output must decrement it symmetrically. If the
|
||||
decrement is dropped the counter grows monotonically and both readers
|
||||
(allocate_slots, _connector_finished) clamp
|
||||
max(0, num_computed_tokens - num_in_flight_tokens) to zero forever,
|
||||
silently freezing sliding-window block freeing.
|
||||
"""
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.RUNNING
|
||||
session.num_in_flight_tokens = 1 # as left by schedule() for this step
|
||||
|
||||
_run_resumable_segment_stop(session)
|
||||
|
||||
assert session.num_in_flight_tokens == 0
|
||||
|
||||
|
||||
def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None:
|
||||
"""A decode step that neither stops nor carries an inter-stage payload.
|
||||
|
||||
|
||||
@@ -76,8 +76,9 @@ def test_finish_requests_purges_finished_request_from_running(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
# super() returning the abort tuple but leaving self.running untouched
|
||||
return [(finished.request_id, 0)]
|
||||
# vLLM 0.26 super() returns the finished Request objects but here
|
||||
# leaves self.running untouched to simulate the missed removal.
|
||||
return [finished]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -88,7 +89,7 @@ def test_finish_requests_purges_finished_request_from_running(
|
||||
)
|
||||
|
||||
assert scheduler.running == []
|
||||
assert result == [(finished.request_id, 0)]
|
||||
assert result == [finished]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
|
||||
@@ -111,7 +112,7 @@ def test_finish_requests_purges_untracked_request_from_running(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
return [(untracked.request_id, 0)]
|
||||
return [untracked]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -122,7 +123,7 @@ def test_finish_requests_purges_untracked_request_from_running(
|
||||
)
|
||||
|
||||
assert scheduler.running == []
|
||||
assert result == [(untracked.request_id, 0)]
|
||||
assert result == [untracked]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
|
||||
@@ -136,7 +137,7 @@ def test_finish_requests_leaves_healthy_running_intact(
|
||||
is conservative and won't aggressively drop healthy entries.
|
||||
"""
|
||||
alive = _StubRequest("req-alive", RequestStatus.RUNNING)
|
||||
leaving_id = "req-leaving"
|
||||
leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED)
|
||||
scheduler = _make_scheduler(
|
||||
scheduler_cls,
|
||||
requests={alive.request_id: alive},
|
||||
@@ -145,13 +146,13 @@ def test_finish_requests_leaves_healthy_running_intact(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
return [(leaving_id, 0)]
|
||||
return [leaving]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
scheduler_cls.finish_requests(
|
||||
scheduler,
|
||||
[leaving_id],
|
||||
[leaving.request_id],
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
|
||||
|
||||
@@ -53,7 +53,11 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
assert request_ids == ["req-a", "req-b"]
|
||||
assert finished_status == RequestStatus.FINISHED_STOPPED
|
||||
return [("req-a", 0), ("req-b", 1)]
|
||||
# vLLM 0.26 returns the finished Request objects, not (req_id, client_index) tuples.
|
||||
return [
|
||||
SimpleNamespace(request_id="req-a", client_index=0),
|
||||
SimpleNamespace(request_id="req-b", client_index=1),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -63,7 +67,7 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
|
||||
assert result == [("req-a", 0), ("req-b", 1)]
|
||||
assert [(r.request_id, r.client_index) for r in result] == [("req-a", 0), ("req-b", 1)]
|
||||
assert coordinator.freed == ["req-a", "req-b"]
|
||||
|
||||
|
||||
@@ -82,7 +86,7 @@ def test_ar_free_request_cleans_input_coordinator_on_normal_free() -> None:
|
||||
|
||||
result = OmniARScheduler._free_request(scheduler, DummyRequest())
|
||||
|
||||
assert result is None
|
||||
assert result == (None, None)
|
||||
assert coordinator.freed == ["req-free"]
|
||||
assert scheduler._omits_kv_transfer_cache == {}
|
||||
assert scheduler._new_prompt_len_snapshot == {}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -338,27 +339,43 @@ def test_transformer_load_weights_round_trip():
|
||||
assert torch.allclose(reloaded[native_name], expected)
|
||||
|
||||
|
||||
def test_transformer_load_weights_warns_for_unexpected_and_unloaded(caplog):
|
||||
def test_transformer_load_weights_warns_for_unexpected_and_unloaded():
|
||||
from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import (
|
||||
BooguImageTransformer2DModel,
|
||||
)
|
||||
|
||||
model = BooguImageTransformer2DModel(od_config=_tiny_od_config())
|
||||
native_params = dict(model.named_parameters())
|
||||
loaded_name = next(iter(native_params))
|
||||
checkpoint_name = _native_to_checkpoint_name(loaded_name)
|
||||
# vllm_omni's logger hierarchy hangs off vLLM's `vllm` logger, which sets
|
||||
# propagate=False, so records never reach the root logger that pytest's
|
||||
# caplog listens on. Capture at the emitting logger instead.
|
||||
messages: list[str] = []
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
(checkpoint_name, torch.randn_like(native_params[loaded_name])),
|
||||
("unexpected.weight", torch.ones(1)),
|
||||
]
|
||||
)
|
||||
class _Capture(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
messages.append(record.getMessage())
|
||||
|
||||
log = logging.getLogger("vllm_omni.diffusion.models.boogu_image.boogu_image_transformer")
|
||||
handler = _Capture(level=logging.WARNING)
|
||||
log.addHandler(handler)
|
||||
try:
|
||||
model = BooguImageTransformer2DModel(od_config=_tiny_od_config())
|
||||
native_params = dict(model.named_parameters())
|
||||
loaded_name = next(iter(native_params))
|
||||
checkpoint_name = _native_to_checkpoint_name(loaded_name)
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
(checkpoint_name, torch.randn_like(native_params[loaded_name])),
|
||||
("unexpected.weight", torch.ones(1)),
|
||||
]
|
||||
)
|
||||
finally:
|
||||
log.removeHandler(handler)
|
||||
|
||||
text = "\n".join(messages)
|
||||
assert loaded == {loaded_name}
|
||||
assert "Skipping unexpected checkpoint weight unexpected.weight" in caplog.text
|
||||
assert "Model parameters not loaded from checkpoint" in caplog.text
|
||||
assert next(name for name in native_params if name != loaded_name) in caplog.text
|
||||
assert "Skipping unexpected checkpoint weight unexpected.weight" in text
|
||||
assert "Model parameters not loaded from checkpoint" in text
|
||||
assert next(name for name in native_params if name != loaded_name) in text
|
||||
|
||||
|
||||
def test_transformer_forward_t2i_shape():
|
||||
|
||||
@@ -780,6 +780,10 @@ def test_restore_queues_skips_requests_missing_from_scheduler_requests(build_ada
|
||||
class _HashableRequest(SimpleNamespace):
|
||||
"""SimpleNamespace that can be added to a set (needed by scheduler internals)."""
|
||||
|
||||
# vLLM 0.26: update_from_output settles this counter for every scheduled
|
||||
# request; real Requests initialise it to 0 (vllm/v1/request.py).
|
||||
num_in_flight_tokens = 0
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.request_id)
|
||||
|
||||
@@ -833,7 +837,7 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc
|
||||
scheduler.requests = {"req-s1": request}
|
||||
|
||||
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=None)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
|
||||
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
|
||||
scheduler.running = [request]
|
||||
scheduler.waiting = mocker.MagicMock()
|
||||
@@ -918,7 +922,7 @@ def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerF
|
||||
scheduler._update_request_with_output = mocker.MagicMock(return_value=([], True))
|
||||
scheduler._process_kv_transfer_trigger = mocker.MagicMock(return_value=False)
|
||||
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=None)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
|
||||
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
|
||||
scheduler.running = [request]
|
||||
scheduler.waiting = mocker.MagicMock()
|
||||
@@ -1047,7 +1051,7 @@ def _build_deferred_finish_scheduler(mocker, *, running, pending_finish_reqs):
|
||||
scheduler._pending_finish_reqs = list(pending_finish_reqs)
|
||||
|
||||
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=None)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
|
||||
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
|
||||
scheduler.running = list(running)
|
||||
scheduler.waiting = mocker.MagicMock()
|
||||
@@ -1103,7 +1107,7 @@ def test_deferred_finish_emits_finished_output(mocker: MockerFixture):
|
||||
running=[request],
|
||||
pending_finish_reqs=[request],
|
||||
)
|
||||
scheduler._free_request.return_value = {"mock": "kv_params"}
|
||||
scheduler._free_request.return_value = ({"mock": "kv_params"}, None)
|
||||
|
||||
result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
|
||||
|
||||
|
||||
@@ -330,6 +330,31 @@ def test_normalize_layer_kv_rejects_invalid_inputs(kv_config, common_constants,
|
||||
assert normalized is None
|
||||
|
||||
|
||||
def test_normalize_layer_kv_unpacks_packed_layout(common_constants):
|
||||
"""The vLLM >= 0.23 packed layout (num_blocks, n_heads, block_size,
|
||||
2*head_dim) is unpacked into (num_blocks, block_size, n_heads, head_dim)
|
||||
key/value blocks when block_size is provided."""
|
||||
block_size = common_constants["block_size"]
|
||||
num_heads = common_constants["num_heads"]
|
||||
head_dim = common_constants["head_dim"]
|
||||
req_id = common_constants["req_id"]
|
||||
num_blocks = 10
|
||||
|
||||
layer_kv = torch.randn(num_blocks, num_heads, block_size, 2 * head_dim)
|
||||
|
||||
# Without block_size the 4-D packed layout is not recognized.
|
||||
assert normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0) is None
|
||||
|
||||
normalized = normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0, block_size=block_size)
|
||||
assert normalized is not None
|
||||
key_blocks, value_blocks = normalized
|
||||
assert key_blocks.shape == (num_blocks, block_size, num_heads, head_dim)
|
||||
assert value_blocks.shape == (num_blocks, block_size, num_heads, head_dim)
|
||||
reference = layer_kv.transpose(1, 2)
|
||||
assert torch.equal(key_blocks, reference[..., :head_dim])
|
||||
assert torch.equal(value_blocks, reference[..., head_dim:])
|
||||
|
||||
|
||||
def test_manager_reception(kv_config, mock_connector, common_constants):
|
||||
"""Test reception and injection logic in OmniKVTransferManager."""
|
||||
num_layers = common_constants["num_layers"]
|
||||
|
||||
@@ -201,6 +201,7 @@ def _assert_realtime_smoke(result: dict) -> None:
|
||||
def _assert_realtime_accuracy(
|
||||
result: dict,
|
||||
whisper_model_size: str = "large-v3",
|
||||
threshold: float = 0.8,
|
||||
) -> None:
|
||||
"""Assert that whisper transcription of audio output matches model text.
|
||||
|
||||
@@ -214,6 +215,12 @@ def _assert_realtime_accuracy(
|
||||
variability even though audio generation was correct. large-v3
|
||||
transcribes these clips reliably, so a failure here now points
|
||||
at the model, not the ASR grader.
|
||||
threshold: Minimum cosine similarity (with length penalty) required to
|
||||
pass. Default 0.8. Do not lower per-callsite without data:
|
||||
at 0.35 the assertion no longer detects real audio
|
||||
regressions. If a variant genuinely needs a different gate
|
||||
(e.g. whisper partial transcripts under async_chunk), propose
|
||||
it in its own PR with measurements.
|
||||
"""
|
||||
final_text = (result["transcription_text"] or "").strip()
|
||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||
@@ -223,8 +230,9 @@ def _assert_realtime_accuracy(
|
||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||
|
||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||
assert sim > 0.8, (
|
||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||
assert sim > threshold, (
|
||||
f"Output audio transcript should match model text (sim={sim:.3f}, "
|
||||
f"threshold={threshold}): "
|
||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||
)
|
||||
|
||||
|
||||
@@ -305,21 +305,11 @@ def mock_get_config(monkeypatch):
|
||||
"""Auto-mock get_config and related model loading functions."""
|
||||
import sys
|
||||
|
||||
fake_tokenizer = _ns()
|
||||
fake_tokenizer.encode = lambda *args, **kwargs: [1, 2, 3]
|
||||
fake_tokenizer.decode = lambda *args, **kwargs: "test"
|
||||
|
||||
def _mock_init_tokenizer_from_configs(model_config=None, **kwargs):
|
||||
return fake_tokenizer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.tokenizer.init_tokenizer_from_configs",
|
||||
_mock_init_tokenizer_from_configs,
|
||||
raising=False,
|
||||
)
|
||||
tokenizer_module_path = "vllm.transformers_utils.tokenizer"
|
||||
if tokenizer_module_path in sys.modules:
|
||||
setattr(sys.modules[tokenizer_module_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
|
||||
# TODO(pd-unskip): the tokenizer mock that lived here targeted
|
||||
# vllm.transformers_utils.tokenizer.init_tokenizer_from_configs, which was
|
||||
# deleted in vLLM 0.26 (the raising=False setattr made it a silent no-op).
|
||||
# When the module-level skip is lifted, mock whatever tokenizer seam the
|
||||
# revived PD entrypoint code actually uses.
|
||||
|
||||
def _mock_length_from_prompt_token_ids_or_embeds(prompt_token_ids=None, prompt_embeds=None):
|
||||
if prompt_token_ids is not None:
|
||||
@@ -344,13 +334,6 @@ def mock_get_config(monkeypatch):
|
||||
_mock_length_from_prompt_token_ids_or_embeds,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.async_omni.init_tokenizer_from_configs", _mock_init_tokenizer_from_configs, raising=False
|
||||
)
|
||||
async_omni_path = "vllm_omni.entrypoints.async_omni"
|
||||
if async_omni_path in sys.modules:
|
||||
setattr(sys.modules[async_omni_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
|
||||
|
||||
fake_hf_config = _ns()
|
||||
fake_hf_config.model_type = "qwen2_5_omni"
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import functools
|
||||
import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.multimodal.utils import set_mm_embedding_modality
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -117,13 +118,30 @@ class TestCheckInterleavedAudioVideo:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mock_model(mocker: MockerFixture, hidden: int = 8):
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _omni_qwen2_5_thinker_cls():
|
||||
"""vllm_omni's thinker override, loaded lazily like the upstream one."""
|
||||
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
return Qwen2_5OmniThinkerForConditionalGeneration
|
||||
|
||||
|
||||
def make_mock_model(mocker: MockerFixture, hidden: int = 8, thinker_cls=None):
|
||||
"""
|
||||
Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
|
||||
that has enough structure to run embed_input_ids.
|
||||
|
||||
``thinker_cls`` selects whose ``embed_input_ids`` the mock dispatches to.
|
||||
It defaults to upstream's. Only the interleaved case passes omni's override:
|
||||
omni's non-interleaved path falls through to a zero-arg ``super()`` call,
|
||||
which raises TypeError when ``self`` is a Mock rather than an instance.
|
||||
"""
|
||||
m = _qwen2_5_omni_thinker_mod()
|
||||
Qwen2_5OmniThinkerForConditionalGeneration = m.Qwen2_5OmniThinkerForConditionalGeneration
|
||||
if thinker_cls is None:
|
||||
thinker_cls = Qwen2_5OmniThinkerForConditionalGeneration
|
||||
|
||||
model = mocker.Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
|
||||
|
||||
@@ -155,9 +173,7 @@ def make_mock_model(mocker: MockerFixture, hidden: int = 8):
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
model.embed_input_ids = lambda *a, **kw: m.Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids( # noqa: E501
|
||||
model, *a, **kw
|
||||
)
|
||||
model.embed_input_ids = lambda *a, **kw: thinker_cls.embed_input_ids(model, *a, **kw)
|
||||
|
||||
model._super_embed_input_ids = fake_super_embed
|
||||
|
||||
@@ -171,11 +187,11 @@ def build_mm_embeds(audio_n, image_n, video_n, hidden, audio_val=10.0, image_val
|
||||
"""
|
||||
embs = []
|
||||
if audio_n:
|
||||
embs.append(torch.full((audio_n, hidden), audio_val))
|
||||
embs.append(set_mm_embedding_modality(torch.full((audio_n, hidden), audio_val), "audio"))
|
||||
if image_n:
|
||||
embs.append(torch.full((image_n, hidden), image_val))
|
||||
embs.append(set_mm_embedding_modality(torch.full((image_n, hidden), image_val), "image"))
|
||||
if video_n:
|
||||
embs.append(torch.full((video_n, hidden), video_val))
|
||||
embs.append(set_mm_embedding_modality(torch.full((video_n, hidden), video_val), "video"))
|
||||
return embs
|
||||
|
||||
|
||||
@@ -259,6 +275,11 @@ class TestEmbedInputIds:
|
||||
"""
|
||||
Interleaved (use_audio_in_video): video chunks interleaved with audio.
|
||||
Video embeddings must go to video positions, audio to audio positions.
|
||||
|
||||
Runs against **omni's** thinker override, not upstream's: this is the
|
||||
only unit-level coverage of omni's call into
|
||||
``merge_interleaved_embeddings``, whose upstream signature dropped
|
||||
``num_video``/``num_audio`` in vLLM #46213.
|
||||
"""
|
||||
hidden = 8
|
||||
audio_val, video_val = 10.0, 30.0
|
||||
@@ -270,11 +291,11 @@ class TestEmbedInputIds:
|
||||
audio_n = sum(audio_chunks) # 6
|
||||
|
||||
mm_embeds = [
|
||||
torch.full((video_n, hidden), video_val),
|
||||
torch.full((audio_n, hidden), audio_val),
|
||||
set_mm_embedding_modality(torch.full((video_n, hidden), video_val), "video"),
|
||||
set_mm_embedding_modality(torch.full((audio_n, hidden), audio_val), "audio"),
|
||||
]
|
||||
|
||||
model, _ = make_mock_model(mocker, hidden)
|
||||
model, _ = make_mock_model(mocker, hidden, thinker_cls=_omni_qwen2_5_thinker_cls())
|
||||
result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal)
|
||||
|
||||
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
@@ -308,8 +329,8 @@ class TestMergeInterleavedEmbeddings:
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
torch.full((num_video, hidden), 30.0),
|
||||
torch.full((num_audio, hidden), 10.0),
|
||||
set_mm_embedding_modality(torch.full((num_video, hidden), 30.0), "video"),
|
||||
set_mm_embedding_modality(torch.full((num_audio, hidden), 10.0), "audio"),
|
||||
]
|
||||
|
||||
result = m.merge_interleaved_embeddings(
|
||||
@@ -318,8 +339,6 @@ class TestMergeInterleavedEmbeddings:
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
num_video,
|
||||
num_audio,
|
||||
)
|
||||
|
||||
video_pos = is_video.nonzero(as_tuple=True)[0]
|
||||
|
||||
@@ -84,6 +84,7 @@ class _CpuFusedMoE3DWithLoRA(FusedMoE3DWithLoRA):
|
||||
self.tp_size = 1
|
||||
self.tp_rank = 0
|
||||
self._w13_slices = 1
|
||||
self.enable_moe_shared_loras = False
|
||||
|
||||
|
||||
def _save_peft_adapter(tmp_path: Path) -> Path:
|
||||
|
||||
@@ -69,6 +69,9 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
# vLLM 0.26: one entry per KV cache group; max_model_len=8 with
|
||||
# block_sizes=[1] means one group of 8 blocks.
|
||||
max_num_blocks_per_req=[8],
|
||||
)
|
||||
input_batch.add_request(
|
||||
CachedRequestState(
|
||||
|
||||
@@ -4,7 +4,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
import vllm_omni.worker.gpu_generation_model_runner as gen_runner_module
|
||||
from vllm_omni.worker.gpu_generation_model_runner import (
|
||||
|
||||
@@ -1245,7 +1245,7 @@ class TestRankAwareKVRouting(unittest.TestCase):
|
||||
|
||||
class TestAttachOmniConnectorOutput(unittest.TestCase):
|
||||
def test_wraps_empty_model_runner_output_when_signals_exist(self):
|
||||
from vllm.v1.worker.gpu_model_runner import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
host = MixinHost()
|
||||
host.get_omni_connector_output = lambda: OmniConnectorOutput(chunk_ready_req_ids={"req-1"})
|
||||
|
||||
Reference in new issue
Block a user