From 08884c4cfbbeb48b23ce67e265885b6c7774ccdd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 3 Sep 2026 13:18:06 +0800 Subject: [PATCH] rebase: adapt to vLLM 0.29 callback, embedding and video decoder changes Four unrelated 0.29 surface changes, each reproduced locally against the pinned f5c3cc24 build: - PromptUpdateDetails.is_embed now takes only the full token list. The Bagel img2img replacement still declared (tok, seq), which crashed stage-0 engine core startup and produced the Distributed and Qwen3-Omni "no live replica" failures rather than anything KV related. - dispatch_unquantized_gemm gained a linear_backend argument and is called with it. The CPU-forcing test stubs took no arguments; they now accept and ignore whatever upstream passes so a further signature change does not break them again. - VocabParallelEmbedding gained a fused kernel that engages whenever tp_size > 1 on CUDA, and it has no CPU implementation. The encoder TP fixtures drive CPU tensors under a faked TP group, so they pin the portable masked path. - The pyav video decoder backend was removed; valid options are now opencv, torchcodec, pynvvideocodec and deepstream. The OpenAI video endpoint hardcoded pyav, which turned every video request into a 400. It moves to opencv, the upstream default. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0146Xu5Sgj4vXxLxvLjV5YPj Signed-off-by: tzhouam --- tests/diffusion/layers/test_adalayernorm.py | 2 +- .../models/boogu_image/test_boogu_image_transformer.py | 2 +- tests/diffusion/models/dreamzero/test_qkv_fusion.py | 2 +- .../models/glm_image/test_glm_image_quantization.py | 2 +- .../models/mistral_encoder/test_mistral_encoder_tp.py | 9 ++++++++- .../models/sana_video/test_transformer_sana_video.py | 2 +- .../models/sana_video/test_transformer_sana_video_tp.py | 2 +- tests/diffusion/models/t5_encoder/test_t5_encoder_tp.py | 7 +++++++ vllm_omni/entrypoints/openai/video_api_utils.py | 4 +++- vllm_omni/model_executor/models/bagel/bagel.py | 3 ++- 10 files changed, 26 insertions(+), 9 deletions(-) diff --git a/tests/diffusion/layers/test_adalayernorm.py b/tests/diffusion/layers/test_adalayernorm.py index 5e41b7a26..51fc52f35 100644 --- a/tests/diffusion/layers/test_adalayernorm.py +++ b/tests/diffusion/layers/test_adalayernorm.py @@ -46,7 +46,7 @@ def _force_default_gemm(monkeypatch): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) diff --git a/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py b/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py index 4296c1cde..43e2f12f4 100644 --- a/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py +++ b/tests/diffusion/models/boogu_image/test_boogu_image_transformer.py @@ -59,7 +59,7 @@ def _force_default_gemm(monkeypatch): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) diff --git a/tests/diffusion/models/dreamzero/test_qkv_fusion.py b/tests/diffusion/models/dreamzero/test_qkv_fusion.py index 16b78693f..c7b2f032f 100644 --- a/tests/diffusion/models/dreamzero/test_qkv_fusion.py +++ b/tests/diffusion/models/dreamzero/test_qkv_fusion.py @@ -52,7 +52,7 @@ def _force_default_gemm(monkeypatch): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) diff --git a/tests/diffusion/models/glm_image/test_glm_image_quantization.py b/tests/diffusion/models/glm_image/test_glm_image_quantization.py index 8ab046f0c..c08bfb894 100644 --- a/tests/diffusion/models/glm_image/test_glm_image_quantization.py +++ b/tests/diffusion/models/glm_image/test_glm_image_quantization.py @@ -69,7 +69,7 @@ def _force_default_gemm(monkeypatch): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) diff --git a/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py b/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py index 20024abe7..17fc1540e 100644 --- a/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py +++ b/tests/diffusion/models/mistral_encoder/test_mistral_encoder_tp.py @@ -64,6 +64,13 @@ def setup_tp_group(monkeypatch, mocker): lambda: 0, ) + # vLLM 0.29 added a CUDA-only fused embedding kernel that engages whenever + # tp_size > 1; these tests drive CPU tensors, so keep the portable masked path. + monkeypatch.setattr( + "vllm.model_executor.layers.vocab_parallel_embedding.current_platform.is_cuda", + lambda: False, + ) + mock_tp_group = mocker.MagicMock() mock_tp_group.world_size = 2 mocker.patch("vllm.distributed.parallel_state.get_tp_group", return_value=mock_tp_group) @@ -94,7 +101,7 @@ def setup_tp_group(monkeypatch, mocker): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) with set_current_vllm_config(VllmConfig(device_config=device_config)): diff --git a/tests/diffusion/models/sana_video/test_transformer_sana_video.py b/tests/diffusion/models/sana_video/test_transformer_sana_video.py index 0bc2ddb8e..3e1e40c2a 100644 --- a/tests/diffusion/models/sana_video/test_transformer_sana_video.py +++ b/tests/diffusion/models/sana_video/test_transformer_sana_video.py @@ -22,7 +22,7 @@ def _init_distributed(monkeypatch): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) os.environ.setdefault("MASTER_ADDR", "localhost") os.environ.setdefault("MASTER_PORT", "29501") diff --git a/tests/diffusion/models/sana_video/test_transformer_sana_video_tp.py b/tests/diffusion/models/sana_video/test_transformer_sana_video_tp.py index 41ca717d6..7105f5b4c 100644 --- a/tests/diffusion/models/sana_video/test_transformer_sana_video_tp.py +++ b/tests/diffusion/models/sana_video/test_transformer_sana_video_tp.py @@ -51,7 +51,7 @@ def force_default_gemm(monkeypatch): monkeypatch.setattr( "vllm.model_executor.layers.linear.dispatch_unquantized_gemm", - lambda: default_unquantized_gemm, + lambda *_args, **_kwargs: default_unquantized_gemm, ) diff --git a/tests/diffusion/models/t5_encoder/test_t5_encoder_tp.py b/tests/diffusion/models/t5_encoder/test_t5_encoder_tp.py index b36acc68a..6eb469fac 100644 --- a/tests/diffusion/models/t5_encoder/test_t5_encoder_tp.py +++ b/tests/diffusion/models/t5_encoder/test_t5_encoder_tp.py @@ -54,6 +54,13 @@ def setup_tp_group(monkeypatch, mocker): lambda: 0, ) + # vLLM 0.29 added a CUDA-only fused embedding kernel that engages whenever + # tp_size > 1; these tests drive CPU tensors, so keep the portable masked path. + monkeypatch.setattr( + "vllm.model_executor.layers.vocab_parallel_embedding.current_platform.is_cuda", + lambda: False, + ) + # TP group mock_tp_group = mocker.MagicMock() mock_tp_group.world_size = 2 diff --git a/vllm_omni/entrypoints/openai/video_api_utils.py b/vllm_omni/entrypoints/openai/video_api_utils.py index 35a39b2ea..0f01fb28f 100644 --- a/vllm_omni/entrypoints/openai/video_api_utils.py +++ b/vllm_omni/entrypoints/openai/video_api_utils.py @@ -156,7 +156,9 @@ def _decode_video_bytes( frames_array, metadata = loader.load_bytes( video_bytes, num_frames=num_frames, - backend="pyav", + # vLLM 0.29 removed the "pyav" decoder backend; "opencv" is + # upstream's default and the remaining CPU-only option. + backend="opencv", keep=keep, ) except Exception as exc: diff --git a/vllm_omni/model_executor/models/bagel/bagel.py b/vllm_omni/model_executor/models/bagel/bagel.py index aa36533df..e8ad443e4 100644 --- a/vllm_omni/model_executor/models/bagel/bagel.py +++ b/vllm_omni/model_executor/models/bagel/bagel.py @@ -433,7 +433,8 @@ class OmniBagelMultiModalProcessor(BaseMultiModalProcessor[OmniBagelProcessingIn embed_mask = [True] * num_vae_total + [False] + [True] * num_vit_total return PromptUpdateDetails( full=tokens, - is_embed=lambda _tok, _seq, _m=embed_mask: torch.tensor(_m, dtype=torch.bool), + # vLLM 0.29 calls is_embed with just the full token list. + is_embed=lambda _full, _m=embed_mask: torch.tensor(_m, dtype=torch.bool), ) replacements.append(