From 722ad6bbaea83ace6478c7c68800033fd76c9c8f Mon Sep 17 00:00:00 2001 From: tzhouam Date: Tue, 17 Mar 2026 03:26:42 +0000 Subject: [PATCH] rebase: align vllm-omni with vLLM 0a0a1a198be8 Signed-off-by: tzhouam --- docker/Dockerfile.ci | 15 +++--- .../qwen2_5_omni_ci_1773717276.yaml | 48 +++++++++++++++++++ .../qwen2_5_omni_ci_1773717938.yaml | 48 +++++++++++++++++++ .../qwen3_omni_ci_1773717276.yaml | 45 +++++++++++++++++ .../qwen3_omni_ci_1773717938.yaml | 45 +++++++++++++++++ .../qwen2_5_omni/qwen2_5_omni_thinker.py | 14 ------ .../qwen3_omni/qwen3_omni_moe_thinker.py | 26 ++-------- 7 files changed, 200 insertions(+), 41 deletions(-) create mode 100644 tests/e2e/stage_configs/qwen2_5_omni_ci_1773717276.yaml create mode 100644 tests/e2e/stage_configs/qwen2_5_omni_ci_1773717938.yaml create mode 100644 tests/e2e/stage_configs/qwen3_omni_ci_1773717276.yaml create mode 100644 tests/e2e/stage_configs/qwen3_omni_ci_1773717938.yaml diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 2306dc48c..15999c8fb 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -14,12 +14,15 @@ RUN apt-get update && \ RUN uv pip uninstall --system -y vllm || true # Install vLLM from source at the selected commit using precompiled artifacts -ENV VLLM_PRECOMPILED_WHEEL_COMMIT=2754231ba3a72f41e62922d1552c33e8f3f6a9d1 -RUN uv pip install --system --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/${VLLM_PRECOMPILED_WHEEL_COMMIT} - -RUN uv pip install --system ".[dev]" - -RUN uv pip install --system --upgrade \ +ENV VLLM_PRECOMPILED_WHEEL_COMMIT=0a0a1a198be88e1782b52fa31738896468200a76 +RUN git clone https://github.com/vllm-project/vllm.git /workspace/vllm && \ + cd /workspace/vllm && \ + git checkout ${VLLM_PRECOMPILED_WHEEL_COMMIT} && \ + (uv pip uninstall --system -y vllm || true) && \ + VLLM_USE_PRECOMPILED=1 uv pip install --system -e . && \ + cd ${APP_DIR} && \ + uv pip install --system ".[dev]" && \ + uv pip install --system --upgrade \ "flashinfer-cubin==0.6.6" \ "nvidia-cublas-cu12==12.9.1.4" \ "numpy==2.2.6" diff --git a/tests/e2e/stage_configs/qwen2_5_omni_ci_1773717276.yaml b/tests/e2e/stage_configs/qwen2_5_omni_ci_1773717276.yaml new file mode 100644 index 000000000..b68bdb71a --- /dev/null +++ b/tests/e2e/stage_configs/qwen2_5_omni_ci_1773717276.yaml @@ -0,0 +1,48 @@ +stage_args: +- stage_id: 0 + runtime: {process: true, devices: '0', max_batch_size: 1} + engine_args: {model_stage: thinker, model_arch: Qwen2_5OmniForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.9, + skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, engine_output_type: latent, + enable_prefix_caching: false, mm_processor_cache_gb: 0} + is_comprehension: true + final_output: true + final_output_type: text + default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 128, + seed: 42, detokenize: true, repetition_penalty: 1.1} +- stage_id: 1 + runtime: {process: true, devices: '1', max_batch_size: 1} + engine_args: {model_stage: talker, model_arch: Qwen2_5OmniForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.4, + skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, enable_prefix_caching: false, + engine_output_type: latent} + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker + default_sampling_params: + temperature: 0.9 + top_p: 0.8 + top_k: 40 + max_tokens: 4096 + seed: 42 + detokenize: true + repetition_penalty: 1.05 + stop_token_ids: [8294] +- stage_id: 2 + runtime: {process: true, devices: '1', max_batch_size: 1} + engine_args: {model_stage: code2wav, model_arch: Qwen2_5OmniForConditionalGeneration, + worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler, + gpu_memory_utilization: 0.5, enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false, + engine_output_type: audio, max_num_batched_tokens: 4096, max_model_len: 4096} + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 4096, + seed: 42, detokenize: true, repetition_penalty: 1.1} +runtime: + enabled: true + defaults: {window_size: -1, max_inflight: 1} + edges: + - {from: 0, to: 1, window_size: -1} + - {from: 1, to: 2, window_size: -1} diff --git a/tests/e2e/stage_configs/qwen2_5_omni_ci_1773717938.yaml b/tests/e2e/stage_configs/qwen2_5_omni_ci_1773717938.yaml new file mode 100644 index 000000000..b68bdb71a --- /dev/null +++ b/tests/e2e/stage_configs/qwen2_5_omni_ci_1773717938.yaml @@ -0,0 +1,48 @@ +stage_args: +- stage_id: 0 + runtime: {process: true, devices: '0', max_batch_size: 1} + engine_args: {model_stage: thinker, model_arch: Qwen2_5OmniForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.9, + skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, engine_output_type: latent, + enable_prefix_caching: false, mm_processor_cache_gb: 0} + is_comprehension: true + final_output: true + final_output_type: text + default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 128, + seed: 42, detokenize: true, repetition_penalty: 1.1} +- stage_id: 1 + runtime: {process: true, devices: '1', max_batch_size: 1} + engine_args: {model_stage: talker, model_arch: Qwen2_5OmniForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.4, + skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, enable_prefix_caching: false, + engine_output_type: latent} + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker + default_sampling_params: + temperature: 0.9 + top_p: 0.8 + top_k: 40 + max_tokens: 4096 + seed: 42 + detokenize: true + repetition_penalty: 1.05 + stop_token_ids: [8294] +- stage_id: 2 + runtime: {process: true, devices: '1', max_batch_size: 1} + engine_args: {model_stage: code2wav, model_arch: Qwen2_5OmniForConditionalGeneration, + worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler, + gpu_memory_utilization: 0.5, enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false, + engine_output_type: audio, max_num_batched_tokens: 4096, max_model_len: 4096} + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 4096, + seed: 42, detokenize: true, repetition_penalty: 1.1} +runtime: + enabled: true + defaults: {window_size: -1, max_inflight: 1} + edges: + - {from: 0, to: 1, window_size: -1} + - {from: 1, to: 2, window_size: -1} diff --git a/tests/e2e/stage_configs/qwen3_omni_ci_1773717276.yaml b/tests/e2e/stage_configs/qwen3_omni_ci_1773717276.yaml new file mode 100644 index 000000000..c763bb2c7 --- /dev/null +++ b/tests/e2e/stage_configs/qwen3_omni_ci_1773717276.yaml @@ -0,0 +1,45 @@ +stage_args: +- stage_id: 0 + runtime: {devices: '0', max_batch_size: 5} + engine_args: {model_stage: thinker, model_arch: Qwen3OmniMoeForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + gpu_memory_utilization: 0.9, enforce_eager: false, trust_remote_code: true, engine_output_type: latent, + distributed_executor_backend: mp, max_num_batched_tokens: 32768, max_model_len: 32768, + enable_prefix_caching: false, mm_processor_cache_gb: 0, hf_config_name: thinker_config, + tensor_parallel_size: 1, load_format: dummy, custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk} + final_output: true + final_output_type: text + is_comprehension: true + default_sampling_params: {temperature: 0.4, top_p: 0.9, top_k: 1, max_tokens: 100, + seed: 42, ignore_eos: false, detokenize: true, repetition_penalty: 1.05} +- stage_id: 1 + runtime: {devices: '1', max_batch_size: 5} + engine_args: {model_stage: talker, model_arch: Qwen3OmniMoeForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + gpu_memory_utilization: 0.5, enforce_eager: false, trust_remote_code: true, engine_output_type: latent, + enable_prefix_caching: false, max_num_batched_tokens: 32768, max_model_len: 32768, + distributed_executor_backend: mp, hf_config_name: talker_config, load_format: dummy, + custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk} + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker + default_sampling_params: + temperature: 0.9 + top_k: 50 + max_tokens: 1000 + seed: 42 + detokenize: false + repetition_penalty: 1.05 + stop_token_ids: [2150] +- stage_id: 2 + runtime: {devices: '1', max_batch_size: 1} + engine_args: {model_stage: code2wav, model_arch: Qwen3OmniMoeForConditionalGeneration, + worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler, + enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false, engine_output_type: audio, + gpu_memory_utilization: 0.1, distributed_executor_backend: mp, max_num_batched_tokens: 100000, + hf_config_name: thinker_config, async_scheduling: false, load_format: dummy} + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 2000, + seed: 42, detokenize: true, repetition_penalty: 1.1} +async_chunk: true diff --git a/tests/e2e/stage_configs/qwen3_omni_ci_1773717938.yaml b/tests/e2e/stage_configs/qwen3_omni_ci_1773717938.yaml new file mode 100644 index 000000000..c763bb2c7 --- /dev/null +++ b/tests/e2e/stage_configs/qwen3_omni_ci_1773717938.yaml @@ -0,0 +1,45 @@ +stage_args: +- stage_id: 0 + runtime: {devices: '0', max_batch_size: 5} + engine_args: {model_stage: thinker, model_arch: Qwen3OmniMoeForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + gpu_memory_utilization: 0.9, enforce_eager: false, trust_remote_code: true, engine_output_type: latent, + distributed_executor_backend: mp, max_num_batched_tokens: 32768, max_model_len: 32768, + enable_prefix_caching: false, mm_processor_cache_gb: 0, hf_config_name: thinker_config, + tensor_parallel_size: 1, load_format: dummy, custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk} + final_output: true + final_output_type: text + is_comprehension: true + default_sampling_params: {temperature: 0.4, top_p: 0.9, top_k: 1, max_tokens: 100, + seed: 42, ignore_eos: false, detokenize: true, repetition_penalty: 1.05} +- stage_id: 1 + runtime: {devices: '1', max_batch_size: 5} + engine_args: {model_stage: talker, model_arch: Qwen3OmniMoeForConditionalGeneration, + worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler, + gpu_memory_utilization: 0.5, enforce_eager: false, trust_remote_code: true, engine_output_type: latent, + enable_prefix_caching: false, max_num_batched_tokens: 32768, max_model_len: 32768, + distributed_executor_backend: mp, hf_config_name: talker_config, load_format: dummy, + custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk} + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker + default_sampling_params: + temperature: 0.9 + top_k: 50 + max_tokens: 1000 + seed: 42 + detokenize: false + repetition_penalty: 1.05 + stop_token_ids: [2150] +- stage_id: 2 + runtime: {devices: '1', max_batch_size: 1} + engine_args: {model_stage: code2wav, model_arch: Qwen3OmniMoeForConditionalGeneration, + worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler, + enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false, engine_output_type: audio, + gpu_memory_utilization: 0.1, distributed_executor_backend: mp, max_num_batched_tokens: 100000, + hf_config_name: thinker_config, async_scheduling: false, load_format: dummy} + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 2000, + seed: 42, detokenize: true, repetition_penalty: 1.1} +async_chunk: true diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index 5c69fca66..030703408 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -58,10 +58,6 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItems, ) from vllm.multimodal.parse import MultiModalDataItems -from vllm.multimodal.processing import ( - ProcessorInputs, - TimingContext, -) from vllm.multimodal.processing.processor import ( MultiModalPromptUpdates, PlaceholderFeaturesInfo, @@ -80,16 +76,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ): """Override to fix use_audio_in_video detection when mm cache returns None.""" - def _cached_apply_hf_processor( - self, - inputs: ProcessorInputs, - timing_ctx: TimingContext, - ): - mm_processor_kwargs = inputs.hf_processor_mm_kwargs - if mm_processor_kwargs.get("use_audio_in_video", False): - return self._apply_hf_processor(inputs, timing_ctx) - return super()._cached_apply_hf_processor(inputs, timing_ctx) - def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index 5111fe1d4..84d8cbace 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -498,19 +498,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( mm_item_counts, ) else: - if use_audio_in_video: - # When use_audio_in_video=True, audio is extracted from video and embedded - # in the video placeholder tokens. We should: - # 1. Filter out audio from prompt updates (audio has no separate placeholder) - # 2. Apply remaining updates (video, image, etc.) - # 3. Derive audio placeholders from video placeholders + if use_audio_in_video and "audio" in mm_prompt_updates: filtered_updates = {k: v for k, v in mm_prompt_updates.items() if k != "audio"} prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, filtered_updates, ) - # Derive audio placeholders from video placeholders - mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_item_counts) + mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates) else: prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, @@ -629,7 +623,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( def get_replacement_qwen2_use_audio_in_video(item_idx: int): nonlocal audio_in_video_item_idx - audio_num_features = audio_output_lengths[audio_in_video_item_idx + item_idx] + audio_num_features = audio_output_lengths[audio_in_video_item_idx] video_grid_thw = out_mm_data["video_grid_thw"][item_idx] audio_in_video_item_idx += 1 @@ -675,27 +669,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( def _derive_audio_from_video_placeholders( self, placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], - mm_item_counts: Mapping[str, int], + mm_prompt_updates: MultiModalPromptUpdates, ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: """ Helper to derive audio placeholders from video placeholders when use_audio_in_video=True. - - In use_audio_in_video mode, audio is extracted from video and embedded - within the video placeholder tokens. This function creates audio placeholder - info by extracting the audio token positions from video placeholders. - - Args: - placeholders: Current placeholders (should contain "video") - mm_item_counts: Counts of multimodal items from mm_items.get_all_counts() """ if "video" not in placeholders: return placeholders - # Validate audio and video counts match - # In use_audio_in_video mode, audio count comes from mm_items (extracted from video) num_videos = len(placeholders["video"]) - num_audios = mm_item_counts.get("audio", 0) + num_audios = len(mm_prompt_updates.get("audio", [])) if num_audios != num_videos: raise ValueError( f"use_audio_in_video requires equal number of audio and video items, got {num_audios=}, {num_videos=}"