anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

rebase: align vllm-omni with vLLM 0a0a1a198be8

Browse Source 
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-03-17 03:26:42 +00:00
1 parent 0eccb77c4f
commit 722ad6bbae
7 files changed
+200 -41
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docker/Dockerfile.ci
+9 -6
View File
@@ -14,12 +14,15 @@ RUN apt-get update && \
RUN uv pip uninstall --system -y vllm || true
# Install vLLM from source at the selected commit using precompiled artifacts
ENV VLLM_PRECOMPILED_WHEEL_COMMIT=2754231ba3a72f41e62922d1552c33e8f3f6a9d1
RUN uv pip install --system --force-reinstall --prerelease=allow vllm --extra-index-url https://wheels.vllm.ai/${VLLM_PRECOMPILED_WHEEL_COMMIT}
RUN uv pip install --system ".[dev]"
RUN uv pip install --system --upgrade \
ENV VLLM_PRECOMPILED_WHEEL_COMMIT=0a0a1a198be88e1782b52fa31738896468200a76
RUN git clone https://github.com/vllm-project/vllm.git /workspace/vllm && \
cd /workspace/vllm && \
git checkout ${VLLM_PRECOMPILED_WHEEL_COMMIT} && \
(uv pip uninstall --system -y vllm || true) && \
VLLM_USE_PRECOMPILED=1 uv pip install --system -e . && \
cd ${APP_DIR} && \
uv pip install --system ".[dev]" && \
uv pip install --system --upgrade \
"flashinfer-cubin==0.6.6" \
"nvidia-cublas-cu12==12.9.1.4" \
"numpy==2.2.6"
tests/e2e/stage_configs/qwen2_5_omni_ci_1773717276.yaml
+48
View File
@@ -0,0 +1,48 @@
stage_args:
- stage_id: 0
runtime: {process: true, devices: '0', max_batch_size: 1}
engine_args: {model_stage: thinker, model_arch: Qwen2_5OmniForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.9,
skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, engine_output_type: latent,
enable_prefix_caching: false, mm_processor_cache_gb: 0}
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 128,
seed: 42, detokenize: true, repetition_penalty: 1.1}
- stage_id: 1
runtime: {process: true, devices: '1', max_batch_size: 1}
engine_args: {model_stage: talker, model_arch: Qwen2_5OmniForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.4,
skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, enable_prefix_caching: false,
engine_output_type: latent}
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: true
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime: {process: true, devices: '1', max_batch_size: 1}
engine_args: {model_stage: code2wav, model_arch: Qwen2_5OmniForConditionalGeneration,
worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler,
gpu_memory_utilization: 0.5, enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false,
engine_output_type: audio, max_num_batched_tokens: 4096, max_model_len: 4096}
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 4096,
seed: 42, detokenize: true, repetition_penalty: 1.1}
runtime:
enabled: true
defaults: {window_size: -1, max_inflight: 1}
edges:
- {from: 0, to: 1, window_size: -1}
- {from: 1, to: 2, window_size: -1}
tests/e2e/stage_configs/qwen2_5_omni_ci_1773717938.yaml
+48
View File
@@ -0,0 +1,48 @@
stage_args:
- stage_id: 0
runtime: {process: true, devices: '0', max_batch_size: 1}
engine_args: {model_stage: thinker, model_arch: Qwen2_5OmniForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.9,
skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, engine_output_type: latent,
enable_prefix_caching: false, mm_processor_cache_gb: 0}
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 128,
seed: 42, detokenize: true, repetition_penalty: 1.1}
- stage_id: 1
runtime: {process: true, devices: '1', max_batch_size: 1}
engine_args: {model_stage: talker, model_arch: Qwen2_5OmniForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
max_model_len: 16384, max_num_batched_tokens: 16384, max_num_seqs: 1, gpu_memory_utilization: 0.4,
skip_mm_profiling: true, enforce_eager: 'true', trust_remote_code: true, enable_prefix_caching: false,
engine_output_type: latent}
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_p: 0.8
top_k: 40
max_tokens: 4096
seed: 42
detokenize: true
repetition_penalty: 1.05
stop_token_ids: [8294]
- stage_id: 2
runtime: {process: true, devices: '1', max_batch_size: 1}
engine_args: {model_stage: code2wav, model_arch: Qwen2_5OmniForConditionalGeneration,
worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler,
gpu_memory_utilization: 0.5, enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false,
engine_output_type: audio, max_num_batched_tokens: 4096, max_model_len: 4096}
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 4096,
seed: 42, detokenize: true, repetition_penalty: 1.1}
runtime:
enabled: true
defaults: {window_size: -1, max_inflight: 1}
edges:
- {from: 0, to: 1, window_size: -1}
- {from: 1, to: 2, window_size: -1}
tests/e2e/stage_configs/qwen3_omni_ci_1773717276.yaml
+45
View File
@@ -0,0 +1,45 @@
stage_args:
- stage_id: 0
runtime: {devices: '0', max_batch_size: 5}
engine_args: {model_stage: thinker, model_arch: Qwen3OmniMoeForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
gpu_memory_utilization: 0.9, enforce_eager: false, trust_remote_code: true, engine_output_type: latent,
distributed_executor_backend: mp, max_num_batched_tokens: 32768, max_model_len: 32768,
enable_prefix_caching: false, mm_processor_cache_gb: 0, hf_config_name: thinker_config,
tensor_parallel_size: 1, load_format: dummy, custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk}
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params: {temperature: 0.4, top_p: 0.9, top_k: 1, max_tokens: 100,
seed: 42, ignore_eos: false, detokenize: true, repetition_penalty: 1.05}
- stage_id: 1
runtime: {devices: '1', max_batch_size: 5}
engine_args: {model_stage: talker, model_arch: Qwen3OmniMoeForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
gpu_memory_utilization: 0.5, enforce_eager: false, trust_remote_code: true, engine_output_type: latent,
enable_prefix_caching: false, max_num_batched_tokens: 32768, max_model_len: 32768,
distributed_executor_backend: mp, hf_config_name: talker_config, load_format: dummy,
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk}
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 1000
seed: 42
detokenize: false
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime: {devices: '1', max_batch_size: 1}
engine_args: {model_stage: code2wav, model_arch: Qwen3OmniMoeForConditionalGeneration,
worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler,
enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false, engine_output_type: audio,
gpu_memory_utilization: 0.1, distributed_executor_backend: mp, max_num_batched_tokens: 100000,
hf_config_name: thinker_config, async_scheduling: false, load_format: dummy}
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 2000,
seed: 42, detokenize: true, repetition_penalty: 1.1}
async_chunk: true
tests/e2e/stage_configs/qwen3_omni_ci_1773717938.yaml
+45
View File
@@ -0,0 +1,45 @@
stage_args:
- stage_id: 0
runtime: {devices: '0', max_batch_size: 5}
engine_args: {model_stage: thinker, model_arch: Qwen3OmniMoeForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
gpu_memory_utilization: 0.9, enforce_eager: false, trust_remote_code: true, engine_output_type: latent,
distributed_executor_backend: mp, max_num_batched_tokens: 32768, max_model_len: 32768,
enable_prefix_caching: false, mm_processor_cache_gb: 0, hf_config_name: thinker_config,
tensor_parallel_size: 1, load_format: dummy, custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk}
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params: {temperature: 0.4, top_p: 0.9, top_k: 1, max_tokens: 100,
seed: 42, ignore_eos: false, detokenize: true, repetition_penalty: 1.05}
- stage_id: 1
runtime: {devices: '1', max_batch_size: 5}
engine_args: {model_stage: talker, model_arch: Qwen3OmniMoeForConditionalGeneration,
worker_type: ar, scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler,
gpu_memory_utilization: 0.5, enforce_eager: false, trust_remote_code: true, engine_output_type: latent,
enable_prefix_caching: false, max_num_batched_tokens: 32768, max_model_len: 32768,
distributed_executor_backend: mp, hf_config_name: talker_config, load_format: dummy,
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk}
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
default_sampling_params:
temperature: 0.9
top_k: 50
max_tokens: 1000
seed: 42
detokenize: false
repetition_penalty: 1.05
stop_token_ids: [2150]
- stage_id: 2
runtime: {devices: '1', max_batch_size: 1}
engine_args: {model_stage: code2wav, model_arch: Qwen3OmniMoeForConditionalGeneration,
worker_type: generation, scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler,
enforce_eager: true, trust_remote_code: true, enable_prefix_caching: false, engine_output_type: audio,
gpu_memory_utilization: 0.1, distributed_executor_backend: mp, max_num_batched_tokens: 100000,
hf_config_name: thinker_config, async_scheduling: false, load_format: dummy}
engine_input_source: [1]
final_output: true
final_output_type: audio
default_sampling_params: {temperature: 0.0, top_p: 1.0, top_k: -1, max_tokens: 2000,
seed: 42, detokenize: true, repetition_penalty: 1.1}
async_chunk: true
vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
-14
View File
@@ -58,10 +58,6 @@ from vllm.multimodal.inputs import (
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import (
ProcessorInputs,
TimingContext,
)
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
@@ -80,16 +76,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
):
"""Override to fix use_audio_in_video detection when mm cache returns None."""
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
):
mm_processor_kwargs = inputs.hf_processor_mm_kwargs
if mm_processor_kwargs.get("use_audio_in_video", False):
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py
+5 -21
View File
@@ -498,19 +498,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
mm_item_counts,
)
else:
if use_audio_in_video:
# When use_audio_in_video=True, audio is extracted from video and embedded
# in the video placeholder tokens. We should:
# 1. Filter out audio from prompt updates (audio has no separate placeholder)
# 2. Apply remaining updates (video, image, etc.)
# 3. Derive audio placeholders from video placeholders
if use_audio_in_video and "audio" in mm_prompt_updates:
filtered_updates = {k: v for k, v in mm_prompt_updates.items() if k != "audio"}
prompt_ids, mm_placeholders = self._apply_prompt_updates(
prompt_ids,
filtered_updates,
)
# Derive audio placeholders from video placeholders
mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_item_counts)
mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates)
else:
prompt_ids, mm_placeholders = self._apply_prompt_updates(
prompt_ids,
@@ -629,7 +623,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
nonlocal audio_in_video_item_idx
audio_num_features = audio_output_lengths[audio_in_video_item_idx + item_idx]
audio_num_features = audio_output_lengths[audio_in_video_item_idx]
video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
audio_in_video_item_idx += 1
@@ -675,27 +669,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
def _derive_audio_from_video_placeholders(
self,
placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
mm_item_counts: Mapping[str, int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
"""
Helper to derive audio placeholders from video placeholders when
use_audio_in_video=True.
In use_audio_in_video mode, audio is extracted from video and embedded
within the video placeholder tokens. This function creates audio placeholder
info by extracting the audio token positions from video placeholders.
Args:
placeholders: Current placeholders (should contain "video")
mm_item_counts: Counts of multimodal items from mm_items.get_all_counts()
"""
if "video" not in placeholders:
return placeholders
# Validate audio and video counts match
# In use_audio_in_video mode, audio count comes from mm_items (extracted from video)
num_videos = len(placeholders["video"])
num_audios = mm_item_counts.get("audio", 0)
num_audios = len(mm_prompt_updates.get("audio", []))
if num_audios != num_videos:
raise ValueError(
f"use_audio_in_video requires equal number of audio and video items, got {num_audios=}, {num_videos=}"