Update vLLM wheel version in Dockerfile and enhance input processing for omni models. Added functionality to reinject omni-specific fields into processed prompts and improved handling of multimodal audio extraction in serving_chat.py. Adjusted SharedMemoryConnector logic for payload size checks and refined input preprocessing methods for better compatibility with multimodal data.

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-02-20 08:34:53 +00:00
1 parent 663976b8ca
commit 9869ea007e
8 files changed
+214 -96

No files matched your search

+25 -2
View File
@@ -1,4 +1,4 @@
from collections.abc import Callable
from collections.abc import Callable, Sequence
from typing import Any
import cloudpickle
@@ -8,9 +8,11 @@ from tqdm import tqdm
# External library imports (vLLM)
from vllm.config import CompilationConfig, StructuredOutputsConfig, is_init_field
from vllm.entrypoints.llm import LLM
from vllm.inputs import ProcessorInputs, PromptType
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput, RequestOutput
from vllm.plugins.io_processors import get_io_processor
from vllm.renderers.inputs.preprocess import parse_model_prompt
from vllm.usage.usage_lib import UsageContext
from vllm.utils.counter import Counter
from vllm.v1.engine.llm_engine import LLMEngine
@@ -19,7 +21,7 @@ from vllm_omni.distributed.omni_connectors import initialize_orchestrator_connec
# Internal imports (our code)
from vllm_omni.engine.arg_utils import OmniEngineArgs
from vllm_omni.engine.input_processor import OmniInputProcessor
from vllm_omni.engine.input_processor import OmniInputProcessor, reinject_omni_fields
from vllm_omni.engine.output_processor import MultimodalOutputProcessor
from vllm_omni.entrypoints.utils import (
load_stage_configs_from_model,
@@ -182,6 +184,27 @@ class OmniLLM(LLM):
self.model_config = self.llm_engine.model_config
self.input_processor = self.llm_engine.input_processor
# ------------------------------------------------------------------
# Override upstream _preprocess_cmpl so that omni-specific fields
# (additional_information, prompt_embeds, …) survive the renderer's
# process_for_engine step which only copies standard vLLM keys.
# ------------------------------------------------------------------
def _preprocess_cmpl(
self,
prompts: Sequence[PromptType],
tokenization_kwargs: dict[str, Any] | None = None,
) -> Sequence[ProcessorInputs]:
renderer = self.renderer
model_config = self.model_config
parsed_prompts = [parse_model_prompt(model_config, prompt) for prompt in prompts]
tok_params = renderer.default_cmpl_tok_params.with_kwargs(**(tokenization_kwargs or {}))
results = renderer.render_cmpl(parsed_prompts, tok_params)
reinject_omni_fields(results, parsed_prompts)
return results
def close(self) -> None:
"""Close resources.
+98 -22
View File
@@ -371,6 +371,15 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
default_template_content_format,
).with_defaults(default_template_kwargs)
# OMNI: When use_audio_in_video=True, the qwen2_5_omni_thinker mm
# processor asserts that audio items are present alongside video items
# (inside render_chat_async). We must inject audio_url items into the
# messages BEFORE calling render_chat_async so the mm processor can
# count them correctly during tokenisation.
mm_proc_kw = getattr(request, "mm_processor_kwargs", None) or {}
if mm_proc_kw.get("use_audio_in_video", False):
messages = await self._inject_audio_from_video_urls(messages)
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
[messages],
chat_params,
@@ -380,28 +389,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
},
)
# OMNI: When use_audio_in_video=True, the upstream renderer does not
# extract audio from video. We do it here after rendering so that the
# audio data is present in multi_modal_data before the engine processes
# the request.
mm_proc_kw = getattr(request, "mm_processor_kwargs", None) or {}
if mm_proc_kw.get("use_audio_in_video", False) and isinstance(engine_prompt, dict):
mm_data = engine_prompt.get("multi_modal_data")
if mm_data is not None and "video" in mm_data and "audio" not in mm_data:
from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async
video_urls: list[str] = []
for msg in messages:
for part in msg.get("content") or []:
if isinstance(part, dict) and part.get("type") == "video_url":
url = part.get("video_url", {}).get("url")
if url:
video_urls.append(url)
if video_urls:
audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls))
engine_prompt.setdefault("multi_modal_data", {})["audio"] = list(audios)
tokenizer = renderer.get_tokenizer()
# tool parsing is done only if a tool_parser has been set and if
@@ -422,6 +409,95 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
return conversation, [engine_prompt]
async def _inject_audio_from_video_urls(
self,
messages: list[ChatCompletionMessageParam],
) -> list[ChatCompletionMessageParam]:
"""Pre-extract audio from video URLs and inject as audio_url content items.
When use_audio_in_video=True, the qwen2_5_omni_thinker multimodal
processor requires that the number of audio items equals the number of
video items (it subtracts mm_counts["video"] from mm_counts["audio"]).
The client only sends video_url items; this method adds the matching
audio_url items on the server side before the renderer processes them.
"""
import io
from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async
new_messages: list[ChatCompletionMessageParam] = []
for msg in messages:
content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
if not isinstance(content, list):
new_messages.append(msg)
continue
video_urls = [
part.get("video_url", {}).get("url")
for part in content
if isinstance(part, dict) and part.get("type") == "video_url" and part.get("video_url", {}).get("url")
]
if not video_urls:
new_messages.append(msg)
continue
audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls))
audio_items: list[dict] = []
for audio_array, sample_rate in audios:
buf = io.BytesIO()
if soundfile is not None:
soundfile.write(buf, audio_array, samplerate=int(sample_rate), format="WAV")
else:
import struct
import numpy as np
audio_np = np.asarray(audio_array, dtype=np.float32)
sr = int(sample_rate)
num_channels = 1
bits_per_sample = 32
num_frames = len(audio_np)
data_size = num_frames * num_channels * (bits_per_sample // 8)
# Write minimal RIFF/WAV header
buf.write(b"RIFF")
buf.write(struct.pack("<I", 36 + data_size))
buf.write(b"WAVE")
buf.write(b"fmt ")
buf.write(
struct.pack(
"<IHHIIHH",
16,
3,
num_channels,
sr,
sr * num_channels * (bits_per_sample // 8),
num_channels * (bits_per_sample // 8),
bits_per_sample,
)
)
buf.write(b"data")
buf.write(struct.pack("<I", data_size))
buf.write(audio_np.tobytes())
audio_b64 = base64.b64encode(buf.getvalue()).decode()
audio_items.append(
{
"type": "audio_url",
"audio_url": {"url": f"data:audio/wav;base64,{audio_b64}"},
}
)
new_content = list(content) + audio_items
if isinstance(msg, dict):
new_msg = {**msg, "content": new_content}
else:
new_msg = msg.model_copy(update={"content": new_content})
new_messages.append(new_msg)
return new_messages
def _to_sampling_params_list(self, sampling_params_list: list[dict]) -> list[SamplingParams]:
final_sampling_params_list = []
for sampling_params in sampling_params_list: