Update vLLM wheel version in Dockerfile and enhance input processing for omni models. Added functionality to reinject omni-specific fields into processed prompts and improved handling of multimodal audio extraction in serving_chat.py. Adjusted SharedMemoryConnector logic for payload size checks and refined input preprocessing methods for better compatibility with multimodal data.
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
663976b8ca
commit
9869ea007e
8 files changed
+214
-96
No files matched your search
@@ -1,4 +1,4 @@
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Any
|
||||
|
||||
import cloudpickle
|
||||
@@ -8,9 +8,11 @@ from tqdm import tqdm
|
||||
# External library imports (vLLM)
|
||||
from vllm.config import CompilationConfig, StructuredOutputsConfig, is_init_field
|
||||
from vllm.entrypoints.llm import LLM
|
||||
from vllm.inputs import ProcessorInputs, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.counter import Counter
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
@@ -19,7 +21,7 @@ from vllm_omni.distributed.omni_connectors import initialize_orchestrator_connec
|
||||
|
||||
# Internal imports (our code)
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
from vllm_omni.engine.input_processor import OmniInputProcessor
|
||||
from vllm_omni.engine.input_processor import OmniInputProcessor, reinject_omni_fields
|
||||
from vllm_omni.engine.output_processor import MultimodalOutputProcessor
|
||||
from vllm_omni.entrypoints.utils import (
|
||||
load_stage_configs_from_model,
|
||||
@@ -182,6 +184,27 @@ class OmniLLM(LLM):
|
||||
self.model_config = self.llm_engine.model_config
|
||||
self.input_processor = self.llm_engine.input_processor
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Override upstream _preprocess_cmpl so that omni-specific fields
|
||||
# (additional_information, prompt_embeds, …) survive the renderer's
|
||||
# process_for_engine step which only copies standard vLLM keys.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _preprocess_cmpl(
|
||||
self,
|
||||
prompts: Sequence[PromptType],
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> Sequence[ProcessorInputs]:
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
parsed_prompts = [parse_model_prompt(model_config, prompt) for prompt in prompts]
|
||||
tok_params = renderer.default_cmpl_tok_params.with_kwargs(**(tokenization_kwargs or {}))
|
||||
results = renderer.render_cmpl(parsed_prompts, tok_params)
|
||||
|
||||
reinject_omni_fields(results, parsed_prompts)
|
||||
return results
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close resources.
|
||||
|
||||
|
||||
@@ -371,6 +371,15 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
default_template_content_format,
|
||||
).with_defaults(default_template_kwargs)
|
||||
|
||||
# OMNI: When use_audio_in_video=True, the qwen2_5_omni_thinker mm
|
||||
# processor asserts that audio items are present alongside video items
|
||||
# (inside render_chat_async). We must inject audio_url items into the
|
||||
# messages BEFORE calling render_chat_async so the mm processor can
|
||||
# count them correctly during tokenisation.
|
||||
mm_proc_kw = getattr(request, "mm_processor_kwargs", None) or {}
|
||||
if mm_proc_kw.get("use_audio_in_video", False):
|
||||
messages = await self._inject_audio_from_video_urls(messages)
|
||||
|
||||
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
chat_params,
|
||||
@@ -380,28 +389,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
},
|
||||
)
|
||||
|
||||
# OMNI: When use_audio_in_video=True, the upstream renderer does not
|
||||
# extract audio from video. We do it here after rendering so that the
|
||||
# audio data is present in multi_modal_data before the engine processes
|
||||
# the request.
|
||||
mm_proc_kw = getattr(request, "mm_processor_kwargs", None) or {}
|
||||
if mm_proc_kw.get("use_audio_in_video", False) and isinstance(engine_prompt, dict):
|
||||
mm_data = engine_prompt.get("multi_modal_data")
|
||||
if mm_data is not None and "video" in mm_data and "audio" not in mm_data:
|
||||
from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async
|
||||
|
||||
video_urls: list[str] = []
|
||||
for msg in messages:
|
||||
for part in msg.get("content") or []:
|
||||
if isinstance(part, dict) and part.get("type") == "video_url":
|
||||
url = part.get("video_url", {}).get("url")
|
||||
if url:
|
||||
video_urls.append(url)
|
||||
|
||||
if video_urls:
|
||||
audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls))
|
||||
engine_prompt.setdefault("multi_modal_data", {})["audio"] = list(audios)
|
||||
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
@@ -422,6 +409,95 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
|
||||
return conversation, [engine_prompt]
|
||||
|
||||
async def _inject_audio_from_video_urls(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
) -> list[ChatCompletionMessageParam]:
|
||||
"""Pre-extract audio from video URLs and inject as audio_url content items.
|
||||
|
||||
When use_audio_in_video=True, the qwen2_5_omni_thinker multimodal
|
||||
processor requires that the number of audio items equals the number of
|
||||
video items (it subtracts mm_counts["video"] from mm_counts["audio"]).
|
||||
The client only sends video_url items; this method adds the matching
|
||||
audio_url items on the server side before the renderer processes them.
|
||||
"""
|
||||
import io
|
||||
|
||||
from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async
|
||||
|
||||
new_messages: list[ChatCompletionMessageParam] = []
|
||||
for msg in messages:
|
||||
content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
|
||||
if not isinstance(content, list):
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
|
||||
video_urls = [
|
||||
part.get("video_url", {}).get("url")
|
||||
for part in content
|
||||
if isinstance(part, dict) and part.get("type") == "video_url" and part.get("video_url", {}).get("url")
|
||||
]
|
||||
|
||||
if not video_urls:
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
|
||||
audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls))
|
||||
|
||||
audio_items: list[dict] = []
|
||||
for audio_array, sample_rate in audios:
|
||||
buf = io.BytesIO()
|
||||
if soundfile is not None:
|
||||
soundfile.write(buf, audio_array, samplerate=int(sample_rate), format="WAV")
|
||||
else:
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
|
||||
audio_np = np.asarray(audio_array, dtype=np.float32)
|
||||
sr = int(sample_rate)
|
||||
num_channels = 1
|
||||
bits_per_sample = 32
|
||||
num_frames = len(audio_np)
|
||||
data_size = num_frames * num_channels * (bits_per_sample // 8)
|
||||
# Write minimal RIFF/WAV header
|
||||
buf.write(b"RIFF")
|
||||
buf.write(struct.pack("<I", 36 + data_size))
|
||||
buf.write(b"WAVE")
|
||||
buf.write(b"fmt ")
|
||||
buf.write(
|
||||
struct.pack(
|
||||
"<IHHIIHH",
|
||||
16,
|
||||
3,
|
||||
num_channels,
|
||||
sr,
|
||||
sr * num_channels * (bits_per_sample // 8),
|
||||
num_channels * (bits_per_sample // 8),
|
||||
bits_per_sample,
|
||||
)
|
||||
)
|
||||
buf.write(b"data")
|
||||
buf.write(struct.pack("<I", data_size))
|
||||
buf.write(audio_np.tobytes())
|
||||
|
||||
audio_b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
audio_items.append(
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": f"data:audio/wav;base64,{audio_b64}"},
|
||||
}
|
||||
)
|
||||
|
||||
new_content = list(content) + audio_items
|
||||
if isinstance(msg, dict):
|
||||
new_msg = {**msg, "content": new_content}
|
||||
else:
|
||||
new_msg = msg.model_copy(update={"content": new_content})
|
||||
new_messages.append(new_msg)
|
||||
|
||||
return new_messages
|
||||
|
||||
def _to_sampling_params_list(self, sampling_params_list: list[dict]) -> list[SamplingParams]:
|
||||
final_sampling_params_list = []
|
||||
for sampling_params in sampling_params_list:
|
||||
|
||||
Reference in new issue
Block a user