[Model] Aura support - Non async chunk path (#4257)

Signed-off-by: Rein Yang <ruiruyang2@gmail.com>
Signed-off-by: R2-Y <ruiruyang2@gmail.com>
This commit is contained in:
rein yang authored and GitHub committed 2026-06-21 22:08:26 +08:00
1 parent 1292c47804
commit afbcc51672
30 files changed
+2958 -26

No files matched your search

+277 -9
View File
@@ -74,6 +74,7 @@ from vllm.entrypoints.serve.utils.api_utils import should_include_usage
from vllm.entrypoints.serve.utils.tool_calls_utils import maybe_filter_parallel_tool_calls
from vllm.inputs import PromptType
from vllm.logger import init_logger
from vllm.multimodal.media.connector import MediaConnector
from vllm.outputs import RequestOutput
from vllm.reasoning import ReasoningParser
from vllm.renderers import BaseRenderer, merge_kwargs
@@ -104,6 +105,7 @@ from vllm_omni.entrypoints.openai.protocol.images import (
)
from vllm_omni.entrypoints.openai.stage_params import (
build_stage_sampling_params_list,
clone_sampling_params,
get_default_sampling_params_list,
)
from vllm_omni.entrypoints.openai.utils import (
@@ -122,6 +124,10 @@ from vllm_omni.utils.audio import audio_chunk_pcm_bytes, audio_chunk_sample_rate
logger = init_logger(__name__)
async def _identity_async(value: Any) -> Any:
return value
class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
"""OpenAI-compatible chat serving for both LLM and Diffusion models.
@@ -654,6 +660,13 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
)
deferred_multi_modal_data: dict[str, Any] | None = None
if self._needs_multistage_multimodal_split():
messages, deferred_multi_modal_data = await self._prepare_multistage_multimodal_inputs(
messages,
request,
)
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
[messages],
chat_params,
@@ -698,18 +711,25 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
if hasattr(request, "cache_salt") and request.cache_salt is not None:
engine_prompt["cache_salt"] = request.cache_salt
additional_information = getattr(request, "additional_information", None)
if isinstance(additional_information, dict):
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
prompt_additional_information.update(additional_information)
if deferred_multi_modal_data:
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
prompt_additional_information["deferred_multi_modal_data"] = deferred_multi_modal_data
speaker = getattr(request, "voice", None) or getattr(request, "speaker", None)
normalized = validate_requested_speaker(speaker, self._get_supported_speakers())
if normalized is not None:
if "additional_information" not in engine_prompt or engine_prompt["additional_information"] is None:
engine_prompt["additional_information"] = {}
engine_prompt["additional_information"]["speaker"] = [normalized]
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
prompt_additional_information["speaker"] = [normalized]
language = getattr(request, "language", None)
if language is not None and isinstance(language, str) and language.strip():
if "additional_information" not in engine_prompt or engine_prompt["additional_information"] is None:
engine_prompt["additional_information"] = {}
engine_prompt["additional_information"]["language"] = [language.strip()]
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
prompt_additional_information["language"] = [language.strip()]
# Style instruction — used by Ming-flash-omni instruct TTS path
# (ming_task="instruct"). For the omni speech path the thinker2talker
@@ -717,20 +737,263 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
# which hardcodes instruction=None.
instructions = getattr(request, "instructions", None)
if instructions is not None and isinstance(instructions, str) and instructions.strip():
if "additional_information" not in engine_prompt or engine_prompt["additional_information"] is None:
engine_prompt["additional_information"] = {}
engine_prompt["additional_information"]["instruction"] = instructions.strip()
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
prompt_additional_information["instruction"] = instructions.strip()
return conversation, [engine_prompt]
@staticmethod
def _ensure_prompt_additional_information(engine_prompt: dict[str, Any]) -> dict[str, Any]:
additional_information = engine_prompt.get("additional_information")
if not isinstance(additional_information, dict):
additional_information = {}
engine_prompt["additional_information"] = additional_information
return additional_information
def _needs_multistage_multimodal_split(self) -> bool:
return bool(self._deferred_multimodal_modalities())
def _deferred_multimodal_modalities(self) -> set[str]:
stage_configs = list(getattr(self.engine_client, "stage_configs", []) or [])
if len(stage_configs) < 2:
return set()
first_stage_modalities = self._stage_input_modalities(stage_configs[0])
if not first_stage_modalities:
return set()
downstream_modalities: set[str] = set()
for stage in stage_configs[1:]:
downstream_modalities.update(self._stage_input_modalities(stage))
return downstream_modalities - first_stage_modalities
@staticmethod
def _stage_input_modalities(stage: Any) -> set[str]:
engine_args = getattr(stage, "engine_args", None)
explicit = (
getattr(stage, "input_modalities", None)
or getattr(stage, "modalities", None)
or getattr(engine_args, "input_modalities", None)
or getattr(engine_args, "modalities", None)
)
if explicit:
return {str(modality) for modality in as_list(explicit)}
model_stage = str(getattr(engine_args, "model_stage", None) or getattr(stage, "model_stage", "")).lower()
if model_stage in {"asr", "stt"} or model_stage.endswith("_asr"):
return {"audio"}
if any(name in model_stage for name in ("vision", "vl", "aura")):
return {"image", "video"}
if any(name in model_stage for name in ("tts", "talker", "code2wav")):
return set()
if getattr(stage, "requires_multimodal_data", False):
return {"audio", "image", "video"}
return set()
async def _prepare_multistage_multimodal_inputs(
self,
messages: list[ChatCompletionMessageParam],
request: ChatLikeRequest | ResponsesRequest,
) -> tuple[list[ChatCompletionMessageParam], dict[str, Any] | None]:
"""Hide modalities unsupported by stage 0 and stash them for downstream stages."""
deferred_modalities = self._deferred_multimodal_modalities()
deferred_parts: dict[str, list[Any]] = {modality: [] for modality in deferred_modalities}
stripped_messages: list[ChatCompletionMessageParam] = []
for message in messages:
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
stripped_messages.append(message)
continue
stripped_content: list[Any] = []
changed = False
for part in content:
modality, payload = self._deferred_multimodal_part(part, deferred_modalities)
if modality is not None:
if payload is not None:
deferred_parts.setdefault(modality, []).append(payload)
changed = True
continue
stripped_content.append(part)
if changed:
stripped_message = dict(message)
stripped_message["content"] = stripped_content
stripped_messages.append(cast(ChatCompletionMessageParam, stripped_message))
else:
stripped_messages.append(message)
deferred_parts = {modality: parts for modality, parts in deferred_parts.items() if parts}
if not deferred_parts:
return messages, None
media_connector = MediaConnector(
media_io_kwargs=getattr(request, "media_io_kwargs", None),
allowed_local_media_path=getattr(self.model_config, "allowed_local_media_path", "") or "",
allowed_media_domains=getattr(self.model_config, "allowed_media_domains", None),
)
multi_modal_data: dict[str, Any] = {}
for modality, parts in deferred_parts.items():
multi_modal_data[modality] = await self._materialize_deferred_multimodal_parts(
media_connector,
modality,
parts,
)
return stripped_messages, multi_modal_data
@staticmethod
def _deferred_multimodal_part(part: Any, deferred_modalities: set[str]) -> tuple[str | None, Any | None]:
if not isinstance(part, dict):
return None, None
part_type = part.get("type")
if part_type in {"video_url", "video"} and "video" in deferred_modalities:
video = part.get("video_url", part.get("video"))
if isinstance(video, dict):
video = video.get("url")
return "video", video
if part_type in {"image_url", "image_pil", "image"} and "image" in deferred_modalities:
image = part.get("image_pil", part.get("image_url", part.get("image")))
if isinstance(image, dict):
image = image.get("url")
return "image", image
if part_type in {"audio_url", "input_audio", "audio"} and "audio" in deferred_modalities:
audio = part.get("audio_url", part.get("input_audio", part.get("audio")))
if isinstance(audio, dict):
audio = audio.get("url") or audio.get("data")
return "audio", audio
return None, None
@staticmethod
async def _materialize_deferred_multimodal_parts(
media_connector: MediaConnector,
modality: str,
parts: list[Any],
) -> list[Any]:
if modality == "video":
return list(await asyncio.gather(*(media_connector.fetch_video_async(part) for part in parts)))
if modality == "image":
fetch_image = getattr(media_connector, "fetch_image_async", None)
if fetch_image is None:
return parts
return list(
await asyncio.gather(
*(fetch_image(part) if isinstance(part, str) else _identity_async(part) for part in parts)
)
)
if modality == "audio":
fetch_audio = getattr(media_connector, "fetch_audio_async", None)
if fetch_audio is None:
return parts
return list(
await asyncio.gather(
*(fetch_audio(part) if isinstance(part, str) else _identity_async(part) for part in parts)
)
)
return parts
async def _inject_audio_from_video_urls(
self,
messages: list[ChatCompletionMessageParam],
) -> list[ChatCompletionMessageParam]:
"""Pre-extract audio from video URLs and inject as audio_url content items.
When use_audio_in_video=True, the qwen2_5_omni_thinker multimodal
processor requires that the number of audio items equals the number of
video items (it subtracts mm_counts["video"] from mm_counts["audio"]).
The client only sends video_url items; this method adds the matching
audio_url items on the server side before the renderer processes them.
"""
import io
from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async
new_messages: list[ChatCompletionMessageParam] = []
for msg in messages:
content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
if not isinstance(content, list):
new_messages.append(msg)
continue
video_urls = [
part.get("video_url", {}).get("url")
for part in content
if isinstance(part, dict) and part.get("type") == "video_url" and part.get("video_url", {}).get("url")
]
if not video_urls:
new_messages.append(msg)
continue
audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls))
audio_items: list[dict] = []
for audio_array, sample_rate in audios:
buf = io.BytesIO()
if soundfile is not None:
soundfile.write(buf, audio_array, samplerate=int(sample_rate), format="WAV")
else:
import struct
import numpy as np
audio_np = np.asarray(audio_array, dtype=np.float32)
sr = int(sample_rate)
num_channels = 1
bits_per_sample = 32
num_frames = len(audio_np)
data_size = num_frames * num_channels * (bits_per_sample // 8)
# Write minimal RIFF/WAV header
buf.write(b"RIFF")
buf.write(struct.pack("<I", 36 + data_size))
buf.write(b"WAVE")
buf.write(b"fmt ")
buf.write(
struct.pack(
"<IHHIIHH",
16,
3,
num_channels,
sr,
sr * num_channels * (bits_per_sample // 8),
num_channels * (bits_per_sample // 8),
bits_per_sample,
)
)
buf.write(b"data")
buf.write(struct.pack("<I", data_size))
buf.write(audio_np.tobytes())
audio_b64 = base64.b64encode(buf.getvalue()).decode()
audio_items.append(
{
"type": "audio_url",
"audio_url": {"url": f"data:audio/wav;base64,{audio_b64}"},
}
)
new_content = list(content) + audio_items
if isinstance(msg, dict):
new_msg = {**msg, "content": new_content}
else:
new_msg = msg.model_copy(update={"content": new_content})
new_messages.append(new_msg)
return new_messages
def _to_sampling_params_list(self, sampling_params_list: list[dict]) -> list[Any]:
"""Convert request dicts to stage-typed sampling params objects.
For diffusion stages, build ``OmniDiffusionSamplingParams`` so
downstream ``StageDiffusionClient._sampling_params_to_dict`` (which
requires a dataclass) works. For LLM stages build ``SamplingParams``.
If callers provide params for fewer stages than the native pipeline has
(for example AURA has three semantic models but four engine stages),
append cloned deploy defaults for the omitted tail stages.
"""
stage_configs = list(getattr(self.engine_client, "stage_configs", []) or [])
default_params_list = list(getattr(self.engine_client, "default_sampling_params_list", []) or [])
final_sampling_params_list: list[Any] = []
for idx, sampling_params in enumerate(sampling_params_list):
stage_type = get_stage_type(stage_configs[idx]) if idx < len(stage_configs) else "llm"
@@ -750,6 +1013,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
final_sampling_params_list.append(target_cls(**as_dict))
else:
raise ValueError(f"Invalid sampling params: {sampling_params}")
for idx in range(len(final_sampling_params_list), len(stage_configs)):
if idx < len(default_params_list):
final_sampling_params_list.append(clone_sampling_params(default_params_list[idx]))
else:
final_sampling_params_list.append(SamplingParams())
return final_sampling_params_list
def _get_comprehension_stage_index(self) -> int: