[Model] Aura support - Non async chunk path (#4257)
Signed-off-by: Rein Yang <ruiruyang2@gmail.com> Signed-off-by: R2-Y <ruiruyang2@gmail.com>
This commit is contained in:
1 parent
1292c47804
commit
afbcc51672
30 files changed
+2958
-26
No files matched your search
@@ -74,6 +74,7 @@ from vllm.entrypoints.serve.utils.api_utils import should_include_usage
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import maybe_filter_parallel_tool_calls
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.media.connector import MediaConnector
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
@@ -104,6 +105,7 @@ from vllm_omni.entrypoints.openai.protocol.images import (
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.stage_params import (
|
||||
build_stage_sampling_params_list,
|
||||
clone_sampling_params,
|
||||
get_default_sampling_params_list,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.utils import (
|
||||
@@ -122,6 +124,10 @@ from vllm_omni.utils.audio import audio_chunk_pcm_bytes, audio_chunk_sample_rate
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
async def _identity_async(value: Any) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
"""OpenAI-compatible chat serving for both LLM and Diffusion models.
|
||||
|
||||
@@ -654,6 +660,13 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
)
|
||||
|
||||
deferred_multi_modal_data: dict[str, Any] | None = None
|
||||
if self._needs_multistage_multimodal_split():
|
||||
messages, deferred_multi_modal_data = await self._prepare_multistage_multimodal_inputs(
|
||||
messages,
|
||||
request,
|
||||
)
|
||||
|
||||
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
chat_params,
|
||||
@@ -698,18 +711,25 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
if hasattr(request, "cache_salt") and request.cache_salt is not None:
|
||||
engine_prompt["cache_salt"] = request.cache_salt
|
||||
|
||||
additional_information = getattr(request, "additional_information", None)
|
||||
if isinstance(additional_information, dict):
|
||||
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
|
||||
prompt_additional_information.update(additional_information)
|
||||
|
||||
if deferred_multi_modal_data:
|
||||
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
|
||||
prompt_additional_information["deferred_multi_modal_data"] = deferred_multi_modal_data
|
||||
|
||||
speaker = getattr(request, "voice", None) or getattr(request, "speaker", None)
|
||||
normalized = validate_requested_speaker(speaker, self._get_supported_speakers())
|
||||
if normalized is not None:
|
||||
if "additional_information" not in engine_prompt or engine_prompt["additional_information"] is None:
|
||||
engine_prompt["additional_information"] = {}
|
||||
engine_prompt["additional_information"]["speaker"] = [normalized]
|
||||
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
|
||||
prompt_additional_information["speaker"] = [normalized]
|
||||
|
||||
language = getattr(request, "language", None)
|
||||
if language is not None and isinstance(language, str) and language.strip():
|
||||
if "additional_information" not in engine_prompt or engine_prompt["additional_information"] is None:
|
||||
engine_prompt["additional_information"] = {}
|
||||
engine_prompt["additional_information"]["language"] = [language.strip()]
|
||||
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
|
||||
prompt_additional_information["language"] = [language.strip()]
|
||||
|
||||
# Style instruction — used by Ming-flash-omni instruct TTS path
|
||||
# (ming_task="instruct"). For the omni speech path the thinker2talker
|
||||
@@ -717,20 +737,263 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
# which hardcodes instruction=None.
|
||||
instructions = getattr(request, "instructions", None)
|
||||
if instructions is not None and isinstance(instructions, str) and instructions.strip():
|
||||
if "additional_information" not in engine_prompt or engine_prompt["additional_information"] is None:
|
||||
engine_prompt["additional_information"] = {}
|
||||
engine_prompt["additional_information"]["instruction"] = instructions.strip()
|
||||
prompt_additional_information = self._ensure_prompt_additional_information(engine_prompt)
|
||||
prompt_additional_information["instruction"] = instructions.strip()
|
||||
|
||||
return conversation, [engine_prompt]
|
||||
|
||||
@staticmethod
|
||||
def _ensure_prompt_additional_information(engine_prompt: dict[str, Any]) -> dict[str, Any]:
|
||||
additional_information = engine_prompt.get("additional_information")
|
||||
if not isinstance(additional_information, dict):
|
||||
additional_information = {}
|
||||
engine_prompt["additional_information"] = additional_information
|
||||
return additional_information
|
||||
|
||||
def _needs_multistage_multimodal_split(self) -> bool:
|
||||
return bool(self._deferred_multimodal_modalities())
|
||||
|
||||
def _deferred_multimodal_modalities(self) -> set[str]:
|
||||
stage_configs = list(getattr(self.engine_client, "stage_configs", []) or [])
|
||||
if len(stage_configs) < 2:
|
||||
return set()
|
||||
|
||||
first_stage_modalities = self._stage_input_modalities(stage_configs[0])
|
||||
if not first_stage_modalities:
|
||||
return set()
|
||||
|
||||
downstream_modalities: set[str] = set()
|
||||
for stage in stage_configs[1:]:
|
||||
downstream_modalities.update(self._stage_input_modalities(stage))
|
||||
return downstream_modalities - first_stage_modalities
|
||||
|
||||
@staticmethod
|
||||
def _stage_input_modalities(stage: Any) -> set[str]:
|
||||
engine_args = getattr(stage, "engine_args", None)
|
||||
explicit = (
|
||||
getattr(stage, "input_modalities", None)
|
||||
or getattr(stage, "modalities", None)
|
||||
or getattr(engine_args, "input_modalities", None)
|
||||
or getattr(engine_args, "modalities", None)
|
||||
)
|
||||
if explicit:
|
||||
return {str(modality) for modality in as_list(explicit)}
|
||||
|
||||
model_stage = str(getattr(engine_args, "model_stage", None) or getattr(stage, "model_stage", "")).lower()
|
||||
if model_stage in {"asr", "stt"} or model_stage.endswith("_asr"):
|
||||
return {"audio"}
|
||||
if any(name in model_stage for name in ("vision", "vl", "aura")):
|
||||
return {"image", "video"}
|
||||
if any(name in model_stage for name in ("tts", "talker", "code2wav")):
|
||||
return set()
|
||||
|
||||
if getattr(stage, "requires_multimodal_data", False):
|
||||
return {"audio", "image", "video"}
|
||||
return set()
|
||||
|
||||
async def _prepare_multistage_multimodal_inputs(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
request: ChatLikeRequest | ResponsesRequest,
|
||||
) -> tuple[list[ChatCompletionMessageParam], dict[str, Any] | None]:
|
||||
"""Hide modalities unsupported by stage 0 and stash them for downstream stages."""
|
||||
deferred_modalities = self._deferred_multimodal_modalities()
|
||||
deferred_parts: dict[str, list[Any]] = {modality: [] for modality in deferred_modalities}
|
||||
stripped_messages: list[ChatCompletionMessageParam] = []
|
||||
|
||||
for message in messages:
|
||||
content = message.get("content") if isinstance(message, dict) else None
|
||||
if not isinstance(content, list):
|
||||
stripped_messages.append(message)
|
||||
continue
|
||||
|
||||
stripped_content: list[Any] = []
|
||||
changed = False
|
||||
for part in content:
|
||||
modality, payload = self._deferred_multimodal_part(part, deferred_modalities)
|
||||
if modality is not None:
|
||||
if payload is not None:
|
||||
deferred_parts.setdefault(modality, []).append(payload)
|
||||
changed = True
|
||||
continue
|
||||
stripped_content.append(part)
|
||||
|
||||
if changed:
|
||||
stripped_message = dict(message)
|
||||
stripped_message["content"] = stripped_content
|
||||
stripped_messages.append(cast(ChatCompletionMessageParam, stripped_message))
|
||||
else:
|
||||
stripped_messages.append(message)
|
||||
|
||||
deferred_parts = {modality: parts for modality, parts in deferred_parts.items() if parts}
|
||||
if not deferred_parts:
|
||||
return messages, None
|
||||
|
||||
media_connector = MediaConnector(
|
||||
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||
allowed_local_media_path=getattr(self.model_config, "allowed_local_media_path", "") or "",
|
||||
allowed_media_domains=getattr(self.model_config, "allowed_media_domains", None),
|
||||
)
|
||||
multi_modal_data: dict[str, Any] = {}
|
||||
for modality, parts in deferred_parts.items():
|
||||
multi_modal_data[modality] = await self._materialize_deferred_multimodal_parts(
|
||||
media_connector,
|
||||
modality,
|
||||
parts,
|
||||
)
|
||||
return stripped_messages, multi_modal_data
|
||||
|
||||
@staticmethod
|
||||
def _deferred_multimodal_part(part: Any, deferred_modalities: set[str]) -> tuple[str | None, Any | None]:
|
||||
if not isinstance(part, dict):
|
||||
return None, None
|
||||
part_type = part.get("type")
|
||||
if part_type in {"video_url", "video"} and "video" in deferred_modalities:
|
||||
video = part.get("video_url", part.get("video"))
|
||||
if isinstance(video, dict):
|
||||
video = video.get("url")
|
||||
return "video", video
|
||||
if part_type in {"image_url", "image_pil", "image"} and "image" in deferred_modalities:
|
||||
image = part.get("image_pil", part.get("image_url", part.get("image")))
|
||||
if isinstance(image, dict):
|
||||
image = image.get("url")
|
||||
return "image", image
|
||||
if part_type in {"audio_url", "input_audio", "audio"} and "audio" in deferred_modalities:
|
||||
audio = part.get("audio_url", part.get("input_audio", part.get("audio")))
|
||||
if isinstance(audio, dict):
|
||||
audio = audio.get("url") or audio.get("data")
|
||||
return "audio", audio
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
async def _materialize_deferred_multimodal_parts(
|
||||
media_connector: MediaConnector,
|
||||
modality: str,
|
||||
parts: list[Any],
|
||||
) -> list[Any]:
|
||||
if modality == "video":
|
||||
return list(await asyncio.gather(*(media_connector.fetch_video_async(part) for part in parts)))
|
||||
if modality == "image":
|
||||
fetch_image = getattr(media_connector, "fetch_image_async", None)
|
||||
if fetch_image is None:
|
||||
return parts
|
||||
return list(
|
||||
await asyncio.gather(
|
||||
*(fetch_image(part) if isinstance(part, str) else _identity_async(part) for part in parts)
|
||||
)
|
||||
)
|
||||
if modality == "audio":
|
||||
fetch_audio = getattr(media_connector, "fetch_audio_async", None)
|
||||
if fetch_audio is None:
|
||||
return parts
|
||||
return list(
|
||||
await asyncio.gather(
|
||||
*(fetch_audio(part) if isinstance(part, str) else _identity_async(part) for part in parts)
|
||||
)
|
||||
)
|
||||
return parts
|
||||
|
||||
async def _inject_audio_from_video_urls(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
) -> list[ChatCompletionMessageParam]:
|
||||
"""Pre-extract audio from video URLs and inject as audio_url content items.
|
||||
|
||||
When use_audio_in_video=True, the qwen2_5_omni_thinker multimodal
|
||||
processor requires that the number of audio items equals the number of
|
||||
video items (it subtracts mm_counts["video"] from mm_counts["audio"]).
|
||||
The client only sends video_url items; this method adds the matching
|
||||
audio_url items on the server side before the renderer processes them.
|
||||
"""
|
||||
import io
|
||||
|
||||
from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async
|
||||
|
||||
new_messages: list[ChatCompletionMessageParam] = []
|
||||
for msg in messages:
|
||||
content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None)
|
||||
if not isinstance(content, list):
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
|
||||
video_urls = [
|
||||
part.get("video_url", {}).get("url")
|
||||
for part in content
|
||||
if isinstance(part, dict) and part.get("type") == "video_url" and part.get("video_url", {}).get("url")
|
||||
]
|
||||
|
||||
if not video_urls:
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
|
||||
audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls))
|
||||
|
||||
audio_items: list[dict] = []
|
||||
for audio_array, sample_rate in audios:
|
||||
buf = io.BytesIO()
|
||||
if soundfile is not None:
|
||||
soundfile.write(buf, audio_array, samplerate=int(sample_rate), format="WAV")
|
||||
else:
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
|
||||
audio_np = np.asarray(audio_array, dtype=np.float32)
|
||||
sr = int(sample_rate)
|
||||
num_channels = 1
|
||||
bits_per_sample = 32
|
||||
num_frames = len(audio_np)
|
||||
data_size = num_frames * num_channels * (bits_per_sample // 8)
|
||||
# Write minimal RIFF/WAV header
|
||||
buf.write(b"RIFF")
|
||||
buf.write(struct.pack("<I", 36 + data_size))
|
||||
buf.write(b"WAVE")
|
||||
buf.write(b"fmt ")
|
||||
buf.write(
|
||||
struct.pack(
|
||||
"<IHHIIHH",
|
||||
16,
|
||||
3,
|
||||
num_channels,
|
||||
sr,
|
||||
sr * num_channels * (bits_per_sample // 8),
|
||||
num_channels * (bits_per_sample // 8),
|
||||
bits_per_sample,
|
||||
)
|
||||
)
|
||||
buf.write(b"data")
|
||||
buf.write(struct.pack("<I", data_size))
|
||||
buf.write(audio_np.tobytes())
|
||||
|
||||
audio_b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
audio_items.append(
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": f"data:audio/wav;base64,{audio_b64}"},
|
||||
}
|
||||
)
|
||||
|
||||
new_content = list(content) + audio_items
|
||||
if isinstance(msg, dict):
|
||||
new_msg = {**msg, "content": new_content}
|
||||
else:
|
||||
new_msg = msg.model_copy(update={"content": new_content})
|
||||
new_messages.append(new_msg)
|
||||
|
||||
return new_messages
|
||||
|
||||
def _to_sampling_params_list(self, sampling_params_list: list[dict]) -> list[Any]:
|
||||
"""Convert request dicts to stage-typed sampling params objects.
|
||||
|
||||
For diffusion stages, build ``OmniDiffusionSamplingParams`` so
|
||||
downstream ``StageDiffusionClient._sampling_params_to_dict`` (which
|
||||
requires a dataclass) works. For LLM stages build ``SamplingParams``.
|
||||
If callers provide params for fewer stages than the native pipeline has
|
||||
(for example AURA has three semantic models but four engine stages),
|
||||
append cloned deploy defaults for the omitted tail stages.
|
||||
"""
|
||||
stage_configs = list(getattr(self.engine_client, "stage_configs", []) or [])
|
||||
default_params_list = list(getattr(self.engine_client, "default_sampling_params_list", []) or [])
|
||||
final_sampling_params_list: list[Any] = []
|
||||
for idx, sampling_params in enumerate(sampling_params_list):
|
||||
stage_type = get_stage_type(stage_configs[idx]) if idx < len(stage_configs) else "llm"
|
||||
@@ -750,6 +1013,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
final_sampling_params_list.append(target_cls(**as_dict))
|
||||
else:
|
||||
raise ValueError(f"Invalid sampling params: {sampling_params}")
|
||||
for idx in range(len(final_sampling_params_list), len(stage_configs)):
|
||||
if idx < len(default_params_list):
|
||||
final_sampling_params_list.append(clone_sampling_params(default_params_list[idx]))
|
||||
else:
|
||||
final_sampling_params_list.append(SamplingParams())
|
||||
return final_sampling_params_list
|
||||
|
||||
def _get_comprehension_stage_index(self) -> int:
|
||||
|
||||
Reference in new issue
Block a user