Merge branch 'main' into dev/vllm-align

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-01-30 02:53:03 +00:00
commit 4b895459e4
196 files changed
+9406 -4132

No files matched your search

+39 -28
View File
@@ -6,13 +6,17 @@ import uuid
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
from datetime import datetime, timedelta, timezone
from io import BytesIO
from typing import TYPE_CHECKING, Any, Final, Optional
from typing import TYPE_CHECKING, Any, Final, Optional, cast
import jinja2
import torch
from fastapi import Request
from PIL import Image
from pydantic import TypeAdapter
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
try:
import soundfile
except ImportError:
@@ -282,16 +286,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
lora_request=lora_request,
)
trace_headers = None if raw_request is None else await self._get_trace_headers(raw_request.headers)
generator = self.engine_client.generate(
prompt=engine_prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
output_modalities=output_modalities,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
)
generators.append(generator)
@@ -1678,10 +1677,13 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
):
choices: list[ChatCompletionResponseChoice] = []
final_res = omni_outputs.request_output
audio_data = final_res.multimodal_output.get("audio")
if stream:
audio_tensor = final_res.multimodal_output["audio"][-1].float().detach().cpu().numpy()
audio_tensor = audio_data[-1].float().detach().cpu().numpy()
else:
audio_tensor = final_res.multimodal_output["audio"].float().detach().cpu().numpy()
if isinstance(audio_data, list):
audio_data = torch.cat(audio_data, dim=-1)
audio_tensor = audio_data.float().detach().cpu().numpy()
# Ensure audio is 1D (flatten if needed)
if audio_tensor.ndim > 1:
@@ -1924,29 +1926,30 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
logger.warning("Failed to decode reference image: %s", e)
# Build generation kwargs
gen_kwargs: dict[str, Any] = {
gen_prompt: OmniTextPrompt = {
"prompt": prompt,
"request_id": request_id,
"num_inference_steps": num_inference_steps,
"height": height,
"width": width,
"negative_prompt": negative_prompt,
"num_outputs_per_prompt": num_outputs_per_prompt,
"seed": seed,
}
gen_params = OmniDiffusionSamplingParams(
num_inference_steps=num_inference_steps,
height=height,
width=width,
num_outputs_per_prompt=num_outputs_per_prompt,
seed=seed,
)
if guidance_scale is not None:
gen_kwargs["guidance_scale"] = guidance_scale
gen_params.guidance_scale = guidance_scale
# Add Qwen-Image specific parameter
if true_cfg_scale is not None:
gen_kwargs["true_cfg_scale"] = true_cfg_scale
gen_params.true_cfg_scale = true_cfg_scale
# Add video generation parameters if set
if num_frames is not None:
gen_kwargs["num_frames"] = num_frames
gen_params.num_frames = num_frames
if guidance_scale_2 is not None:
gen_kwargs["guidance_scale_2"] = guidance_scale_2
gen_params.guidance_scale_2 = guidance_scale_2
# Parse per-request LoRA (works for both AsyncOmniDiffusion and AsyncOmni).
if lora_body and isinstance(lora_body, dict):
@@ -1969,16 +1972,17 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
lora_int_id = stable_lora_int_id(str(lora_path))
if lora_name and lora_path:
lora_req = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path))
gen_kwargs["lora_request"] = lora_req
gen_params.lora_request = lora_req
if lora_scale is not None:
gen_kwargs["lora_scale"] = float(lora_scale)
gen_params.lora_scale = float(lora_scale)
except Exception as e: # pragma: no cover - safeguard
logger.warning("Failed to parse LoRA request: %s", e)
# Add reference image if provided
if pil_images:
if len(pil_images) == 1:
gen_kwargs["pil_image"] = pil_images[0]
gen_prompt["multi_modal_data"] = {}
gen_prompt["multi_modal_data"]["image"] = pil_images[0]
else:
od_config = getattr(self._diffusion_engine, "od_config", None)
supports_multimodal_inputs = getattr(od_config, "supports_multimodal_inputs", False)
@@ -1986,7 +1990,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
# TODO: entry is asyncOmni. We hack the od config here.
supports_multimodal_inputs = True
if supports_multimodal_inputs:
gen_kwargs["pil_image"] = pil_images
gen_prompt["multi_modal_data"] = {}
gen_prompt["multi_modal_data"]["image"] = pil_images
else:
return self._create_error_response(
"Multiple input images are not supported by the current diffusion model. "
@@ -1999,18 +2004,24 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
# Handle both AsyncOmniDiffusion (returns OmniRequestOutput) and AsyncOmni (returns AsyncGenerator)
if hasattr(self._diffusion_engine, "stage_list"):
# AsyncOmni: iterate through async generator to get final output
diffusion_engine = cast(AsyncOmni, self._diffusion_engine)
result = None
async for output in self._diffusion_engine.generate(
prompt=gen_kwargs["prompt"],
request_id=gen_kwargs.get("request_id"),
sampling_params_list=[gen_kwargs], # Pass as single-stage params
async for output in diffusion_engine.generate(
prompt=gen_prompt,
sampling_params_list=[gen_params], # Pass as single-stage params
request_id=request_id,
):
result = output
if result is None:
return self._create_error_response("No output generated from AsyncOmni")
else:
# AsyncOmniDiffusion: direct call
result = await self._diffusion_engine.generate(**gen_kwargs)
diffusion_engine = cast(AsyncOmniDiffusion, self._diffusion_engine)
result = await diffusion_engine.generate(
prompt=gen_prompt,
sampling_params=gen_params,
request_id=request_id,
)
# Extract images from result
# Handle nested OmniRequestOutput structure where images might be in request_output
images = getattr(result.request_output, "images", [])