[Frontend][Model] Support batch request with refined OmniDiffusionReq… (#797)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-01-28 16:07:42 +08:00
1 parent 9ece725fc5
commit d64bbde561
64 files changed
+2995 -1414

No files matched your search

+11 -10
View File
@@ -7,11 +7,12 @@ Pytest configuration and fixtures for vllm-omni tests.
from typing import Any
import pytest
from vllm import TextPrompt
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm.sampling_params import SamplingParams
from tests.conftest import _run_post_test_cleanup
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
@@ -64,7 +65,7 @@ class OmniRunner:
**kwargs,
)
def get_default_sampling_params_list(self) -> list[SamplingParams]:
def get_default_sampling_params_list(self) -> list[OmniSamplingParams]:
"""
Get a list of default sampling parameters for all stages.
@@ -82,7 +83,7 @@ class OmniRunner:
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
modalities: list[str] | None = None,
) -> list[dict[str, Any]]:
) -> list[TextPrompt]:
"""
Construct Omni input format from prompts and multimodal data.
@@ -175,7 +176,7 @@ class OmniRunner:
f"<|im_start|>assistant\n"
)
input_dict: dict[str, Any] = {"prompt": full_prompt}
input_dict: TextPrompt = {"prompt": full_prompt}
if multi_modal_data:
input_dict["multi_modal_data"] = multi_modal_data
if modalities:
@@ -189,8 +190,8 @@ class OmniRunner:
def generate(
self,
prompts: list[dict[str, Any]],
sampling_params_list: list[SamplingParams] | None = None,
prompts: list[TextPrompt],
sampling_params_list: list[OmniSamplingParams] | None = None,
) -> list[OmniRequestOutput]:
"""
Generate outputs for the given prompts.
@@ -212,7 +213,7 @@ class OmniRunner:
def generate_multimodal(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
images: PromptImageInput = None,
@@ -249,7 +250,7 @@ class OmniRunner:
def generate_audio(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -276,7 +277,7 @@ class OmniRunner:
def generate_video(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -303,7 +304,7 @@ class OmniRunner:
def generate_image(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
images: PromptImageInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
+10 -6
View File
@@ -15,6 +15,8 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -55,12 +57,14 @@ def test_cache_dit(model_name: str):
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
),
)
# Extract images from request_output[0]['images']
first_output = outputs[0]
@@ -6,6 +6,7 @@ import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.utils import GPUMemoryMonitor
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.utils.platform_utils import is_npu, is_rocm
# ruff: noqa: E402
@@ -30,11 +31,13 @@ def inference(model_name: str, offload: bool = True):
m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=9,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=9,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
),
)
peak = monitor.peak_used_mb
monitor.stop()
@@ -7,6 +7,7 @@ import pytest
import torch
from safetensors.torch import save_file
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.utils.platform_utils import is_npu
@@ -88,12 +89,14 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
outputs = m.generate(
prompt,
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
),
)
images = _extract_images(outputs)
@@ -116,14 +119,16 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
)
outputs_lora = m.generate(
prompt,
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
lora_request=lora_request,
lora_scale=2.0,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
lora_request=lora_request,
lora_scale=2.0,
),
)
images_lora = _extract_images(outputs_lora)
assert len(images_lora) == 1
+25 -14
View File
@@ -25,6 +25,7 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image import OvisImagePipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@pytest.fixture
@@ -162,11 +163,13 @@ def test_basic_generation(ovis_pipeline):
"""Test the forward pass logic."""
# Setup request
req = OmniDiffusionRequest(
prompt="A photo of a cat",
height=256,
width=256,
num_inference_steps=2,
guidance_scale=1.0,
prompts=["A photo of a cat"],
sampling_params=OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=2,
guidance_scale=1.0,
),
)
output = ovis_pipeline(req)
@@ -184,12 +187,18 @@ def test_basic_generation(ovis_pipeline):
def test_guidance_scale(ovis_pipeline):
"""Test that classifier-free guidance path is taken when scale > 1.0."""
req = OmniDiffusionRequest(
prompt="A photo of a cat",
negative_prompt="bad quality",
height=256,
width=256,
num_inference_steps=1,
guidance_scale=2.0, # Trigger CFG
prompts=[
{
"prompt": "A photo of a cat",
"negative_prompt": "bad quality",
}
],
sampling_params=OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=1,
guidance_scale=2.0, # Trigger CFG
),
)
ovis_pipeline(req)
@@ -200,9 +209,11 @@ def test_resolution_check(ovis_pipeline):
"""Test resolution divisible validation logic if present."""
# Pass odd resolution
req = OmniDiffusionRequest(
prompt="test",
height=250, # Not divisible by 16 (8*2)
width=250,
prompts=["test"],
sampling_params=OmniDiffusionSamplingParams(
height=250, # Not divisible by 16 (8*2)
width=250,
),
)
# Should warn but proceed (as per code I read earlier) or resize?
@@ -21,6 +21,8 @@ import torch
import torch.distributed as dist
from PIL import Image
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
@@ -81,12 +83,14 @@ def _run_baseline(model_name: str, dtype: torch.dtype, attn_backend: str, height
try:
outputs = baseline.generate(
PROMPT,
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
return outputs[0].request_output[0].images
finally:
@@ -116,12 +120,14 @@ def _run_sp(
try:
outputs = sp.generate(
PROMPT,
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
return outputs[0].request_output[0].images
finally:
@@ -5,6 +5,7 @@ import numpy as np
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
# ruff: noqa: E402
@@ -29,16 +30,20 @@ def test_stable_audio_model(model_name: str):
sample_rate = 44100 # Stable Audio uses 44100 Hz
outputs = m.generate(
"The sound of a dog barking",
negative_prompt="Low quality.",
num_inference_steps=4, # Minimal steps for speed
guidance_scale=7.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
extra={
"audio_start_in_s": audio_start_in_s,
"audio_end_in_s": audio_end_in_s,
prompts={
"prompt": "The sound of a dog barking",
"negative_prompt": "Low quality.",
},
sampling_params_list=OmniDiffusionSamplingParams(
num_inference_steps=4, # Minimal steps for speed
guidance_scale=7.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
extra_args={
"audio_start_in_s": audio_start_in_s,
"audio_end_in_s": audio_end_in_s,
},
),
)
# Extract audio from OmniRequestOutput
@@ -5,6 +5,7 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.utils.platform_utils import is_npu, is_rocm
@@ -42,12 +43,14 @@ def test_diffusion_model(model_name: str):
width = 256
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=2,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=2,
),
)
# Extract images from request_output[0]['images']
first_output = outputs[0]
+11 -7
View File
@@ -5,6 +5,8 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -33,13 +35,15 @@ def test_video_diffusion_model(model_name: str):
width = 640
num_frames = 5
outputs = m.generate(
"A cat sitting on a table",
height=height,
width=width,
num_frames=num_frames,
num_inference_steps=2,
guidance_scale=1.0,
generator=torch.Generator("cuda").manual_seed(42),
prompts="A cat sitting on a table",
sampling_params_list=OmniDiffusionSamplingParams(
height=height,
width=width,
num_frames=num_frames,
num_inference_steps=2,
guidance_scale=1.0,
generator=torch.Generator("cuda").manual_seed(42),
),
)
first_output = outputs[0]
assert first_output.final_output_type == "image"
+10 -6
View File
@@ -15,6 +15,8 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -51,12 +53,14 @@ def test_teacache(model_name: str):
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
),
)
# Extract images from request_output[0]['images']
first_output = outputs[0]
@@ -12,6 +12,8 @@ import torch
from PIL import Image
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -90,12 +92,14 @@ def _run_zimage_generate(
num_requests = 4 # 1 warmup + 3 timed
gen = m.generate(
[PROMPT] * num_requests,
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
seed=seed,
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
seed=seed,
num_outputs_per_prompt=1,
),
py_generator=True,
)
+10 -9
View File
@@ -5,6 +5,7 @@ from contextlib import ExitStack
from pathlib import Path
import pytest
from vllm import SamplingParams
from vllm.inputs import PromptType
from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState
@@ -25,15 +26,15 @@ async def generate(
) -> tuple[int, str]:
# Ensure generate doesn't complete too fast for cancellation test.
await asyncio.sleep(0.2)
thinker_sampling_params = {
"temperature": 0.4, # Deterministic
"top_p": 0.9,
"top_k": 1,
"max_tokens": max_tokens,
"repetition_penalty": 1.05,
"stop_token_ids": [151645], # Qwen EOS token <|im_end|>
"seed": SEED,
}
thinker_sampling_params = SamplingParams(
temperature=0.4, # Deterministic
top_p=0.9,
top_k=1,
max_tokens=max_tokens,
repetition_penalty=1.05,
stop_token_ids=[151645], # Qwen EOS token <|im_end|>
seed=SEED,
)
sampling_params_list = [
thinker_sampling_params,