[Frontend][Model] Support batch request with refined OmniDiffusionReq… (#797)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-01-28 16:07:42 +08:00
1 parent 9ece725fc5
commit d64bbde561
64 files changed
+2995 -1414

No files matched your search

@@ -1,10 +1,12 @@
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from vllm_omni.diffusion.models.bagel.pipeline_bagel import BagelPipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
@@ -109,11 +111,12 @@ class TestKVFlow(unittest.TestCase):
transfer_data.metadata = {"kv_lens": [self.seq_len], "ropes": [0]}
# 2. Setup Request with Injected Data
req = OmniDiffusionRequest(prompt="test")
from types import SimpleNamespace
sp = OmniDiffusionSamplingParams(
past_key_values=SimpleNamespace(**layer_blocks),
kv_metadata=transfer_data.metadata,
)
req.past_key_values = SimpleNamespace(**layer_blocks)
req.kv_metadata = transfer_data.metadata
req = OmniDiffusionRequest(["test"], sp)
# 3. Setup Pipeline
pipeline = MockBagelPipeline()
@@ -144,7 +147,7 @@ class TestKVFlow(unittest.TestCase):
current_cache = RealNaiveCache(self.num_layers)
# --- Logic from Source Code ---
injected_kv = req.past_key_values
injected_kv = req.sampling_params.past_key_values
if isinstance(current_cache, RealNaiveCache) and hasattr(injected_kv, "key_cache"):
# Assuming injected_kv is SimpleNamespace or object with list attrs
for layer_idx in range(len(injected_kv.key_cache)):
@@ -176,9 +179,11 @@ class TestKVFlow(unittest.TestCase):
data_dict = runner_test_result.to_dict()
# 3. Receiver (Request Setup)
req = OmniDiffusionRequest(prompt="integration_test")
req.past_key_values = data_dict["layer_blocks"]
req.kv_metadata = data_dict["metadata"]
sp = OmniDiffusionSamplingParams(
past_key_values=data_dict["layer_blocks"],
kv_metadata=data_dict["metadata"],
)
req = OmniDiffusionRequest(["integration_test"], sp) # noqa: F841
# 4. Receiver (Injection Simulation)
# Use the logic verification again
+11 -10
View File
@@ -7,11 +7,12 @@ Pytest configuration and fixtures for vllm-omni tests.
from typing import Any
import pytest
from vllm import TextPrompt
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm.sampling_params import SamplingParams
from tests.conftest import _run_post_test_cleanup
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniSamplingParams
from vllm_omni.outputs import OmniRequestOutput
PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
@@ -64,7 +65,7 @@ class OmniRunner:
**kwargs,
)
def get_default_sampling_params_list(self) -> list[SamplingParams]:
def get_default_sampling_params_list(self) -> list[OmniSamplingParams]:
"""
Get a list of default sampling parameters for all stages.
@@ -82,7 +83,7 @@ class OmniRunner:
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
modalities: list[str] | None = None,
) -> list[dict[str, Any]]:
) -> list[TextPrompt]:
"""
Construct Omni input format from prompts and multimodal data.
@@ -175,7 +176,7 @@ class OmniRunner:
f"<|im_start|>assistant\n"
)
input_dict: dict[str, Any] = {"prompt": full_prompt}
input_dict: TextPrompt = {"prompt": full_prompt}
if multi_modal_data:
input_dict["multi_modal_data"] = multi_modal_data
if modalities:
@@ -189,8 +190,8 @@ class OmniRunner:
def generate(
self,
prompts: list[dict[str, Any]],
sampling_params_list: list[SamplingParams] | None = None,
prompts: list[TextPrompt],
sampling_params_list: list[OmniSamplingParams] | None = None,
) -> list[OmniRequestOutput]:
"""
Generate outputs for the given prompts.
@@ -212,7 +213,7 @@ class OmniRunner:
def generate_multimodal(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
images: PromptImageInput = None,
@@ -249,7 +250,7 @@ class OmniRunner:
def generate_audio(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
audios: PromptAudioInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -276,7 +277,7 @@ class OmniRunner:
def generate_video(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
videos: PromptVideoInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
@@ -303,7 +304,7 @@ class OmniRunner:
def generate_image(
self,
prompts: list[str] | str,
sampling_params_list: list[SamplingParams] | None = None,
sampling_params_list: list[OmniSamplingParams] | None = None,
system_prompt: str | None = None,
images: PromptImageInput = None,
mm_processor_kwargs: dict[str, Any] | None = None,
+10 -6
View File
@@ -15,6 +15,8 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -55,12 +57,14 @@ def test_cache_dit(model_name: str):
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
),
)
# Extract images from request_output[0]['images']
first_output = outputs[0]
@@ -6,6 +6,7 @@ import torch
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from tests.utils import GPUMemoryMonitor
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.utils.platform_utils import is_npu, is_rocm
# ruff: noqa: E402
@@ -30,11 +31,13 @@ def inference(model_name: str, offload: bool = True):
m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=9,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=9,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
),
)
peak = monitor.peak_used_mb
monitor.stop()
@@ -7,6 +7,7 @@ import pytest
import torch
from safetensors.torch import save_file
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.utils.platform_utils import is_npu
@@ -88,12 +89,14 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
outputs = m.generate(
prompt,
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
),
)
images = _extract_images(outputs)
@@ -116,14 +119,16 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
)
outputs_lora = m.generate(
prompt,
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
lora_request=lora_request,
lora_scale=2.0,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
lora_request=lora_request,
lora_scale=2.0,
),
)
images_lora = _extract_images(outputs_lora)
assert len(images_lora) == 1
+25 -14
View File
@@ -25,6 +25,7 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image import OvisImagePipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@pytest.fixture
@@ -162,11 +163,13 @@ def test_basic_generation(ovis_pipeline):
"""Test the forward pass logic."""
# Setup request
req = OmniDiffusionRequest(
prompt="A photo of a cat",
height=256,
width=256,
num_inference_steps=2,
guidance_scale=1.0,
prompts=["A photo of a cat"],
sampling_params=OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=2,
guidance_scale=1.0,
),
)
output = ovis_pipeline(req)
@@ -184,12 +187,18 @@ def test_basic_generation(ovis_pipeline):
def test_guidance_scale(ovis_pipeline):
"""Test that classifier-free guidance path is taken when scale > 1.0."""
req = OmniDiffusionRequest(
prompt="A photo of a cat",
negative_prompt="bad quality",
height=256,
width=256,
num_inference_steps=1,
guidance_scale=2.0, # Trigger CFG
prompts=[
{
"prompt": "A photo of a cat",
"negative_prompt": "bad quality",
}
],
sampling_params=OmniDiffusionSamplingParams(
height=256,
width=256,
num_inference_steps=1,
guidance_scale=2.0, # Trigger CFG
),
)
ovis_pipeline(req)
@@ -200,9 +209,11 @@ def test_resolution_check(ovis_pipeline):
"""Test resolution divisible validation logic if present."""
# Pass odd resolution
req = OmniDiffusionRequest(
prompt="test",
height=250, # Not divisible by 16 (8*2)
width=250,
prompts=["test"],
sampling_params=OmniDiffusionSamplingParams(
height=250, # Not divisible by 16 (8*2)
width=250,
),
)
# Should warn but proceed (as per code I read earlier) or resize?
@@ -21,6 +21,8 @@ import torch
import torch.distributed as dist
from PIL import Image
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
@@ -81,12 +83,14 @@ def _run_baseline(model_name: str, dtype: torch.dtype, attn_backend: str, height
try:
outputs = baseline.generate(
PROMPT,
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
return outputs[0].request_output[0].images
finally:
@@ -116,12 +120,14 @@ def _run_sp(
try:
outputs = sp.generate(
PROMPT,
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=4,
guidance_scale=0.0,
generator=torch.Generator(get_device_name()).manual_seed(seed),
num_outputs_per_prompt=1,
),
)
return outputs[0].request_output[0].images
finally:
@@ -5,6 +5,7 @@ import numpy as np
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
# ruff: noqa: E402
@@ -29,16 +30,20 @@ def test_stable_audio_model(model_name: str):
sample_rate = 44100 # Stable Audio uses 44100 Hz
outputs = m.generate(
"The sound of a dog barking",
negative_prompt="Low quality.",
num_inference_steps=4, # Minimal steps for speed
guidance_scale=7.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
extra={
"audio_start_in_s": audio_start_in_s,
"audio_end_in_s": audio_end_in_s,
prompts={
"prompt": "The sound of a dog barking",
"negative_prompt": "Low quality.",
},
sampling_params_list=OmniDiffusionSamplingParams(
num_inference_steps=4, # Minimal steps for speed
guidance_scale=7.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1,
extra_args={
"audio_start_in_s": audio_start_in_s,
"audio_end_in_s": audio_end_in_s,
},
),
)
# Extract audio from OmniRequestOutput
@@ -5,6 +5,7 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.utils.platform_utils import is_npu, is_rocm
@@ -42,12 +43,14 @@ def test_diffusion_model(model_name: str):
width = 256
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=2,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=2,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=2,
),
)
# Extract images from request_output[0]['images']
first_output = outputs[0]
+11 -7
View File
@@ -5,6 +5,8 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -33,13 +35,15 @@ def test_video_diffusion_model(model_name: str):
width = 640
num_frames = 5
outputs = m.generate(
"A cat sitting on a table",
height=height,
width=width,
num_frames=num_frames,
num_inference_steps=2,
guidance_scale=1.0,
generator=torch.Generator("cuda").manual_seed(42),
prompts="A cat sitting on a table",
sampling_params_list=OmniDiffusionSamplingParams(
height=height,
width=width,
num_frames=num_frames,
num_inference_steps=2,
guidance_scale=1.0,
generator=torch.Generator("cuda").manual_seed(42),
),
)
first_output = outputs[0]
assert first_output.final_output_type == "image"
+10 -6
View File
@@ -15,6 +15,8 @@ from pathlib import Path
import pytest
import torch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -51,12 +53,14 @@ def test_teacache(model_name: str):
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
generator=torch.Generator("cuda").manual_seed(42),
num_outputs_per_prompt=1, # Single output for speed
),
)
# Extract images from request_output[0]['images']
first_output = outputs[0]
@@ -12,6 +12,8 @@ import torch
from PIL import Image
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
@@ -90,12 +92,14 @@ def _run_zimage_generate(
num_requests = 4 # 1 warmup + 3 timed
gen = m.generate(
[PROMPT] * num_requests,
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
seed=seed,
num_outputs_per_prompt=1,
OmniDiffusionSamplingParams(
height=height,
width=width,
num_inference_steps=num_inference_steps,
guidance_scale=0.0,
seed=seed,
num_outputs_per_prompt=1,
),
py_generator=True,
)
+10 -9
View File
@@ -5,6 +5,7 @@ from contextlib import ExitStack
from pathlib import Path
import pytest
from vllm import SamplingParams
from vllm.inputs import PromptType
from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState
@@ -25,15 +26,15 @@ async def generate(
) -> tuple[int, str]:
# Ensure generate doesn't complete too fast for cancellation test.
await asyncio.sleep(0.2)
thinker_sampling_params = {
"temperature": 0.4, # Deterministic
"top_p": 0.9,
"top_k": 1,
"max_tokens": max_tokens,
"repetition_penalty": 1.05,
"stop_token_ids": [151645], # Qwen EOS token <|im_end|>
"seed": SEED,
}
thinker_sampling_params = SamplingParams(
temperature=0.4, # Deterministic
top_p=0.9,
top_k=1,
max_tokens=max_tokens,
repetition_penalty=1.05,
stop_token_ids=[151645], # Qwen EOS token <|im_end|>
seed=SEED,
)
sampling_params_list = [
thinker_sampling_params,
@@ -14,11 +14,13 @@ from unittest.mock import AsyncMock, Mock
import pytest
from fastapi.testclient import TestClient
from PIL import Image
from vllm import SamplingParams
from vllm_omni.entrypoints.openai.image_api_utils import (
encode_image_base64,
parse_size,
)
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
# Unit Tests
@@ -111,7 +113,7 @@ class FakeAsyncOmni:
def __init__(self):
self.stage_list = ["llm", "diffusion"]
self.default_sampling_params_list = [{"temperature": 0.1}, {"top_p": 0.9}]
self.default_sampling_params_list = [SamplingParams(temperature=0.1), OmniDiffusionSamplingParams()]
self.captured_sampling_params_list = None
async def generate(self, prompt, request_id, sampling_params_list):
@@ -129,7 +131,8 @@ def mock_async_diffusion():
async def generate(**kwargs):
# Return n PIL images wrapped in result object
n = kwargs.get("num_outputs_per_prompt", 1)
print("!!!!!!!!!!!!!!!!!!!!! kwargs", kwargs)
n = kwargs["sampling_params_list"][0].num_outputs_per_prompt
images = [Image.new("RGB", (64, 64), color="blue") for _ in range(n)]
return MockGenerationResult(images)
@@ -268,11 +271,11 @@ def test_generate_images_async_omni_sampling_params(async_omni_test_client):
captured = engine.captured_sampling_params_list
assert captured is not None
assert len(captured) == 2
assert captured[0] == {"temperature": 0.1}
assert captured[1]["num_outputs_per_prompt"] == 2
assert captured[1]["height"] == 256
assert captured[1]["width"] == 256
assert captured[1]["seed"] == 7
assert captured[0].temperature == 0.1
assert captured[1].num_outputs_per_prompt == 2
assert captured[1].height == 256
assert captured[1].width == 256
assert captured[1].seed == 7
def test_generate_multiple_images(test_client):
@@ -505,30 +508,11 @@ def test_parameters_passed_through(test_client, mock_async_diffusion):
# Ensure generate() was called exactly once
mock_async_diffusion.generate.assert_awaited_once()
call_kwargs = mock_async_diffusion.generate.call_args[1]
assert call_kwargs["num_inference_steps"] == 100
assert call_kwargs["guidance_scale"] == 7.5
assert call_kwargs["true_cfg_scale"] == 3.0
assert call_kwargs["seed"] == 42
def test_optional_parameters_omitted(test_client, mock_async_diffusion):
"""Verify optional parameters not passed when omitted"""
response = test_client.post(
"/v1/images/generations",
json={
"prompt": "test",
"size": "512x512",
},
)
assert response.status_code == 200
# Ensure generate() was called exactly once
mock_async_diffusion.generate.assert_awaited_once()
call_kwargs = mock_async_diffusion.generate.call_args[1]
assert "num_inference_steps" not in call_kwargs
assert "guidance_scale" not in call_kwargs
assert "true_cfg_scale" not in call_kwargs
call_kwargs = mock_async_diffusion.generate.call_args[1]["sampling_params_list"][0]
assert call_kwargs.num_inference_steps == 100
assert call_kwargs.guidance_scale == 7.5
assert call_kwargs.true_cfg_scale == 3.0
assert call_kwargs.seed == 42
def test_model_field_omitted_works(test_client):
File diff suppressed because it is too large. Load diff
+15 -7
View File
@@ -5,6 +5,7 @@ from typing import Any
from unittest.mock import MagicMock
import pytest
from vllm import SamplingParams
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK
@@ -82,8 +83,9 @@ class _FakeStage:
self.stage_id = getattr(config, "stage_id", 0)
self.engine_args = config.engine_args
self.model_stage = getattr(config.engine_args, "model_stage", None)
self.stage_type = "llm"
# set default sampling params
self.default_sampling_params = {"temperature": 1.0}
self.default_sampling_params = SamplingParams(temperature=1.0)
# Allow configuring final_output and final_output_type
self.final_output = config.final_output if hasattr(config, "final_output") else False
self.final_output_type = getattr(config, "final_output_type", None)
@@ -637,8 +639,10 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
}
)
# Use dicts instead of object() for serializable sampling params
sampling_params_list = [{"temperature": 0.7}, {"temperature": 0.8}]
sampling_params_list = [
SamplingParams(temperature=0.7),
SamplingParams(temperature=0.8),
]
prompts = ["hi"]
outputs = omni.generate(prompts=prompts, sampling_params_list=sampling_params_list)
@@ -722,8 +726,13 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
}
)
# Use dicts instead of object() for serializable sampling params
outputs = omni.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}])
outputs = omni.generate(
prompts=["p"],
sampling_params_list=[
SamplingParams(temperature=0.7),
SamplingParams(temperature=0.8),
],
)
assert outputs == []
@@ -922,8 +931,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
)
# Generate should handle error gracefully (log but continue)
# Use dict instead of object() for serializable sampling params
sampling_params_list = [{"temperature": 0.7}]
sampling_params_list = [SamplingParams(temperature=0.7)]
outputs = omni.generate(prompts=["hi"], sampling_params_list=sampling_params_list)
# Should return final output (error was logged but didn't stop processing)
assert isinstance(outputs, list)