[Frontend][Model] Support batch request with refined OmniDiffusionReq… (#797)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
9ece725fc5
commit
d64bbde561
64 files changed
+2995
-1414
No files matched your search
@@ -1,10 +1,12 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.bagel.pipeline_bagel import BagelPipeline
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
|
||||
|
||||
|
||||
@@ -109,11 +111,12 @@ class TestKVFlow(unittest.TestCase):
|
||||
transfer_data.metadata = {"kv_lens": [self.seq_len], "ropes": [0]}
|
||||
|
||||
# 2. Setup Request with Injected Data
|
||||
req = OmniDiffusionRequest(prompt="test")
|
||||
from types import SimpleNamespace
|
||||
sp = OmniDiffusionSamplingParams(
|
||||
past_key_values=SimpleNamespace(**layer_blocks),
|
||||
kv_metadata=transfer_data.metadata,
|
||||
)
|
||||
|
||||
req.past_key_values = SimpleNamespace(**layer_blocks)
|
||||
req.kv_metadata = transfer_data.metadata
|
||||
req = OmniDiffusionRequest(["test"], sp)
|
||||
|
||||
# 3. Setup Pipeline
|
||||
pipeline = MockBagelPipeline()
|
||||
@@ -144,7 +147,7 @@ class TestKVFlow(unittest.TestCase):
|
||||
current_cache = RealNaiveCache(self.num_layers)
|
||||
|
||||
# --- Logic from Source Code ---
|
||||
injected_kv = req.past_key_values
|
||||
injected_kv = req.sampling_params.past_key_values
|
||||
if isinstance(current_cache, RealNaiveCache) and hasattr(injected_kv, "key_cache"):
|
||||
# Assuming injected_kv is SimpleNamespace or object with list attrs
|
||||
for layer_idx in range(len(injected_kv.key_cache)):
|
||||
@@ -176,9 +179,11 @@ class TestKVFlow(unittest.TestCase):
|
||||
data_dict = runner_test_result.to_dict()
|
||||
|
||||
# 3. Receiver (Request Setup)
|
||||
req = OmniDiffusionRequest(prompt="integration_test")
|
||||
req.past_key_values = data_dict["layer_blocks"]
|
||||
req.kv_metadata = data_dict["metadata"]
|
||||
sp = OmniDiffusionSamplingParams(
|
||||
past_key_values=data_dict["layer_blocks"],
|
||||
kv_metadata=data_dict["metadata"],
|
||||
)
|
||||
req = OmniDiffusionRequest(["integration_test"], sp) # noqa: F841
|
||||
|
||||
# 4. Receiver (Injection Simulation)
|
||||
# Use the logic verification again
|
||||
|
||||
@@ -7,11 +7,12 @@ Pytest configuration and fixtures for vllm-omni tests.
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from vllm import TextPrompt
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.conftest import _run_post_test_cleanup
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
|
||||
@@ -64,7 +65,7 @@ class OmniRunner:
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_default_sampling_params_list(self) -> list[SamplingParams]:
|
||||
def get_default_sampling_params_list(self) -> list[OmniSamplingParams]:
|
||||
"""
|
||||
Get a list of default sampling parameters for all stages.
|
||||
|
||||
@@ -82,7 +83,7 @@ class OmniRunner:
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
modalities: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
) -> list[TextPrompt]:
|
||||
"""
|
||||
Construct Omni input format from prompts and multimodal data.
|
||||
|
||||
@@ -175,7 +176,7 @@ class OmniRunner:
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
input_dict: dict[str, Any] = {"prompt": full_prompt}
|
||||
input_dict: TextPrompt = {"prompt": full_prompt}
|
||||
if multi_modal_data:
|
||||
input_dict["multi_modal_data"] = multi_modal_data
|
||||
if modalities:
|
||||
@@ -189,8 +190,8 @@ class OmniRunner:
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: list[dict[str, Any]],
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
prompts: list[TextPrompt],
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
) -> list[OmniRequestOutput]:
|
||||
"""
|
||||
Generate outputs for the given prompts.
|
||||
@@ -212,7 +213,7 @@ class OmniRunner:
|
||||
def generate_multimodal(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
images: PromptImageInput = None,
|
||||
@@ -249,7 +250,7 @@ class OmniRunner:
|
||||
def generate_audio(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
@@ -276,7 +277,7 @@ class OmniRunner:
|
||||
def generate_video(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
@@ -303,7 +304,7 @@ class OmniRunner:
|
||||
def generate_image(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
sampling_params_list: list[OmniSamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
images: PromptImageInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
|
||||
@@ -15,6 +15,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
@@ -55,12 +57,14 @@ def test_cache_dit(model_name: str):
|
||||
|
||||
outputs = m.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1, # Single output for speed
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1, # Single output for speed
|
||||
),
|
||||
)
|
||||
# Extract images from request_output[0]['images']
|
||||
first_output = outputs[0]
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from tests.utils import GPUMemoryMonitor
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.utils.platform_utils import is_npu, is_rocm
|
||||
|
||||
# ruff: noqa: E402
|
||||
@@ -30,11 +31,13 @@ def inference(model_name: str, offload: bool = True):
|
||||
|
||||
m.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=9,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=9,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
),
|
||||
)
|
||||
peak = monitor.peak_used_mb
|
||||
monitor.stop()
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
|
||||
@@ -88,12 +89,14 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
|
||||
|
||||
outputs = m.generate(
|
||||
prompt,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
images = _extract_images(outputs)
|
||||
|
||||
@@ -116,14 +119,16 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
|
||||
)
|
||||
outputs_lora = m.generate(
|
||||
prompt,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
lora_request=lora_request,
|
||||
lora_scale=2.0,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
lora_request=lora_request,
|
||||
lora_scale=2.0,
|
||||
),
|
||||
)
|
||||
images_lora = _extract_images(outputs_lora)
|
||||
assert len(images_lora) == 1
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image import OvisImagePipeline
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -162,11 +163,13 @@ def test_basic_generation(ovis_pipeline):
|
||||
"""Test the forward pass logic."""
|
||||
# Setup request
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="A photo of a cat",
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
prompts=["A photo of a cat"],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
),
|
||||
)
|
||||
|
||||
output = ovis_pipeline(req)
|
||||
@@ -184,12 +187,18 @@ def test_basic_generation(ovis_pipeline):
|
||||
def test_guidance_scale(ovis_pipeline):
|
||||
"""Test that classifier-free guidance path is taken when scale > 1.0."""
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="A photo of a cat",
|
||||
negative_prompt="bad quality",
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=1,
|
||||
guidance_scale=2.0, # Trigger CFG
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "A photo of a cat",
|
||||
"negative_prompt": "bad quality",
|
||||
}
|
||||
],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=256,
|
||||
num_inference_steps=1,
|
||||
guidance_scale=2.0, # Trigger CFG
|
||||
),
|
||||
)
|
||||
|
||||
ovis_pipeline(req)
|
||||
@@ -200,9 +209,11 @@ def test_resolution_check(ovis_pipeline):
|
||||
"""Test resolution divisible validation logic if present."""
|
||||
# Pass odd resolution
|
||||
req = OmniDiffusionRequest(
|
||||
prompt="test",
|
||||
height=250, # Not divisible by 16 (8*2)
|
||||
width=250,
|
||||
prompts=["test"],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
height=250, # Not divisible by 16 (8*2)
|
||||
width=250,
|
||||
),
|
||||
)
|
||||
|
||||
# Should warn but proceed (as per code I read earlier) or resize?
|
||||
|
||||
@@ -21,6 +21,8 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
@@ -81,12 +83,14 @@ def _run_baseline(model_name: str, dtype: torch.dtype, attn_backend: str, height
|
||||
try:
|
||||
outputs = baseline.generate(
|
||||
PROMPT,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=4,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(get_device_name()).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=4,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(get_device_name()).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
return outputs[0].request_output[0].images
|
||||
finally:
|
||||
@@ -116,12 +120,14 @@ def _run_sp(
|
||||
try:
|
||||
outputs = sp.generate(
|
||||
PROMPT,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=4,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(get_device_name()).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=4,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator(get_device_name()).manual_seed(seed),
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
)
|
||||
return outputs[0].request_output[0].images
|
||||
finally:
|
||||
|
||||
@@ -5,6 +5,7 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
# ruff: noqa: E402
|
||||
@@ -29,16 +30,20 @@ def test_stable_audio_model(model_name: str):
|
||||
sample_rate = 44100 # Stable Audio uses 44100 Hz
|
||||
|
||||
outputs = m.generate(
|
||||
"The sound of a dog barking",
|
||||
negative_prompt="Low quality.",
|
||||
num_inference_steps=4, # Minimal steps for speed
|
||||
guidance_scale=7.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
extra={
|
||||
"audio_start_in_s": audio_start_in_s,
|
||||
"audio_end_in_s": audio_end_in_s,
|
||||
prompts={
|
||||
"prompt": "The sound of a dog barking",
|
||||
"negative_prompt": "Low quality.",
|
||||
},
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
num_inference_steps=4, # Minimal steps for speed
|
||||
guidance_scale=7.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1,
|
||||
extra_args={
|
||||
"audio_start_in_s": audio_start_in_s,
|
||||
"audio_end_in_s": audio_end_in_s,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
# Extract audio from OmniRequestOutput
|
||||
|
||||
@@ -5,6 +5,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.utils.platform_utils import is_npu, is_rocm
|
||||
|
||||
@@ -42,12 +43,14 @@ def test_diffusion_model(model_name: str):
|
||||
width = 256
|
||||
outputs = m.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=2,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=2,
|
||||
),
|
||||
)
|
||||
# Extract images from request_output[0]['images']
|
||||
first_output = outputs[0]
|
||||
|
||||
@@ -5,6 +5,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
@@ -33,13 +35,15 @@ def test_video_diffusion_model(model_name: str):
|
||||
width = 640
|
||||
num_frames = 5
|
||||
outputs = m.generate(
|
||||
"A cat sitting on a table",
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
prompts="A cat sitting on a table",
|
||||
sampling_params_list=OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
),
|
||||
)
|
||||
first_output = outputs[0]
|
||||
assert first_output.final_output_type == "image"
|
||||
|
||||
@@ -15,6 +15,8 @@ from pathlib import Path
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
@@ -51,12 +53,14 @@ def test_teacache(model_name: str):
|
||||
|
||||
outputs = m.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1, # Single output for speed
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=1, # Single output for speed
|
||||
),
|
||||
)
|
||||
# Extract images from request_output[0]['images']
|
||||
first_output = outputs[0]
|
||||
|
||||
@@ -12,6 +12,8 @@ import torch
|
||||
from PIL import Image
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
@@ -90,12 +92,14 @@ def _run_zimage_generate(
|
||||
num_requests = 4 # 1 warmup + 3 timed
|
||||
gen = m.generate(
|
||||
[PROMPT] * num_requests,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
seed=seed,
|
||||
num_outputs_per_prompt=1,
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=0.0,
|
||||
seed=seed,
|
||||
num_outputs_per_prompt=1,
|
||||
),
|
||||
py_generator=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm import SamplingParams
|
||||
from vllm.inputs import PromptType
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState
|
||||
@@ -25,15 +26,15 @@ async def generate(
|
||||
) -> tuple[int, str]:
|
||||
# Ensure generate doesn't complete too fast for cancellation test.
|
||||
await asyncio.sleep(0.2)
|
||||
thinker_sampling_params = {
|
||||
"temperature": 0.4, # Deterministic
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"max_tokens": max_tokens,
|
||||
"repetition_penalty": 1.05,
|
||||
"stop_token_ids": [151645], # Qwen EOS token <|im_end|>
|
||||
"seed": SEED,
|
||||
}
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.4, # Deterministic
|
||||
top_p=0.9,
|
||||
top_k=1,
|
||||
max_tokens=max_tokens,
|
||||
repetition_penalty=1.05,
|
||||
stop_token_ids=[151645], # Qwen EOS token <|im_end|>
|
||||
seed=SEED,
|
||||
)
|
||||
|
||||
sampling_params_list = [
|
||||
thinker_sampling_params,
|
||||
|
||||
@@ -14,11 +14,13 @@ from unittest.mock import AsyncMock, Mock
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.openai.image_api_utils import (
|
||||
encode_image_base64,
|
||||
parse_size,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
# Unit Tests
|
||||
|
||||
@@ -111,7 +113,7 @@ class FakeAsyncOmni:
|
||||
|
||||
def __init__(self):
|
||||
self.stage_list = ["llm", "diffusion"]
|
||||
self.default_sampling_params_list = [{"temperature": 0.1}, {"top_p": 0.9}]
|
||||
self.default_sampling_params_list = [SamplingParams(temperature=0.1), OmniDiffusionSamplingParams()]
|
||||
self.captured_sampling_params_list = None
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params_list):
|
||||
@@ -129,7 +131,8 @@ def mock_async_diffusion():
|
||||
|
||||
async def generate(**kwargs):
|
||||
# Return n PIL images wrapped in result object
|
||||
n = kwargs.get("num_outputs_per_prompt", 1)
|
||||
print("!!!!!!!!!!!!!!!!!!!!! kwargs", kwargs)
|
||||
n = kwargs["sampling_params_list"][0].num_outputs_per_prompt
|
||||
images = [Image.new("RGB", (64, 64), color="blue") for _ in range(n)]
|
||||
return MockGenerationResult(images)
|
||||
|
||||
@@ -268,11 +271,11 @@ def test_generate_images_async_omni_sampling_params(async_omni_test_client):
|
||||
captured = engine.captured_sampling_params_list
|
||||
assert captured is not None
|
||||
assert len(captured) == 2
|
||||
assert captured[0] == {"temperature": 0.1}
|
||||
assert captured[1]["num_outputs_per_prompt"] == 2
|
||||
assert captured[1]["height"] == 256
|
||||
assert captured[1]["width"] == 256
|
||||
assert captured[1]["seed"] == 7
|
||||
assert captured[0].temperature == 0.1
|
||||
assert captured[1].num_outputs_per_prompt == 2
|
||||
assert captured[1].height == 256
|
||||
assert captured[1].width == 256
|
||||
assert captured[1].seed == 7
|
||||
|
||||
|
||||
def test_generate_multiple_images(test_client):
|
||||
@@ -505,30 +508,11 @@ def test_parameters_passed_through(test_client, mock_async_diffusion):
|
||||
|
||||
# Ensure generate() was called exactly once
|
||||
mock_async_diffusion.generate.assert_awaited_once()
|
||||
call_kwargs = mock_async_diffusion.generate.call_args[1]
|
||||
assert call_kwargs["num_inference_steps"] == 100
|
||||
assert call_kwargs["guidance_scale"] == 7.5
|
||||
assert call_kwargs["true_cfg_scale"] == 3.0
|
||||
assert call_kwargs["seed"] == 42
|
||||
|
||||
|
||||
def test_optional_parameters_omitted(test_client, mock_async_diffusion):
|
||||
"""Verify optional parameters not passed when omitted"""
|
||||
response = test_client.post(
|
||||
"/v1/images/generations",
|
||||
json={
|
||||
"prompt": "test",
|
||||
"size": "512x512",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Ensure generate() was called exactly once
|
||||
mock_async_diffusion.generate.assert_awaited_once()
|
||||
call_kwargs = mock_async_diffusion.generate.call_args[1]
|
||||
assert "num_inference_steps" not in call_kwargs
|
||||
assert "guidance_scale" not in call_kwargs
|
||||
assert "true_cfg_scale" not in call_kwargs
|
||||
call_kwargs = mock_async_diffusion.generate.call_args[1]["sampling_params_list"][0]
|
||||
assert call_kwargs.num_inference_steps == 100
|
||||
assert call_kwargs.guidance_scale == 7.5
|
||||
assert call_kwargs.true_cfg_scale == 3.0
|
||||
assert call_kwargs.seed == 42
|
||||
|
||||
|
||||
def test_model_field_omitted_works(test_client):
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,6 +5,7 @@ from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from vllm import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK
|
||||
|
||||
@@ -82,8 +83,9 @@ class _FakeStage:
|
||||
self.stage_id = getattr(config, "stage_id", 0)
|
||||
self.engine_args = config.engine_args
|
||||
self.model_stage = getattr(config.engine_args, "model_stage", None)
|
||||
self.stage_type = "llm"
|
||||
# set default sampling params
|
||||
self.default_sampling_params = {"temperature": 1.0}
|
||||
self.default_sampling_params = SamplingParams(temperature=1.0)
|
||||
# Allow configuring final_output and final_output_type
|
||||
self.final_output = config.final_output if hasattr(config, "final_output") else False
|
||||
self.final_output_type = getattr(config, "final_output_type", None)
|
||||
@@ -637,8 +639,10 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config):
|
||||
}
|
||||
)
|
||||
|
||||
# Use dicts instead of object() for serializable sampling params
|
||||
sampling_params_list = [{"temperature": 0.7}, {"temperature": 0.8}]
|
||||
sampling_params_list = [
|
||||
SamplingParams(temperature=0.7),
|
||||
SamplingParams(temperature=0.8),
|
||||
]
|
||||
prompts = ["hi"]
|
||||
outputs = omni.generate(prompts=prompts, sampling_params_list=sampling_params_list)
|
||||
|
||||
@@ -722,8 +726,13 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config):
|
||||
}
|
||||
)
|
||||
|
||||
# Use dicts instead of object() for serializable sampling params
|
||||
outputs = omni.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}])
|
||||
outputs = omni.generate(
|
||||
prompts=["p"],
|
||||
sampling_params_list=[
|
||||
SamplingParams(temperature=0.7),
|
||||
SamplingParams(temperature=0.8),
|
||||
],
|
||||
)
|
||||
assert outputs == []
|
||||
|
||||
|
||||
@@ -922,8 +931,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config):
|
||||
)
|
||||
|
||||
# Generate should handle error gracefully (log but continue)
|
||||
# Use dict instead of object() for serializable sampling params
|
||||
sampling_params_list = [{"temperature": 0.7}]
|
||||
sampling_params_list = [SamplingParams(temperature=0.7)]
|
||||
outputs = omni.generate(prompts=["hi"], sampling_params_list=sampling_params_list)
|
||||
# Should return final output (error was logged but didn't stop processing)
|
||||
assert isinstance(outputs, list)
|
||||
|
||||
Reference in new issue
Block a user