[Feature][RL] Support batching for QwenImage in async mode (#1593)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
This commit is contained in:
knlnguyen1802 authored and GitHub committed 2026-03-23 10:08:24 +00:00
1 parent 560023d5d1
commit 65d76a4127
9 files changed
+887 -24

No files matched your search

+17
View File
@@ -50,6 +50,23 @@ steps:
volumes:
- "/fsx/hf_cache:/fsx/hf_cache"
- label: "Diffusion Batching Test"
depends_on: upload-ready-pipeline
commands:
- timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
agents:
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
plugins:
- docker#v5.2.0:
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
always-pull: true
propagate-environment: true
environment:
- "HF_HOME=/fsx/hf_cache"
- "HF_TOKEN"
volumes:
- "/fsx/hf_cache:/fsx/hf_cache"
- label: "Diffusion Model CPU offloading Test"
depends_on: upload-ready-pipeline
commands:
@@ -0,0 +1,641 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end test for diffusion batching via AsyncOmni.
This test fires multiple concurrent ``AsyncOmni.generate()`` calls for a
diffusion model and validates that every caller receives its correct
individual result. When the underlying diffusion stage is configured with
``batch_size > 1`` (via stage config or ``StageDiffusionClient``), the
requests will be batched internally.
Even without explicit batching config this test is useful for verifying
that concurrent async requests are handled correctly.
Usage (standalone):
python tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py \
--model <model_name_or_path> \
--num-prompts 8
Or via pytest:
pytest tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -s
"""
from __future__ import annotations
import argparse
import asyncio
import os
import sys
import time
import uuid
from pathlib import Path
import pytest
import torch
from tests.utils import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.platforms import current_omni_platform
# ruff: noqa: E402
REPO_ROOT = Path(__file__).resolve().parents[3]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from vllm_omni import Omni
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
# ------------------------------------------------------------------
models = ["tiny-random/Qwen-Image"]
# ------------------------------------------------------------------
# Prompt fixtures
# ------------------------------------------------------------------
WARMUP_PROMPTS: list[dict[str, str]] = [
{"prompt": "a sunflower in a glass vase", "negative_prompt": "blurry"},
{"prompt": "a rocket launching into space", "negative_prompt": "low detail"},
{"prompt": "a small cottage in the snowy mountains", "negative_prompt": "foggy"},
{"prompt": "a colorful parrot sitting on a tree branch", "negative_prompt": "low contrast"},
]
TEST_PROMPTS: list[dict[str, str]] = [
{"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"},
{"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": "cinematic, realistic"},
{"prompt": "a futuristic city skyline at sunset", "negative_prompt": "blurry, foggy"},
{"prompt": "a bowl of fresh strawberries", "negative_prompt": "low detail"},
{"prompt": "a medieval knight standing in the rain", "negative_prompt": "modern clothing"},
{"prompt": "a cat wearing sunglasses lounging in a garden", "negative_prompt": "dark lighting"},
{"prompt": "a spaceship flying above a volcano", "negative_prompt": "low contrast"},
{"prompt": "a watercolor painting of a mountain lake", "negative_prompt": "photo, realistic"},
]
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _default_sampling_params(**overrides) -> OmniDiffusionSamplingParams:
defaults = dict(
num_inference_steps=2,
width=256,
height=256,
guidance_scale=0.0,
)
defaults.update(overrides)
return OmniDiffusionSamplingParams(**defaults)
def _default_sync_sampling_params(**overrides) -> OmniDiffusionSamplingParams:
"""Create sampling params for the synchronous Omni.generate() API."""
defaults = dict(
num_inference_steps=2,
width=256,
height=256,
guidance_scale=0.0,
generator=torch.Generator(current_omni_platform.device_type).manual_seed(42),
)
defaults.update(overrides)
return OmniDiffusionSamplingParams(**defaults)
async def _collect_generate(omni: AsyncOmni, prompt, request_id, sampling_params_list) -> OmniRequestOutput:
"""Consume the AsyncOmni.generate() async generator and return the last output."""
last_output: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompt,
request_id=request_id,
sampling_params_list=sampling_params_list,
):
last_output = output
if last_output is None:
raise RuntimeError(f"No output received for request {request_id}")
return last_output
def _extract_images(output: OmniRequestOutput) -> list:
"""Extract images from an OmniRequestOutput, handling both direct
and nested request_output structures."""
if output.images:
return output.images
# When the output comes from the orchestrator pipeline, images may be
# nested inside request_output.
inner = getattr(output, "request_output", None)
if inner is not None and hasattr(inner, "images") and inner.images:
return inner.images
return []
# ------------------------------------------------------------------
# Warm-up (async)
# ------------------------------------------------------------------
async def warmup(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
"""Warm-up: send prompts in parallel to pre-load the model."""
print(f"🔥 Warming up with {len(prompts)} prompts ...")
sp = _default_sampling_params(num_inference_steps=2)
start = time.perf_counter()
tasks = [
_collect_generate(
omni,
prompt=p,
request_id=f"warmup-{i}-{uuid.uuid4().hex[:8]}",
sampling_params_list=[sp],
)
for i, p in enumerate(prompts)
]
await asyncio.gather(*tasks)
elapsed = time.perf_counter() - start
print(f" Warm-up done in {elapsed:.2f}s\n")
# ------------------------------------------------------------------
# Single (sequential) benchmark
# ------------------------------------------------------------------
async def run_single(omni: AsyncOmni, prompts: list[dict[str, str]]) -> float:
"""Run prompts one-by-one sequentially."""
print(f"🧩 Running SINGLE (sequential) mode – {len(prompts)} prompts ...")
sp = _default_sampling_params()
total_start = time.perf_counter()
for i, prompt in enumerate(prompts):
start = time.perf_counter()
result = await _collect_generate(
omni,
prompt=prompt,
request_id=f"single-{i}-{uuid.uuid4().hex[:8]}",
sampling_params_list=[sp],
)
elapsed = time.perf_counter() - start
images = _extract_images(result)
print(f" prompt {i}: {elapsed:.2f}s ({len(images)} images)")
total = time.perf_counter() - total_start
print(f" ✅ Total single-mode: {total:.2f}s\n")
return total
# ------------------------------------------------------------------
# Batch (parallel) benchmark — concurrent individual requests
# ------------------------------------------------------------------
async def run_batch(
omni: AsyncOmni,
prompts: list[dict[str, str]],
label: str = "batch",
) -> float:
"""Send all prompts concurrently via asyncio.gather (one request per prompt)."""
print(f"⚙️ Running {label.upper()} mode – {len(prompts)} prompts concurrently ...")
sp = _default_sampling_params()
start = time.perf_counter()
tasks = [
_collect_generate(
omni,
prompt=p,
request_id=f"{label}-{i}-{uuid.uuid4().hex[:8]}",
sampling_params_list=[sp],
)
for i, p in enumerate(prompts)
]
results = await asyncio.gather(*tasks)
elapsed = time.perf_counter() - start
for i, result in enumerate(results):
images = _extract_images(result)
print(f" prompt {i}: {len(images)} images, request_id={result.request_id}")
print(f" ✅ Total {label} mode: {elapsed:.2f}s\n")
return elapsed
# ------------------------------------------------------------------
# Explicit batch — single generate() call with list of prompts
# ------------------------------------------------------------------
async def run_batch_explicit(
omni: AsyncOmni,
prompts: list[dict[str, str]],
label: str = "batch_explicit",
) -> float:
"""Send all prompts as a single batch via generate(prompt=[...]).
Passes a *list* of prompts so that all are processed in **one**
``DiffusionEngine.step()`` call. The orchestrator detects
``isinstance(prompt, list)`` and routes to the batch path.
A single ``OmniRequestOutput`` is yielded containing ALL generated
images combined.
"""
print(f"⚡ Running {label.upper()} mode – {len(prompts)} prompts in ONE engine call ...")
sp = _default_sampling_params()
request_id = f"{label}-{uuid.uuid4().hex[:8]}"
start = time.perf_counter()
result: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompts,
request_id=request_id,
sampling_params_list=[sp],
):
result = output
elapsed = time.perf_counter() - start
if result is not None:
images = _extract_images(result)
print(f" Got {len(images)} images total from batch, request_id={result.request_id}")
else:
print(" ⚠️ No output received from batch generate()")
print(f" ✅ Total {label} mode: {elapsed:.2f}s\n")
return elapsed
# ------------------------------------------------------------------
# Async validation helpers
# ------------------------------------------------------------------
async def validate_batch_explicit(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
"""Validate generate(prompt=[...]) returns a single result with all images."""
print(f"🔍 Validating batch generate() correctness with {len(prompts)} prompts ...")
sp = _default_sampling_params()
request_id = f"validate-batch-{uuid.uuid4().hex[:8]}"
result: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompts,
request_id=request_id,
sampling_params_list=[sp],
):
result = output
assert result is not None, "No output received from batch generate()"
images = _extract_images(result)
# Batch mode returns ONE output with ALL images combined
assert len(images) == len(prompts), f"Expected {len(prompts)} images (one per prompt), got {len(images)}"
assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}"
print(f" ✅ Batch returned {len(images)} images with correct request_id.\n")
async def validate_concurrent(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None:
"""Validate that every concurrent request receives a distinct result
with its own request_id."""
print(f"🔍 Validating concurrent correctness with {len(prompts)} prompts ...")
sp = _default_sampling_params()
request_ids = [f"validate-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))]
tasks = [
_collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp])
for p, rid in zip(prompts, request_ids)
]
results = await asyncio.gather(*tasks)
assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}"
returned_ids = [r.request_id for r in results]
for rid in request_ids:
assert rid in returned_ids, f"Missing request_id {rid} in results"
print(" ✅ All request_ids matched, results count correct.\n")
# ------------------------------------------------------------------
# Single vs Parallel comparison (CLI only)
# ------------------------------------------------------------------
async def compare_single_vs_parallel(
model: str,
prompts: list[dict[str, str]],
batch_size: int = 1,
) -> None:
"""Run the same prompts sequentially then in parallel and print a comparison."""
omni = AsyncOmni(model=model, diffusion_batch_size=batch_size)
try:
await warmup(omni, WARMUP_PROMPTS)
single_time = await run_single(omni, prompts)
parallel_time = await run_batch(omni, prompts, label="parallel")
explicit_time = await run_batch_explicit(omni, prompts, label="batch_explicit")
finally:
omni.shutdown()
speedup_parallel = single_time / parallel_time if parallel_time > 0 else float("inf")
speedup_explicit = single_time / explicit_time if explicit_time > 0 else float("inf")
print("=" * 60)
print(f"📊 Summary ({len(prompts)} prompts)")
print(f" Sequential : {single_time:.2f}s")
print(f" Parallel (gather) : {parallel_time:.2f}s ({speedup_parallel:.2f}x)")
print(f" Explicit batch : {explicit_time:.2f}s ({speedup_explicit:.2f}x)")
print("=" * 60)
# ------------------------------------------------------------------
# CLI main entrypoint
# ------------------------------------------------------------------
async def main(model: str, num_prompts: int, mode: str, batch_size: int = 1) -> None:
prompts = (TEST_PROMPTS * ((num_prompts // len(TEST_PROMPTS)) + 1))[:num_prompts]
if mode == "compare":
await compare_single_vs_parallel(model, prompts, batch_size=batch_size)
return
omni = AsyncOmni(model=model, diffusion_batch_size=batch_size)
try:
await warmup(omni, WARMUP_PROMPTS)
if mode == "validate":
await validate_concurrent(omni, prompts)
elif mode == "validate_batch":
await validate_batch_explicit(omni, prompts)
elif mode == "batch":
await run_batch(omni, prompts, label="measurement")
elif mode == "batch_explicit":
await run_batch_explicit(omni, prompts)
elif mode == "single":
await run_single(omni, prompts)
else:
raise ValueError(f"Unknown mode: {mode}")
finally:
omni.shutdown()
# ==================================================================
# pytest test cases
# ==================================================================
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_sync_sequential(model_name: str):
"""Test that synchronous Omni can generate images for multiple prompts
submitted sequentially (one at a time) and each returns a valid image."""
m = None
try:
m = Omni(model=model_name)
sp = _default_sync_sampling_params()
prompts = TEST_PROMPTS[:4]
for i, prompt in enumerate(prompts):
outputs = m.generate(prompt, sp)
first_output = outputs[0]
assert first_output.final_output_type == "image", (
f"Expected 'image', got '{first_output.final_output_type}'"
)
# Images are surfaced both at top-level and inside request_output
images = _extract_images(first_output)
assert len(images) >= 1, f"Expected at least 1 image for prompt {i}, got {len(images)}"
assert images[0].width == 256
assert images[0].height == 256
print(f" prompt {i}: OK ({len(images)} images)")
except Exception as e:
print(f"Test failed with error: {e}")
raise
finally:
if m is not None and hasattr(m, "close"):
m.close()
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_sync_multi_prompt(model_name: str):
"""Test that synchronous Omni correctly handles a list of multiple
prompts submitted at once and returns one result per prompt.
Note: Omni.generate() iterates the list and submits each prompt
individually with its own request_id. This tests concurrent request
handling at the diffusion stage, not the explicit list-batch path
(which is only available via AsyncOmni).
"""
m = None
try:
m = Omni(model=model_name)
sp = _default_sync_sampling_params()
prompts = TEST_PROMPTS[:4]
outputs = m.generate(prompts, sp)
assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}"
for i, output in enumerate(outputs):
assert output.final_output_type == "image", (
f"Output {i} final_output_type expected 'image', got '{output.final_output_type}'"
)
images = _extract_images(output)
assert images and len(images) >= 1, f"Expected at least 1 image for prompt {i}"
assert images[0].width == 256
assert images[0].height == 256
print(f" prompt {i}: OK ({len(images)} images, request_id={output.request_id})")
# Verify all request_ids are distinct
request_ids = [o.request_id for o in outputs]
assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids found: {request_ids}"
except Exception as e:
print(f"Test failed with error: {e}")
raise
finally:
if m is not None and hasattr(m, "close"):
m.close()
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_async_concurrent(model_name: str):
"""Test that AsyncOmni correctly handles multiple concurrent requests
fired via asyncio.gather. Each request_id must appear in the results."""
async def _inner():
omni = AsyncOmni(model=model_name, diffusion_batch_size=1)
try:
prompts = TEST_PROMPTS[:4]
sp = _default_sampling_params()
request_ids = [f"async-concurrent-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))]
tasks = [
_collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp])
for p, rid in zip(prompts, request_ids)
]
results = await asyncio.gather(*tasks)
assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}"
returned_ids = [r.request_id for r in results]
for rid in request_ids:
assert rid in returned_ids, f"Missing request_id {rid} in results"
for i, result in enumerate(results):
images = _extract_images(result)
assert len(images) >= 1, f"No images for prompt {i}"
assert images[0].width == 256
assert images[0].height == 256
print(f" prompt {i}: OK ({len(images)} images, request_id={result.request_id})")
finally:
omni.shutdown()
asyncio.run(_inner())
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_async_explicit_batch(model_name: str):
"""Test that AsyncOmni batch mode (generate(prompt=[...])) dispatches
all prompts in a single engine call and returns a single combined result.
The list-prompt path routes through the orchestrator's
``add_batch_request_async`` → ``AsyncOmniDiffusion.generate_batch``
and yields ONE ``OmniRequestOutput`` with ALL images combined.
"""
async def _inner():
omni = AsyncOmni(model=model_name, diffusion_batch_size=4)
try:
prompts = TEST_PROMPTS[:4]
sp = _default_sampling_params()
request_id = f"explicit-batch-{uuid.uuid4().hex[:8]}"
# Batch mode: pass list of prompts → single request_id
result: OmniRequestOutput | None = None
async for output in omni.generate(
prompt=prompts,
request_id=request_id,
sampling_params_list=[sp],
):
result = output
assert result is not None, "No output received from batch generate()"
images = _extract_images(result)
# One image per prompt, all in a single output
assert len(images) == len(prompts), f"Expected {len(prompts)} images in combined output, got {len(images)}"
assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}"
for i, img in enumerate(images):
assert img.width == 256, f"Image {i} width mismatch"
assert img.height == 256, f"Image {i} height mismatch"
print(f" ✅ Batch returned {len(images)} images, request_id={result.request_id}")
finally:
omni.shutdown()
asyncio.run(_inner())
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_num_outputs(model_name: str):
"""Test that the diffusion model respects num_outputs_per_prompt and
generates the correct number of images per request."""
m = None
try:
m = Omni(model=model_name)
num_outputs = 2
sp = _default_sync_sampling_params(num_outputs_per_prompt=num_outputs)
outputs = m.generate(
"a photo of a cat sitting on a laptop keyboard",
sp,
)
first_output = outputs[0]
assert first_output.final_output_type == "image"
images = _extract_images(first_output)
assert images is not None and len(images) == num_outputs, (
f"Expected {num_outputs} images, got {len(images) if images else 0}"
)
for img in images:
assert img.width == 256
assert img.height == 256
except Exception as e:
print(f"Test failed with error: {e}")
raise
finally:
if m is not None and hasattr(m, "close"):
m.close()
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"})
@pytest.mark.parametrize("model_name", models)
def test_diffusion_batching_distinct_results(model_name: str):
"""Test that different prompts produce distinct images when batched,
ensuring the batching logic does not mix up results across requests."""
m = None
try:
m = Omni(model=model_name)
sp = _default_sync_sampling_params()
prompts = [
{"prompt": "a bright red apple on a white table", "negative_prompt": "blurry"},
{"prompt": "a blue ocean with white waves crashing", "negative_prompt": "blurry"},
]
outputs = m.generate(prompts, sp)
assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}"
# Verify each output has a unique request_id
request_ids = [o.request_id for o in outputs]
assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids: {request_ids}"
# Verify each output has images
for i, output in enumerate(outputs):
images = _extract_images(output)
assert images and len(images) >= 1, f"No images for prompt {i}"
assert images[0].width == 256
assert images[0].height == 256
except Exception as e:
print(f"Test failed with error: {e}")
raise
finally:
if m is not None and hasattr(m, "close"):
m.close()
# ------------------------------------------------------------------
# CLI
# ------------------------------------------------------------------
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="E2E diffusion concurrent benchmark / validation")
parser.add_argument("--model", type=str, required=True, help="Model name or path")
parser.add_argument("--num-prompts", type=int, default=8, help="Number of prompts to run")
parser.add_argument("--batch-size", type=int, default=1, help="Diffusion batch size (1 = no batching)")
parser.add_argument(
"--mode",
choices=["batch", "batch_explicit", "single", "compare", "validate", "validate_batch"],
default="compare",
help=(
"Run mode: 'batch' (parallel gather), 'batch_explicit' (list-prompt batch API), "
"'single' (sequential), 'compare' (all three), "
"'validate' (concurrent correctness), 'validate_batch' (list-prompt correctness)"
),
)
args = parser.parse_args()
asyncio.run(main(args.model, args.num_prompts, args.mode, batch_size=args.batch_size))
+46 -2
View File
@@ -37,6 +37,7 @@ class StageDiffusionClient:
model: str,
od_config: OmniDiffusionConfig,
metadata: StageMetadata,
batch_size: int = 1,
) -> None:
self.stage_id = metadata.stage_id
self.final_output = metadata.final_output
@@ -45,11 +46,11 @@ class StageDiffusionClient:
self.custom_process_input_func = metadata.custom_process_input_func
self.engine_input_source = metadata.engine_input_source
self._engine = AsyncOmniDiffusion(model=model, od_config=od_config)
self._engine = AsyncOmniDiffusion(model=model, od_config=od_config, batch_size=batch_size)
self._output_queue: asyncio.Queue[OmniRequestOutput] = asyncio.Queue()
self._tasks: dict[str, asyncio.Task] = {}
logger.info("[StageDiffusionClient] Stage-%s initialized", self.stage_id)
logger.info("[StageDiffusionClient] Stage-%s initialized (batch_size=%d)", self.stage_id, batch_size)
async def add_request_async(
self,
@@ -82,6 +83,49 @@ class StageDiffusionClient:
finally:
self._tasks.pop(request_id, None)
# TODO(Long): Temporary solution to boost performance of diffusion stages.
# Remove this after scheduling algorithm is implemented
async def add_batch_request_async(
self,
request_id: str,
prompts: list[OmniPromptType],
sampling_params: OmniDiffusionSamplingParams,
) -> None:
"""Submit a list of prompts as a single batched engine call.
All prompts are processed in one ``DiffusionEngine.step()`` call
and the combined result is placed on the output queue with a single
*request_id*.
"""
task = asyncio.create_task(
self._run_batch(request_id, prompts, sampling_params),
name=f"diffusion-batch-{request_id}",
)
self._tasks[request_id] = task
async def _run_batch(
self,
request_id: str,
prompts: list[OmniPromptType],
sampling_params: OmniDiffusionSamplingParams,
) -> None:
try:
result = await self._engine.generate_batch(
prompts,
sampling_params,
request_id,
)
await self._output_queue.put(result)
except Exception as e:
logger.exception(
"[StageDiffusionClient] Stage-%s batch req=%s failed: %s",
self.stage_id,
request_id,
e,
)
finally:
self._tasks.pop(request_id, None)
def get_diffusion_output_async(self) -> OmniRequestOutput | None:
try:
return self._output_queue.get_nowait()
+13 -2
View File
@@ -203,9 +203,11 @@ class AsyncOmniEngine:
engine_args: OmniEngineArgs | None = None,
stage_init_timeout: int = 300,
init_timeout: int = 600,
diffusion_batch_size: int = 1,
**kwargs: Any,
) -> None:
self.model = model
self.diffusion_batch_size = diffusion_batch_size
startup_timeout = int(init_timeout)
logger.info(f"[AsyncOmniEngine] Initializing with model {model}")
@@ -473,8 +475,17 @@ class AsyncOmniEngine:
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
_inject_kv_stage_info(stage_cfg, stage_id)
stage_clients[stage_id] = initialize_diffusion_stage(self.model, stage_cfg, metadata)
logger.info("[AsyncOmniEngine] Stage %s initialized (diffusion)", stage_id)
stage_clients[stage_id] = initialize_diffusion_stage(
self.model,
stage_cfg,
metadata,
batch_size=self.diffusion_batch_size,
)
logger.info(
"[AsyncOmniEngine] Stage %s initialized (diffusion, batch_size=%d)",
stage_id,
self.diffusion_batch_size,
)
continue
llm_stage_ids.append(stage_id)
+16 -2
View File
@@ -485,7 +485,14 @@ class Orchestrator:
req_id,
)
await next_client.add_request_async(req_id, diffusion_prompt, params)
if isinstance(diffusion_prompt, list):
await next_client.add_batch_request_async(
req_id,
diffusion_prompt,
params,
)
else:
await next_client.add_request_async(req_id, diffusion_prompt, params)
req_state.stage_submit_ts[next_stage_id] = _time.time()
return
@@ -603,7 +610,14 @@ class Orchestrator:
request = prompt
stage_client = self.stage_clients[stage_id]
if stage_client.stage_type == "diffusion":
await stage_client.add_request_async(request_id, prompt, params)
if isinstance(prompt, list):
await stage_client.add_batch_request_async(
request_id,
prompt,
params,
)
else:
await stage_client.add_request_async(request_id, prompt, params)
else:
await stage_client.add_request_async(request)
+17 -3
View File
@@ -441,8 +441,22 @@ def get_stage_connector_spec(
return {}
def initialize_diffusion_stage(model: str, stage_cfg: Any, metadata: StageMetadata) -> Any:
"""Build a diffusion stage client."""
def initialize_diffusion_stage(
model: str,
stage_cfg: Any,
metadata: StageMetadata,
batch_size: int = 1,
) -> Any:
"""Build a diffusion stage client.
Args:
model: Model name or path.
stage_cfg: Stage configuration.
metadata: Extracted stage metadata.
batch_size: Maximum number of requests to batch together in the
diffusion engine. Passed through to ``StageDiffusionClient``
and ultimately to ``AsyncOmniDiffusion``.
"""
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
@@ -452,7 +466,7 @@ def initialize_diffusion_stage(model: str, stage_cfg: Any, metadata: StageMetada
)
if metadata.cfg_kv_collect_func is not None:
od_config.cfg_kv_collect_func = metadata.cfg_kv_collect_func
return StageDiffusionClient(model, od_config, metadata)
return StageDiffusionClient(model, od_config, metadata, batch_size=batch_size)
def close_started_llm_stage(started: StartedLlmStage) -> None:
+18 -10
View File
@@ -130,8 +130,7 @@ class AsyncOmni(EngineClient, OmniBase):
async def generate(
self,
prompt: OmniPromptType,
sampling_params: Any = None,
prompt: OmniPromptType | list[OmniPromptType],
request_id: str = "",
*,
prompt_text: str | None = None,
@@ -140,22 +139,31 @@ class AsyncOmni(EngineClient, OmniBase):
sampling_params_list: Sequence[OmniSamplingParams] | None = None,
output_modalities: list[str] | None = None,
) -> AsyncGenerator[OmniRequestOutput, None]:
"""Generate outputs for the given prompt asynchronously.
"""Generate outputs for the given prompt(s) asynchronously.
Coordinates multi-stage pipeline execution. Processes the prompt through
all stages in the pipeline and yields outputs as they become available.
Coordinates multi-stage pipeline execution. Processes the prompt
through all stages in the pipeline and yields outputs as they become
available.
**Batch mode (diffusion only):**
When *prompt* is a ``list``, all prompts are dispatched in a single
``DiffusionEngine.step()`` call at the diffusion stage. The combined
result is yielded as one ``OmniRequestOutput`` with all generated
images. Only a single *request_id* is used for the whole batch.
Args:
prompt: Prompt to process. Can be a text string, token IDs,
or multimodal prompt.
request_id: Unique identifier for this request
sampling_params_list: List of SamplingParams, one for each stage.
prompt: A single prompt **or** a list of prompts. A list
triggers batch mode when the diffusion stage is reached.
request_id: Unique identifier for this request.
sampling_params_list: List of SamplingParams, one per stage.
Must have the same length as the number of stages.
If None, uses default sampling params for each stage.
If *None*, uses default sampling params for each stage.
output_modalities: Optional list of output modalities.
Yields:
OmniRequestOutput objects as they are produced by each stage.
In batch mode the diffusion stage yields one output containing
all generated images.
Raises:
ValueError: If sampling_params_list has incorrect length.
+116 -5
View File
@@ -66,10 +66,14 @@ class AsyncOmniDiffusion:
self,
model: str,
od_config: OmniDiffusionConfig | None = None,
batch_size: int = 1,
**kwargs: Any,
):
self.model = model
# Set batch size (default 1 for backward compatibility)
self._batch_size = max(1, batch_size)
# Capture stage info from kwargs before they might be filtered out
stage_id = kwargs.get("stage_id")
engine_input_source = kwargs.get("engine_input_source")
@@ -147,7 +151,109 @@ class AsyncOmniDiffusion:
self._executor,
)
logger.info("AsyncOmniDiffusion initialized with model: %s", model)
logger.info("AsyncOmniDiffusion initialized with model: %s, batch_size: %d", model, self._batch_size)
# ------------------------------------------------------------------
# batch_size property
# ------------------------------------------------------------------
@property
def batch_size(self) -> int:
"""Return the configured batch size for request batching."""
return self._batch_size
@batch_size.setter
def batch_size(self, value: int) -> None:
if not isinstance(value, int) or value < 1:
raise ValueError("batch_size must be a positive integer")
self._batch_size = value
# ------------------------------------------------------------------
# Public batch generation API
# ------------------------------------------------------------------
async def generate_batch(
self,
prompts: list[OmniPromptType],
sampling_params: OmniDiffusionSamplingParams,
request_id: str | None = None,
lora_request: LoRARequest | None = None,
) -> OmniRequestOutput:
"""Generate images from multiple prompts in a single engine call.
Batches the given prompts into **one** ``DiffusionEngine.step()``
call and returns a single ``OmniRequestOutput`` containing all
generated images. Called by ``StageDiffusionClient._run_batch``
when the orchestrator receives a list-prompt request.
Args:
prompts: List of text prompts describing the desired images.
sampling_params: Shared sampling parameters for all prompts.
request_id: Optional unique identifier. Auto-generated when *None*.
lora_request: Optional LoRA adapter to apply.
Returns:
A single ``OmniRequestOutput`` with all images combined.
"""
if request_id is None:
request_id = f"diff-batch-{uuid.uuid4().hex[:8]}"
return await self._generate_batch(prompts, sampling_params, request_id, lora_request)
# ------------------------------------------------------------------
# Internal batch generation
# ------------------------------------------------------------------
async def _generate_batch(
self,
prompts: list[OmniPromptType],
sampling_params: OmniDiffusionSamplingParams,
request_id: str,
lora_request: LoRARequest | None = None,
) -> OmniRequestOutput:
"""Generate images from multiple prompts in a single engine call."""
if not prompts:
return OmniRequestOutput(request_id=request_id, images=[], final_output_type="image")
if sampling_params.guidance_scale:
sampling_params.guidance_scale_provided = True
if lora_request is not None:
sampling_params.lora_request = lora_request
request = OmniDiffusionRequest(
prompts=prompts,
sampling_params=sampling_params,
request_ids=[f"{request_id}-{i}" for i in range(len(prompts))],
)
logger.debug("Starting batch generation for %d prompts, request_id=%s", len(prompts), request_id)
loop = asyncio.get_event_loop()
try:
results = await loop.run_in_executor(
self._executor,
self.engine.step,
request,
)
except Exception as e:
logger.error("Batch generation failed for request %s: %s", request_id, e)
raise RuntimeError(f"Diffusion batch generation failed: {e}") from e
# Combine all per-prompt results into a single OmniRequestOutput
all_images = []
for result in results:
all_images.extend(result.images)
return OmniRequestOutput(
request_id=request_id,
images=all_images,
final_output_type="image",
finished=True,
)
# ------------------------------------------------------------------
# Public generate API
# ------------------------------------------------------------------
async def generate(
self,
@@ -156,12 +262,17 @@ class AsyncOmniDiffusion:
request_id: str | None = None,
lora_request: LoRARequest | None = None,
) -> OmniRequestOutput:
"""Generate images asynchronously from a text prompt.
"""Generate images asynchronously from a single text prompt.
For batched generation (multiple prompts in one engine call), use
:meth:`generate_batch` instead. This method always processes
exactly one prompt per call.
Args:
prompt: Text prompt describing the desired image
sampling_params: Sampling parameters
request_id: Optional unique identifier for tracking the request
lora_request: Optional LoRA adapter to apply
Returns:
OmniRequestOutput containing generated images
@@ -171,6 +282,8 @@ class AsyncOmniDiffusion:
"""
if request_id is None:
request_id = f"diff-{uuid.uuid4().hex[:16]}"
if sampling_params.guidance_scale:
sampling_params.guidance_scale_provided = True
if lora_request is not None:
sampling_params.lora_request = lora_request
@@ -183,10 +296,8 @@ class AsyncOmniDiffusion:
logger.debug("Starting generation for request %s", request_id)
# Run engine in thread pool
loop = asyncio.get_event_loop()
try:
# In async mode, only a single request is submitted at a time
result = await loop.run_in_executor(
self._executor,
self.engine.step,
@@ -197,7 +308,6 @@ class AsyncOmniDiffusion:
logger.error("Generation failed for request %s: %s", request_id, e)
raise RuntimeError(f"Diffusion generation failed: {e}") from e
# Update request_id if needed
if not result.request_id:
result.request_id = request_id
return result
@@ -233,6 +343,7 @@ class AsyncOmniDiffusion:
if self._closed:
return
self._closed = True
finalizer = getattr(self, "_weak_finalizer", None)
if finalizer is not None and finalizer.alive:
finalizer.detach()
+3
View File
@@ -79,6 +79,7 @@ class OmniBase:
log_stats = kwargs.pop("log_stats", False)
async_chunk = kwargs.pop("async_chunk", False)
output_modalities = kwargs.pop("output_modalities", None)
diffusion_batch_size: int = kwargs.pop("diffusion_batch_size", 1)
if "log_requests" in kwargs:
raise TypeError("`log_requests` has been removed in Omni/AsyncOmni. Use `log_stats`.")
@@ -95,6 +96,7 @@ class OmniBase:
engine_args=engine_args,
init_timeout=init_timeout,
stage_init_timeout=stage_init_timeout,
diffusion_batch_size=diffusion_batch_size,
**kwargs,
)
self._shutdown_called = False
@@ -273,6 +275,7 @@ class OmniBase:
images = getattr(engine_outputs, "images", []) if stage_meta["final_output_type"] == "image" else []
return OmniRequestOutput(
request_id=req_id or "",
stage_id=stage_id,
final_output_type=stage_meta["final_output_type"],
request_output=engine_outputs,