diff --git a/.buildkite/test-ready.yml b/.buildkite/test-ready.yml index a772e673e..3719b5d3c 100644 --- a/.buildkite/test-ready.yml +++ b/.buildkite/test-ready.yml @@ -50,6 +50,23 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Batching Test" + depends_on: upload-ready-pipeline + commands: + - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model" + agents: + queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Model CPU offloading Test" depends_on: upload-ready-pipeline commands: diff --git a/tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py b/tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py new file mode 100644 index 000000000..e5c738726 --- /dev/null +++ b/tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py @@ -0,0 +1,641 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +End-to-end test for diffusion batching via AsyncOmni. + +This test fires multiple concurrent ``AsyncOmni.generate()`` calls for a +diffusion model and validates that every caller receives its correct +individual result. When the underlying diffusion stage is configured with +``batch_size > 1`` (via stage config or ``StageDiffusionClient``), the +requests will be batched internally. + +Even without explicit batching config this test is useful for verifying +that concurrent async requests are handled correctly. + +Usage (standalone): + + python tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py \ + --model \ + --num-prompts 8 + +Or via pytest: + + pytest tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -s +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys +import time +import uuid +from pathlib import Path + +import pytest +import torch + +from tests.utils import hardware_test +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.platforms import current_omni_platform + +# ruff: noqa: E402 +REPO_ROOT = Path(__file__).resolve().parents[3] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from vllm_omni import Omni + +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" + +# ------------------------------------------------------------------ +models = ["tiny-random/Qwen-Image"] + +# ------------------------------------------------------------------ +# Prompt fixtures +# ------------------------------------------------------------------ + +WARMUP_PROMPTS: list[dict[str, str]] = [ + {"prompt": "a sunflower in a glass vase", "negative_prompt": "blurry"}, + {"prompt": "a rocket launching into space", "negative_prompt": "low detail"}, + {"prompt": "a small cottage in the snowy mountains", "negative_prompt": "foggy"}, + {"prompt": "a colorful parrot sitting on a tree branch", "negative_prompt": "low contrast"}, +] + +TEST_PROMPTS: list[dict[str, str]] = [ + {"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"}, + {"prompt": "a toy dinosaur on a sandy beach", "negative_prompt": "cinematic, realistic"}, + {"prompt": "a futuristic city skyline at sunset", "negative_prompt": "blurry, foggy"}, + {"prompt": "a bowl of fresh strawberries", "negative_prompt": "low detail"}, + {"prompt": "a medieval knight standing in the rain", "negative_prompt": "modern clothing"}, + {"prompt": "a cat wearing sunglasses lounging in a garden", "negative_prompt": "dark lighting"}, + {"prompt": "a spaceship flying above a volcano", "negative_prompt": "low contrast"}, + {"prompt": "a watercolor painting of a mountain lake", "negative_prompt": "photo, realistic"}, +] + + +# ------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------ + + +def _default_sampling_params(**overrides) -> OmniDiffusionSamplingParams: + defaults = dict( + num_inference_steps=2, + width=256, + height=256, + guidance_scale=0.0, + ) + defaults.update(overrides) + return OmniDiffusionSamplingParams(**defaults) + + +def _default_sync_sampling_params(**overrides) -> OmniDiffusionSamplingParams: + """Create sampling params for the synchronous Omni.generate() API.""" + defaults = dict( + num_inference_steps=2, + width=256, + height=256, + guidance_scale=0.0, + generator=torch.Generator(current_omni_platform.device_type).manual_seed(42), + ) + defaults.update(overrides) + return OmniDiffusionSamplingParams(**defaults) + + +async def _collect_generate(omni: AsyncOmni, prompt, request_id, sampling_params_list) -> OmniRequestOutput: + """Consume the AsyncOmni.generate() async generator and return the last output.""" + last_output: OmniRequestOutput | None = None + async for output in omni.generate( + prompt=prompt, + request_id=request_id, + sampling_params_list=sampling_params_list, + ): + last_output = output + if last_output is None: + raise RuntimeError(f"No output received for request {request_id}") + return last_output + + +def _extract_images(output: OmniRequestOutput) -> list: + """Extract images from an OmniRequestOutput, handling both direct + and nested request_output structures.""" + if output.images: + return output.images + # When the output comes from the orchestrator pipeline, images may be + # nested inside request_output. + inner = getattr(output, "request_output", None) + if inner is not None and hasattr(inner, "images") and inner.images: + return inner.images + return [] + + +# ------------------------------------------------------------------ +# Warm-up (async) +# ------------------------------------------------------------------ + + +async def warmup(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None: + """Warm-up: send prompts in parallel to pre-load the model.""" + print(f"🔥 Warming up with {len(prompts)} prompts ...") + sp = _default_sampling_params(num_inference_steps=2) + start = time.perf_counter() + + tasks = [ + _collect_generate( + omni, + prompt=p, + request_id=f"warmup-{i}-{uuid.uuid4().hex[:8]}", + sampling_params_list=[sp], + ) + for i, p in enumerate(prompts) + ] + await asyncio.gather(*tasks) + + elapsed = time.perf_counter() - start + print(f" Warm-up done in {elapsed:.2f}s\n") + + +# ------------------------------------------------------------------ +# Single (sequential) benchmark +# ------------------------------------------------------------------ + + +async def run_single(omni: AsyncOmni, prompts: list[dict[str, str]]) -> float: + """Run prompts one-by-one sequentially.""" + print(f"🧩 Running SINGLE (sequential) mode – {len(prompts)} prompts ...") + sp = _default_sampling_params() + total_start = time.perf_counter() + + for i, prompt in enumerate(prompts): + start = time.perf_counter() + result = await _collect_generate( + omni, + prompt=prompt, + request_id=f"single-{i}-{uuid.uuid4().hex[:8]}", + sampling_params_list=[sp], + ) + elapsed = time.perf_counter() - start + images = _extract_images(result) + print(f" prompt {i}: {elapsed:.2f}s ({len(images)} images)") + + total = time.perf_counter() - total_start + print(f" ✅ Total single-mode: {total:.2f}s\n") + return total + + +# ------------------------------------------------------------------ +# Batch (parallel) benchmark — concurrent individual requests +# ------------------------------------------------------------------ + + +async def run_batch( + omni: AsyncOmni, + prompts: list[dict[str, str]], + label: str = "batch", +) -> float: + """Send all prompts concurrently via asyncio.gather (one request per prompt).""" + print(f"⚙️ Running {label.upper()} mode – {len(prompts)} prompts concurrently ...") + sp = _default_sampling_params() + start = time.perf_counter() + + tasks = [ + _collect_generate( + omni, + prompt=p, + request_id=f"{label}-{i}-{uuid.uuid4().hex[:8]}", + sampling_params_list=[sp], + ) + for i, p in enumerate(prompts) + ] + results = await asyncio.gather(*tasks) + + elapsed = time.perf_counter() - start + for i, result in enumerate(results): + images = _extract_images(result) + print(f" prompt {i}: {len(images)} images, request_id={result.request_id}") + + print(f" ✅ Total {label} mode: {elapsed:.2f}s\n") + return elapsed + + +# ------------------------------------------------------------------ +# Explicit batch — single generate() call with list of prompts +# ------------------------------------------------------------------ + + +async def run_batch_explicit( + omni: AsyncOmni, + prompts: list[dict[str, str]], + label: str = "batch_explicit", +) -> float: + """Send all prompts as a single batch via generate(prompt=[...]). + + Passes a *list* of prompts so that all are processed in **one** + ``DiffusionEngine.step()`` call. The orchestrator detects + ``isinstance(prompt, list)`` and routes to the batch path. + + A single ``OmniRequestOutput`` is yielded containing ALL generated + images combined. + """ + print(f"⚡ Running {label.upper()} mode – {len(prompts)} prompts in ONE engine call ...") + sp = _default_sampling_params() + request_id = f"{label}-{uuid.uuid4().hex[:8]}" + start = time.perf_counter() + + result: OmniRequestOutput | None = None + async for output in omni.generate( + prompt=prompts, + request_id=request_id, + sampling_params_list=[sp], + ): + result = output + + elapsed = time.perf_counter() - start + if result is not None: + images = _extract_images(result) + print(f" Got {len(images)} images total from batch, request_id={result.request_id}") + else: + print(" ⚠️ No output received from batch generate()") + + print(f" ✅ Total {label} mode: {elapsed:.2f}s\n") + return elapsed + + +# ------------------------------------------------------------------ +# Async validation helpers +# ------------------------------------------------------------------ + + +async def validate_batch_explicit(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None: + """Validate generate(prompt=[...]) returns a single result with all images.""" + print(f"🔍 Validating batch generate() correctness with {len(prompts)} prompts ...") + sp = _default_sampling_params() + request_id = f"validate-batch-{uuid.uuid4().hex[:8]}" + + result: OmniRequestOutput | None = None + async for output in omni.generate( + prompt=prompts, + request_id=request_id, + sampling_params_list=[sp], + ): + result = output + + assert result is not None, "No output received from batch generate()" + images = _extract_images(result) + # Batch mode returns ONE output with ALL images combined + assert len(images) == len(prompts), f"Expected {len(prompts)} images (one per prompt), got {len(images)}" + assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}" + print(f" ✅ Batch returned {len(images)} images with correct request_id.\n") + + +async def validate_concurrent(omni: AsyncOmni, prompts: list[dict[str, str]]) -> None: + """Validate that every concurrent request receives a distinct result + with its own request_id.""" + print(f"🔍 Validating concurrent correctness with {len(prompts)} prompts ...") + sp = _default_sampling_params() + + request_ids = [f"validate-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))] + + tasks = [ + _collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp]) + for p, rid in zip(prompts, request_ids) + ] + results = await asyncio.gather(*tasks) + + assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}" + + returned_ids = [r.request_id for r in results] + for rid in request_ids: + assert rid in returned_ids, f"Missing request_id {rid} in results" + + print(" ✅ All request_ids matched, results count correct.\n") + + +# ------------------------------------------------------------------ +# Single vs Parallel comparison (CLI only) +# ------------------------------------------------------------------ + + +async def compare_single_vs_parallel( + model: str, + prompts: list[dict[str, str]], + batch_size: int = 1, +) -> None: + """Run the same prompts sequentially then in parallel and print a comparison.""" + + omni = AsyncOmni(model=model, diffusion_batch_size=batch_size) + try: + await warmup(omni, WARMUP_PROMPTS) + single_time = await run_single(omni, prompts) + parallel_time = await run_batch(omni, prompts, label="parallel") + explicit_time = await run_batch_explicit(omni, prompts, label="batch_explicit") + finally: + omni.shutdown() + + speedup_parallel = single_time / parallel_time if parallel_time > 0 else float("inf") + speedup_explicit = single_time / explicit_time if explicit_time > 0 else float("inf") + print("=" * 60) + print(f"📊 Summary ({len(prompts)} prompts)") + print(f" Sequential : {single_time:.2f}s") + print(f" Parallel (gather) : {parallel_time:.2f}s ({speedup_parallel:.2f}x)") + print(f" Explicit batch : {explicit_time:.2f}s ({speedup_explicit:.2f}x)") + print("=" * 60) + + +# ------------------------------------------------------------------ +# CLI main entrypoint +# ------------------------------------------------------------------ + + +async def main(model: str, num_prompts: int, mode: str, batch_size: int = 1) -> None: + prompts = (TEST_PROMPTS * ((num_prompts // len(TEST_PROMPTS)) + 1))[:num_prompts] + + if mode == "compare": + await compare_single_vs_parallel(model, prompts, batch_size=batch_size) + return + + omni = AsyncOmni(model=model, diffusion_batch_size=batch_size) + try: + await warmup(omni, WARMUP_PROMPTS) + + if mode == "validate": + await validate_concurrent(omni, prompts) + elif mode == "validate_batch": + await validate_batch_explicit(omni, prompts) + elif mode == "batch": + await run_batch(omni, prompts, label="measurement") + elif mode == "batch_explicit": + await run_batch_explicit(omni, prompts) + elif mode == "single": + await run_single(omni, prompts) + else: + raise ValueError(f"Unknown mode: {mode}") + finally: + omni.shutdown() + + +# ================================================================== +# pytest test cases +# ================================================================== + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}) +@pytest.mark.parametrize("model_name", models) +def test_diffusion_batching_sync_sequential(model_name: str): + """Test that synchronous Omni can generate images for multiple prompts + submitted sequentially (one at a time) and each returns a valid image.""" + m = None + try: + m = Omni(model=model_name) + sp = _default_sync_sampling_params() + prompts = TEST_PROMPTS[:4] + + for i, prompt in enumerate(prompts): + outputs = m.generate(prompt, sp) + first_output = outputs[0] + assert first_output.final_output_type == "image", ( + f"Expected 'image', got '{first_output.final_output_type}'" + ) + + # Images are surfaced both at top-level and inside request_output + images = _extract_images(first_output) + assert len(images) >= 1, f"Expected at least 1 image for prompt {i}, got {len(images)}" + assert images[0].width == 256 + assert images[0].height == 256 + print(f" prompt {i}: OK ({len(images)} images)") + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}) +@pytest.mark.parametrize("model_name", models) +def test_diffusion_batching_sync_multi_prompt(model_name: str): + """Test that synchronous Omni correctly handles a list of multiple + prompts submitted at once and returns one result per prompt. + + Note: Omni.generate() iterates the list and submits each prompt + individually with its own request_id. This tests concurrent request + handling at the diffusion stage, not the explicit list-batch path + (which is only available via AsyncOmni). + """ + m = None + try: + m = Omni(model=model_name) + sp = _default_sync_sampling_params() + prompts = TEST_PROMPTS[:4] + + outputs = m.generate(prompts, sp) + assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}" + + for i, output in enumerate(outputs): + assert output.final_output_type == "image", ( + f"Output {i} final_output_type expected 'image', got '{output.final_output_type}'" + ) + images = _extract_images(output) + assert images and len(images) >= 1, f"Expected at least 1 image for prompt {i}" + assert images[0].width == 256 + assert images[0].height == 256 + print(f" prompt {i}: OK ({len(images)} images, request_id={output.request_id})") + + # Verify all request_ids are distinct + request_ids = [o.request_id for o in outputs] + assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids found: {request_ids}" + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}) +@pytest.mark.parametrize("model_name", models) +def test_diffusion_batching_async_concurrent(model_name: str): + """Test that AsyncOmni correctly handles multiple concurrent requests + fired via asyncio.gather. Each request_id must appear in the results.""" + + async def _inner(): + omni = AsyncOmni(model=model_name, diffusion_batch_size=1) + try: + prompts = TEST_PROMPTS[:4] + sp = _default_sampling_params() + request_ids = [f"async-concurrent-{i}-{uuid.uuid4().hex[:8]}" for i in range(len(prompts))] + + tasks = [ + _collect_generate(omni, prompt=p, request_id=rid, sampling_params_list=[sp]) + for p, rid in zip(prompts, request_ids) + ] + results = await asyncio.gather(*tasks) + + assert len(results) == len(prompts), f"Expected {len(prompts)} results, got {len(results)}" + + returned_ids = [r.request_id for r in results] + for rid in request_ids: + assert rid in returned_ids, f"Missing request_id {rid} in results" + + for i, result in enumerate(results): + images = _extract_images(result) + assert len(images) >= 1, f"No images for prompt {i}" + assert images[0].width == 256 + assert images[0].height == 256 + print(f" prompt {i}: OK ({len(images)} images, request_id={result.request_id})") + finally: + omni.shutdown() + + asyncio.run(_inner()) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}) +@pytest.mark.parametrize("model_name", models) +def test_diffusion_batching_async_explicit_batch(model_name: str): + """Test that AsyncOmni batch mode (generate(prompt=[...])) dispatches + all prompts in a single engine call and returns a single combined result. + + The list-prompt path routes through the orchestrator's + ``add_batch_request_async`` → ``AsyncOmniDiffusion.generate_batch`` + and yields ONE ``OmniRequestOutput`` with ALL images combined. + """ + + async def _inner(): + omni = AsyncOmni(model=model_name, diffusion_batch_size=4) + try: + prompts = TEST_PROMPTS[:4] + sp = _default_sampling_params() + request_id = f"explicit-batch-{uuid.uuid4().hex[:8]}" + + # Batch mode: pass list of prompts → single request_id + result: OmniRequestOutput | None = None + async for output in omni.generate( + prompt=prompts, + request_id=request_id, + sampling_params_list=[sp], + ): + result = output + + assert result is not None, "No output received from batch generate()" + + images = _extract_images(result) + # One image per prompt, all in a single output + assert len(images) == len(prompts), f"Expected {len(prompts)} images in combined output, got {len(images)}" + assert result.request_id == request_id, f"Expected request_id={request_id}, got {result.request_id}" + for i, img in enumerate(images): + assert img.width == 256, f"Image {i} width mismatch" + assert img.height == 256, f"Image {i} height mismatch" + print(f" ✅ Batch returned {len(images)} images, request_id={result.request_id}") + finally: + omni.shutdown() + + asyncio.run(_inner()) + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}) +@pytest.mark.parametrize("model_name", models) +def test_diffusion_batching_num_outputs(model_name: str): + """Test that the diffusion model respects num_outputs_per_prompt and + generates the correct number of images per request.""" + m = None + try: + m = Omni(model=model_name) + num_outputs = 2 + sp = _default_sync_sampling_params(num_outputs_per_prompt=num_outputs) + + outputs = m.generate( + "a photo of a cat sitting on a laptop keyboard", + sp, + ) + + first_output = outputs[0] + assert first_output.final_output_type == "image" + images = _extract_images(first_output) + assert images is not None and len(images) == num_outputs, ( + f"Expected {num_outputs} images, got {len(images) if images else 0}" + ) + for img in images: + assert img.width == 256 + assert img.height == 256 + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() + + +@pytest.mark.core_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}) +@pytest.mark.parametrize("model_name", models) +def test_diffusion_batching_distinct_results(model_name: str): + """Test that different prompts produce distinct images when batched, + ensuring the batching logic does not mix up results across requests.""" + m = None + try: + m = Omni(model=model_name) + sp = _default_sync_sampling_params() + prompts = [ + {"prompt": "a bright red apple on a white table", "negative_prompt": "blurry"}, + {"prompt": "a blue ocean with white waves crashing", "negative_prompt": "blurry"}, + ] + + outputs = m.generate(prompts, sp) + assert len(outputs) == len(prompts), f"Expected {len(prompts)} outputs, got {len(outputs)}" + + # Verify each output has a unique request_id + request_ids = [o.request_id for o in outputs] + assert len(set(request_ids)) == len(request_ids), f"Duplicate request_ids: {request_ids}" + + # Verify each output has images + for i, output in enumerate(outputs): + images = _extract_images(output) + assert images and len(images) >= 1, f"No images for prompt {i}" + assert images[0].width == 256 + assert images[0].height == 256 + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() + + +# ------------------------------------------------------------------ +# CLI +# ------------------------------------------------------------------ + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="E2E diffusion concurrent benchmark / validation") + parser.add_argument("--model", type=str, required=True, help="Model name or path") + parser.add_argument("--num-prompts", type=int, default=8, help="Number of prompts to run") + parser.add_argument("--batch-size", type=int, default=1, help="Diffusion batch size (1 = no batching)") + parser.add_argument( + "--mode", + choices=["batch", "batch_explicit", "single", "compare", "validate", "validate_batch"], + default="compare", + help=( + "Run mode: 'batch' (parallel gather), 'batch_explicit' (list-prompt batch API), " + "'single' (sequential), 'compare' (all three), " + "'validate' (concurrent correctness), 'validate_batch' (list-prompt correctness)" + ), + ) + args = parser.parse_args() + + asyncio.run(main(args.model, args.num_prompts, args.mode, batch_size=args.batch_size)) diff --git a/vllm_omni/diffusion/stage_diffusion_client.py b/vllm_omni/diffusion/stage_diffusion_client.py index 85cfd1be4..317d05630 100644 --- a/vllm_omni/diffusion/stage_diffusion_client.py +++ b/vllm_omni/diffusion/stage_diffusion_client.py @@ -37,6 +37,7 @@ class StageDiffusionClient: model: str, od_config: OmniDiffusionConfig, metadata: StageMetadata, + batch_size: int = 1, ) -> None: self.stage_id = metadata.stage_id self.final_output = metadata.final_output @@ -45,11 +46,11 @@ class StageDiffusionClient: self.custom_process_input_func = metadata.custom_process_input_func self.engine_input_source = metadata.engine_input_source - self._engine = AsyncOmniDiffusion(model=model, od_config=od_config) + self._engine = AsyncOmniDiffusion(model=model, od_config=od_config, batch_size=batch_size) self._output_queue: asyncio.Queue[OmniRequestOutput] = asyncio.Queue() self._tasks: dict[str, asyncio.Task] = {} - logger.info("[StageDiffusionClient] Stage-%s initialized", self.stage_id) + logger.info("[StageDiffusionClient] Stage-%s initialized (batch_size=%d)", self.stage_id, batch_size) async def add_request_async( self, @@ -82,6 +83,49 @@ class StageDiffusionClient: finally: self._tasks.pop(request_id, None) + # TODO(Long): Temporary solution to boost performance of diffusion stages. + # Remove this after scheduling algorithm is implemented + async def add_batch_request_async( + self, + request_id: str, + prompts: list[OmniPromptType], + sampling_params: OmniDiffusionSamplingParams, + ) -> None: + """Submit a list of prompts as a single batched engine call. + + All prompts are processed in one ``DiffusionEngine.step()`` call + and the combined result is placed on the output queue with a single + *request_id*. + """ + task = asyncio.create_task( + self._run_batch(request_id, prompts, sampling_params), + name=f"diffusion-batch-{request_id}", + ) + self._tasks[request_id] = task + + async def _run_batch( + self, + request_id: str, + prompts: list[OmniPromptType], + sampling_params: OmniDiffusionSamplingParams, + ) -> None: + try: + result = await self._engine.generate_batch( + prompts, + sampling_params, + request_id, + ) + await self._output_queue.put(result) + except Exception as e: + logger.exception( + "[StageDiffusionClient] Stage-%s batch req=%s failed: %s", + self.stage_id, + request_id, + e, + ) + finally: + self._tasks.pop(request_id, None) + def get_diffusion_output_async(self) -> OmniRequestOutput | None: try: return self._output_queue.get_nowait() diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 5562b84ff..75203bee7 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -203,9 +203,11 @@ class AsyncOmniEngine: engine_args: OmniEngineArgs | None = None, stage_init_timeout: int = 300, init_timeout: int = 600, + diffusion_batch_size: int = 1, **kwargs: Any, ) -> None: self.model = model + self.diffusion_batch_size = diffusion_batch_size startup_timeout = int(init_timeout) logger.info(f"[AsyncOmniEngine] Initializing with model {model}") @@ -473,8 +475,17 @@ class AsyncOmniEngine: inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to) _inject_kv_stage_info(stage_cfg, stage_id) - stage_clients[stage_id] = initialize_diffusion_stage(self.model, stage_cfg, metadata) - logger.info("[AsyncOmniEngine] Stage %s initialized (diffusion)", stage_id) + stage_clients[stage_id] = initialize_diffusion_stage( + self.model, + stage_cfg, + metadata, + batch_size=self.diffusion_batch_size, + ) + logger.info( + "[AsyncOmniEngine] Stage %s initialized (diffusion, batch_size=%d)", + stage_id, + self.diffusion_batch_size, + ) continue llm_stage_ids.append(stage_id) diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index a94879133..e6373ec96 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -485,7 +485,14 @@ class Orchestrator: req_id, ) - await next_client.add_request_async(req_id, diffusion_prompt, params) + if isinstance(diffusion_prompt, list): + await next_client.add_batch_request_async( + req_id, + diffusion_prompt, + params, + ) + else: + await next_client.add_request_async(req_id, diffusion_prompt, params) req_state.stage_submit_ts[next_stage_id] = _time.time() return @@ -603,7 +610,14 @@ class Orchestrator: request = prompt stage_client = self.stage_clients[stage_id] if stage_client.stage_type == "diffusion": - await stage_client.add_request_async(request_id, prompt, params) + if isinstance(prompt, list): + await stage_client.add_batch_request_async( + request_id, + prompt, + params, + ) + else: + await stage_client.add_request_async(request_id, prompt, params) else: await stage_client.add_request_async(request) diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index a15eb08c8..cd1331dfb 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -441,8 +441,22 @@ def get_stage_connector_spec( return {} -def initialize_diffusion_stage(model: str, stage_cfg: Any, metadata: StageMetadata) -> Any: - """Build a diffusion stage client.""" +def initialize_diffusion_stage( + model: str, + stage_cfg: Any, + metadata: StageMetadata, + batch_size: int = 1, +) -> Any: + """Build a diffusion stage client. + + Args: + model: Model name or path. + stage_cfg: Stage configuration. + metadata: Extracted stage metadata. + batch_size: Maximum number of requests to batch together in the + diffusion engine. Passed through to ``StageDiffusionClient`` + and ultimately to ``AsyncOmniDiffusion``. + """ from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient @@ -452,7 +466,7 @@ def initialize_diffusion_stage(model: str, stage_cfg: Any, metadata: StageMetada ) if metadata.cfg_kv_collect_func is not None: od_config.cfg_kv_collect_func = metadata.cfg_kv_collect_func - return StageDiffusionClient(model, od_config, metadata) + return StageDiffusionClient(model, od_config, metadata, batch_size=batch_size) def close_started_llm_stage(started: StartedLlmStage) -> None: diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 18b4f2003..73f7ad260 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -130,8 +130,7 @@ class AsyncOmni(EngineClient, OmniBase): async def generate( self, - prompt: OmniPromptType, - sampling_params: Any = None, + prompt: OmniPromptType | list[OmniPromptType], request_id: str = "", *, prompt_text: str | None = None, @@ -140,22 +139,31 @@ class AsyncOmni(EngineClient, OmniBase): sampling_params_list: Sequence[OmniSamplingParams] | None = None, output_modalities: list[str] | None = None, ) -> AsyncGenerator[OmniRequestOutput, None]: - """Generate outputs for the given prompt asynchronously. + """Generate outputs for the given prompt(s) asynchronously. - Coordinates multi-stage pipeline execution. Processes the prompt through - all stages in the pipeline and yields outputs as they become available. + Coordinates multi-stage pipeline execution. Processes the prompt + through all stages in the pipeline and yields outputs as they become + available. + + **Batch mode (diffusion only):** + When *prompt* is a ``list``, all prompts are dispatched in a single + ``DiffusionEngine.step()`` call at the diffusion stage. The combined + result is yielded as one ``OmniRequestOutput`` with all generated + images. Only a single *request_id* is used for the whole batch. Args: - prompt: Prompt to process. Can be a text string, token IDs, - or multimodal prompt. - request_id: Unique identifier for this request - sampling_params_list: List of SamplingParams, one for each stage. + prompt: A single prompt **or** a list of prompts. A list + triggers batch mode when the diffusion stage is reached. + request_id: Unique identifier for this request. + sampling_params_list: List of SamplingParams, one per stage. Must have the same length as the number of stages. - If None, uses default sampling params for each stage. + If *None*, uses default sampling params for each stage. output_modalities: Optional list of output modalities. Yields: OmniRequestOutput objects as they are produced by each stage. + In batch mode the diffusion stage yields one output containing + all generated images. Raises: ValueError: If sampling_params_list has incorrect length. diff --git a/vllm_omni/entrypoints/async_omni_diffusion.py b/vllm_omni/entrypoints/async_omni_diffusion.py index 52bd6031c..7dd213738 100644 --- a/vllm_omni/entrypoints/async_omni_diffusion.py +++ b/vllm_omni/entrypoints/async_omni_diffusion.py @@ -66,10 +66,14 @@ class AsyncOmniDiffusion: self, model: str, od_config: OmniDiffusionConfig | None = None, + batch_size: int = 1, **kwargs: Any, ): self.model = model + # Set batch size (default 1 for backward compatibility) + self._batch_size = max(1, batch_size) + # Capture stage info from kwargs before they might be filtered out stage_id = kwargs.get("stage_id") engine_input_source = kwargs.get("engine_input_source") @@ -147,7 +151,109 @@ class AsyncOmniDiffusion: self._executor, ) - logger.info("AsyncOmniDiffusion initialized with model: %s", model) + logger.info("AsyncOmniDiffusion initialized with model: %s, batch_size: %d", model, self._batch_size) + + # ------------------------------------------------------------------ + # batch_size property + # ------------------------------------------------------------------ + + @property + def batch_size(self) -> int: + """Return the configured batch size for request batching.""" + return self._batch_size + + @batch_size.setter + def batch_size(self, value: int) -> None: + if not isinstance(value, int) or value < 1: + raise ValueError("batch_size must be a positive integer") + self._batch_size = value + + # ------------------------------------------------------------------ + # Public batch generation API + # ------------------------------------------------------------------ + + async def generate_batch( + self, + prompts: list[OmniPromptType], + sampling_params: OmniDiffusionSamplingParams, + request_id: str | None = None, + lora_request: LoRARequest | None = None, + ) -> OmniRequestOutput: + """Generate images from multiple prompts in a single engine call. + + Batches the given prompts into **one** ``DiffusionEngine.step()`` + call and returns a single ``OmniRequestOutput`` containing all + generated images. Called by ``StageDiffusionClient._run_batch`` + when the orchestrator receives a list-prompt request. + + Args: + prompts: List of text prompts describing the desired images. + sampling_params: Shared sampling parameters for all prompts. + request_id: Optional unique identifier. Auto-generated when *None*. + lora_request: Optional LoRA adapter to apply. + + Returns: + A single ``OmniRequestOutput`` with all images combined. + """ + if request_id is None: + request_id = f"diff-batch-{uuid.uuid4().hex[:8]}" + return await self._generate_batch(prompts, sampling_params, request_id, lora_request) + + # ------------------------------------------------------------------ + # Internal batch generation + # ------------------------------------------------------------------ + + async def _generate_batch( + self, + prompts: list[OmniPromptType], + sampling_params: OmniDiffusionSamplingParams, + request_id: str, + lora_request: LoRARequest | None = None, + ) -> OmniRequestOutput: + """Generate images from multiple prompts in a single engine call.""" + if not prompts: + return OmniRequestOutput(request_id=request_id, images=[], final_output_type="image") + + if sampling_params.guidance_scale: + sampling_params.guidance_scale_provided = True + + if lora_request is not None: + sampling_params.lora_request = lora_request + + request = OmniDiffusionRequest( + prompts=prompts, + sampling_params=sampling_params, + request_ids=[f"{request_id}-{i}" for i in range(len(prompts))], + ) + + logger.debug("Starting batch generation for %d prompts, request_id=%s", len(prompts), request_id) + + loop = asyncio.get_event_loop() + try: + results = await loop.run_in_executor( + self._executor, + self.engine.step, + request, + ) + except Exception as e: + logger.error("Batch generation failed for request %s: %s", request_id, e) + raise RuntimeError(f"Diffusion batch generation failed: {e}") from e + + # Combine all per-prompt results into a single OmniRequestOutput + all_images = [] + for result in results: + all_images.extend(result.images) + + return OmniRequestOutput( + request_id=request_id, + images=all_images, + final_output_type="image", + finished=True, + ) + + # ------------------------------------------------------------------ + # Public generate API + # ------------------------------------------------------------------ async def generate( self, @@ -156,12 +262,17 @@ class AsyncOmniDiffusion: request_id: str | None = None, lora_request: LoRARequest | None = None, ) -> OmniRequestOutput: - """Generate images asynchronously from a text prompt. + """Generate images asynchronously from a single text prompt. + + For batched generation (multiple prompts in one engine call), use + :meth:`generate_batch` instead. This method always processes + exactly one prompt per call. Args: prompt: Text prompt describing the desired image sampling_params: Sampling parameters request_id: Optional unique identifier for tracking the request + lora_request: Optional LoRA adapter to apply Returns: OmniRequestOutput containing generated images @@ -171,6 +282,8 @@ class AsyncOmniDiffusion: """ if request_id is None: request_id = f"diff-{uuid.uuid4().hex[:16]}" + if sampling_params.guidance_scale: + sampling_params.guidance_scale_provided = True if lora_request is not None: sampling_params.lora_request = lora_request @@ -183,10 +296,8 @@ class AsyncOmniDiffusion: logger.debug("Starting generation for request %s", request_id) - # Run engine in thread pool loop = asyncio.get_event_loop() try: - # In async mode, only a single request is submitted at a time result = await loop.run_in_executor( self._executor, self.engine.step, @@ -197,7 +308,6 @@ class AsyncOmniDiffusion: logger.error("Generation failed for request %s: %s", request_id, e) raise RuntimeError(f"Diffusion generation failed: {e}") from e - # Update request_id if needed if not result.request_id: result.request_id = request_id return result @@ -233,6 +343,7 @@ class AsyncOmniDiffusion: if self._closed: return self._closed = True + finalizer = getattr(self, "_weak_finalizer", None) if finalizer is not None and finalizer.alive: finalizer.detach() diff --git a/vllm_omni/entrypoints/omni_base.py b/vllm_omni/entrypoints/omni_base.py index 458996ff4..ba77f83f4 100644 --- a/vllm_omni/entrypoints/omni_base.py +++ b/vllm_omni/entrypoints/omni_base.py @@ -79,6 +79,7 @@ class OmniBase: log_stats = kwargs.pop("log_stats", False) async_chunk = kwargs.pop("async_chunk", False) output_modalities = kwargs.pop("output_modalities", None) + diffusion_batch_size: int = kwargs.pop("diffusion_batch_size", 1) if "log_requests" in kwargs: raise TypeError("`log_requests` has been removed in Omni/AsyncOmni. Use `log_stats`.") @@ -95,6 +96,7 @@ class OmniBase: engine_args=engine_args, init_timeout=init_timeout, stage_init_timeout=stage_init_timeout, + diffusion_batch_size=diffusion_batch_size, **kwargs, ) self._shutdown_called = False @@ -273,6 +275,7 @@ class OmniBase: images = getattr(engine_outputs, "images", []) if stage_meta["final_output_type"] == "image" else [] return OmniRequestOutput( + request_id=req_id or "", stage_id=stage_id, final_output_type=stage_meta["final_output_type"], request_output=engine_outputs,