[Core][Frontend] Support request-level batching for diffusion pipelines (#4079)

Signed-off-by: jader <yjader@foxmail.com>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
JiangJie ZhangandSamit authored and GitHub committed 2026-06-29 18:09:19 +02:00
1 parent 504698dec8
commit c45ac74d43
123 files changed
+4160 -2233

No files matched your search

+106 -6
View File
@@ -17,6 +17,7 @@ import numpy as np
import PIL.Image
import torch
from vllm.logger import init_logger
from vllm.utils.import_utils import resolve_obj_by_qualname
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.diffusion.data import (
@@ -38,6 +39,7 @@ from vllm_omni.diffusion.output_formatter import (
normalize_diffusion_postprocess_output,
)
from vllm_omni.diffusion.registry import (
DiffusionModelRegistry,
get_diffusion_action_post_process_func,
get_diffusion_post_process_func,
get_diffusion_pre_process_func,
@@ -74,6 +76,37 @@ def _func_accepts_parameter(func: object | None, parameter_name: str) -> bool:
)
def _resolve_custom_pipeline_cls(custom_pipeline_args: dict[str, Any] | None) -> type | None:
if custom_pipeline_args is None:
return None
try:
pipeline_cls = custom_pipeline_args["pipeline_class"]
except KeyError as exc:
raise ValueError("custom_pipeline_args must include 'pipeline_class'.") from exc
if isinstance(pipeline_cls, type):
return pipeline_cls
if isinstance(pipeline_cls, str):
try:
return resolve_obj_by_qualname(pipeline_cls)
except (AttributeError, ImportError, ValueError) as exc:
raise ValueError(f"Failed to resolve custom diffusion pipeline class {pipeline_cls!r}.") from exc
raise TypeError(
f"custom_pipeline_args['pipeline_class'] must be a qualified name string or a class, "
f"got {type(pipeline_cls).__name__}"
)
def supports_request_batch(od_config: OmniDiffusionConfig) -> bool:
model_cls = _resolve_custom_pipeline_cls(getattr(od_config, "custom_pipeline_args", None))
if model_cls is None:
model_cls = DiffusionModelRegistry._try_load_model_cls(getattr(od_config, "model_class_name", None))
if model_cls is None:
return False
return bool(getattr(model_cls, "supports_request_batch", False))
def _move_tensor_tree_to_cpu(value: object) -> object:
if isinstance(value, torch.Tensor):
return value.cpu() if value.device.type != "cpu" else value
@@ -138,10 +171,7 @@ class DiffusionEngine:
StepScheduler() if self.step_execution else RequestScheduler()
)
self.scheduler.initialize(od_config)
if self.scheduler.max_num_running_reqs > 1 and not self.step_execution:
max_num_seqs = self.scheduler.max_num_running_reqs
self.scheduler.max_num_running_reqs = 1
logger.warning(f"Non-stepwise-execution does not support max-num-seqs={max_num_seqs}, set it to 1.")
self.supports_request_batch = False if self.step_execution else supports_request_batch(od_config)
self.main_loop: asyncio.AbstractEventLoop | None = None
self.stop_event: threading.Event | None = None
self.worker_thread: threading.Thread | None = None
@@ -161,9 +191,18 @@ class DiffusionEngine:
self._rpc_queue: queue.Queue[_RpcTask] = queue.Queue()
if self.step_execution:
self.execute_fn = self.executor.execute_step
elif self.supports_request_batch:
self.execute_fn = self.executor.execute_batch
else:
self.execute_fn = self.executor.execute_request
if self.supports_request_batch:
logger.info(
"[RequestBatch] engine init max_num_seqs=%s max_wait_ms=%s",
getattr(od_config, "max_num_seqs", None),
getattr(od_config, "request_batch_max_wait_ms", None),
)
try:
self._dummy_run()
except Exception as e:
@@ -348,6 +387,9 @@ class DiffusionEngine:
# Only RPC / abort work pending; loop back to drain it.
continue
if self.supports_request_batch:
self._wait_for_request_batch_admission_locked()
sched_output = self.scheduler.schedule()
if sched_output.is_empty:
@@ -390,6 +432,61 @@ class DiffusionEngine:
# Engine is stopping: fail any RPCs still queued so callers don't hang.
self._fail_pending_rpcs(RuntimeError("DiffusionEngine is shutting down."))
def _wait_for_request_batch_admission_locked(self) -> None:
"""Wait for compatible requests to accumulate before scheduling a wave.
Caller must hold ``self._cv``.
"""
if self.step_execution or not self.supports_request_batch:
return
max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0
if max_wait_s == 0:
return
max_batch = self.scheduler.max_num_running_reqs
waiting = self.scheduler.num_waiting_requests()
running = self.scheduler.num_running_requests()
if running > 0:
return
start = time.monotonic()
deadline = start + max_wait_s
last_waiting = -1
stable_since = start
# Require a short idle period with no queue growth so bursty HTTP
# ingress can land before the first schedule() of a wave.
stable_window_s = min(0.05, max_wait_s / 5.0)
while not self.stop_event.is_set():
waiting = self.scheduler.num_waiting_requests()
now = time.monotonic()
if waiting >= max_batch:
break
if waiting > 0 and (now - stable_since) >= stable_window_s:
break
if now >= deadline:
break
if waiting > last_waiting:
stable_since = now
last_waiting = waiting
remaining = deadline - now
self._cv.wait(timeout=min(remaining, 0.002))
waited_ms = (time.monotonic() - start) * 1000.0
final_waiting = self.scheduler.num_waiting_requests()
if final_waiting > 0:
logger.info(
"[RequestBatch] admission wait done waiting=%d max_batch=%d waited_ms=%.1f",
final_waiting,
max_batch,
waited_ms,
)
def _process_rpc_queue(self) -> None:
"""Execute pending collective_rpc tasks from the busy-loop thread.
@@ -680,7 +777,7 @@ class DiffusionEngine:
logger.info("Skipping dummy warmup run (num_frames=0)")
return
req = OmniDiffusionRequest(
prompts=[prompt],
prompt=prompt,
request_id=DUMMY_DIFFUSION_REQUEST_ID,
sampling_params=OmniDiffusionSamplingParams(
height=height,
@@ -930,9 +1027,12 @@ class DiffusionEngine:
raise RuntimeError(f"Diffusion scheduler lost state for request {request_id}.")
if state.status == DiffusionRequestStatus.FINISHED_ABORTED:
# Preserve runner-provided abort details when available.
if runner_output is not None and runner_output.result is not None and runner_output.result.aborted:
return runner_output.result
return DiffusionOutput(
aborted=True,
abort_message=f"Request {state.req.request_id} aborted.",
abort_message=f"Request {request_id} aborted.",
)
if runner_output is not None and runner_output.result is not None: