Enable Diffusion Image models asynchronous output (#4981)

Signed-off-by: zzh <943967662@qq.com>
Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
zzhanddengyunyang authored and GitHub committed 2026-07-29 17:21:55 +00:00
1 parent f2d9eb1630
commit dba05c3517
22 files changed
+1440 -72

No files matched your search

+32
View File
@@ -1273,6 +1273,7 @@ class DiffusionOutput:
trajectory_latents: torch.Tensor | dict[str, Any] | None = None
trajectory_log_probs: torch.Tensor | dict[str, Any] | None = None
trajectory_decoded: list[Image.Image] | None = None
async_output_id: str | None = None
error: str | None = None
error_status_code: int | None = None
error_type: str | None = None
@@ -1333,6 +1334,37 @@ class DiffusionOutput:
)
class AsyncOutputKind(Enum):
"""Message kind for ``AsyncDiffusionOutput`` — routes async results.
* ``RPC_RESULT`` — ordinary RPC return (sleep, wake, profile, etc.)
* ``COMPUTE_DONE`` — worker forward finished, GPU can start next request
* ``OUTPUT_READY`` — background D2H/SHM packing finished, final output
is available via ``async_output_id``
"""
RPC_RESULT = "rpc_result"
COMPUTE_DONE = "compute_done"
OUTPUT_READY = "output_ready"
@dataclass
class AsyncDiffusionOutput:
"""Async protocol envelope for ``result_mq`` messages.
In request-mode (``step_execution=False``), all ``result_mq``
messages use this envelope. The ``kind`` field routes the message
to the correct consumer.
"""
kind: AsyncOutputKind
rpc_id: str | None = None
async_output_id: str | None = None
result: Any | None = None
output: DiffusionOutput | None = None
error: str | None = None
class DiffusionRequestAbortedError(RuntimeError):
"""Raised when a diffusion request ends via user-visible abort."""
+14 -1
View File
@@ -55,6 +55,8 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
_ASYNC_OUTPUT_TIMEOUT = 30.0 # seconds
__all__ = [
"DiffusionEngine",
"DiffusionExecutionMode",
@@ -288,6 +290,10 @@ class DiffusionEngine:
generator = self.async_add_req_and_stream_response(request)
async for output in generator:
exec_total_time = time.perf_counter() - exec_start_time
# Async mode: wait for background D2H/SHM to complete.
if output.async_output_id:
fut = self.executor.wait_output_ready(output.async_output_id)
output = await asyncio.wait_for(asyncio.wrap_future(fut), timeout=_ASYNC_OUTPUT_TIMEOUT)
postprocess_start_time = time.perf_counter()
formatted_outputs = self.postprocess_output(request, output)
postprocess_time = time.perf_counter() - postprocess_start_time
@@ -744,11 +750,15 @@ class DiffusionEngine:
raise ValueError("Sync func should receive one result at one time")
if target_request_id in finished_req_ids:
req_output = runner_output.get_request_output(target_request_id)
return self._finalize_finished_request(
output = self._finalize_finished_request(
target_request_id,
runner_output=req_output,
missing_result_error="Diffusion execution finished without a final output.",
)
if output.async_output_id:
fut = self.executor.wait_output_ready(output.async_output_id)
output = fut.result(timeout=_ASYNC_OUTPUT_TIMEOUT)
return output
def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None:
"""Start or stop profiling on all diffusion workers.
@@ -1036,4 +1046,7 @@ class DiffusionEngine:
if runner_output is not None and runner_output.result is not None:
return runner_output.result
if runner_output is not None and runner_output.async_output_id is not None:
return DiffusionOutput(async_output_id=runner_output.async_output_id)
return DiffusionOutput(error=missing_result_error)
@@ -1,7 +1,9 @@
from __future__ import annotations
import concurrent.futures
import multiprocessing as mp
import multiprocessing.connection
import queue
import threading
import time
import weakref
@@ -15,7 +17,7 @@ from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQ
from vllm.logger import init_logger
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput
from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, AsyncDiffusionOutput, AsyncOutputKind, DiffusionOutput
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, unpack_diffusion_output_shm
from vllm_omni.diffusion.worker import WorkerProc
@@ -93,6 +95,23 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
self.shutdown()
raise
# Async output: result pump thread for dispatching AsyncDiffusionOutput
# messages in request-mode (step_execution=False).
self._rpc_id_counter = 0
self._rpc_id_lock = threading.Lock()
self._rpc_futures: dict[str, concurrent.futures.Future[AsyncDiffusionOutput]] = {}
self._output_futures: dict[str, concurrent.futures.Future[DiffusionOutput]] = {}
self._completed_outputs: dict[str, concurrent.futures.Future[DiffusionOutput]] = {}
self._batch_split_map: dict[str, dict[str, str]] = {} # batch_id -> {per_req_id: request_id}
self._futures_lock = threading.RLock()
self._pump_running = False
self._pump_stop = threading.Event()
# When pump is active it is the sole reader of result_mq; non-async
# messages are placed here for collective_rpc() to consume.
self._sync_result_buffer: queue.Queue = queue.Queue()
if not self.od_config.step_execution:
self._start_result_pump()
self._start_worker_monitor()
def _init_broadcast_queue(self, num_workers: int) -> MessageQueue:
@@ -116,7 +135,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
raise RuntimeError("Broadcast queue is closed")
def _dequeue_one_with_failure_polling(self, deadline: float | None, method: str) -> Any:
"""Block until one result message, polling ``_is_failed`` between chunk timeouts."""
"""Block until one result message, polling ``_is_failed`` between chunk timeouts.
When async output is enabled, pump is the sole reader of result_mq;
non-async messages are placed in _sync_result_buffer, so this method
reads from there instead of result_mq directly.
"""
while True:
if deadline is None:
chunk_timeout = _DEQUEUE_TIMEOUT_S
@@ -125,12 +149,20 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
if remaining <= 0:
raise TimeoutError(f"RPC call to {method} timed out.")
chunk_timeout = min(_DEQUEUE_TIMEOUT_S, remaining)
try:
return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
except (TimeoutError, zmq.error.Again):
if self._is_failed:
raise EngineDeadError()
continue
if not self.od_config.step_execution:
try:
return self._sync_result_buffer.get(timeout=chunk_timeout)
except queue.Empty:
if self._is_failed or self._closed:
raise EngineDeadError()
continue
else:
try:
return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
except (TimeoutError, zmq.error.Again):
if self._is_failed:
raise EngineDeadError()
continue
@staticmethod
def _raise_for_rpc_error_dict(response: Any) -> None:
@@ -318,6 +350,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
"""Adapt request-mode scheduler output to worker execute_model RPCs.
Returns a BatchRunnerOutput with one RunnerOutput per scheduled request.
In async mode the result may carry async_output_id instead of the final output.
"""
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput
@@ -333,16 +366,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
unique_reply_rank=0,
exec_all_ranks=True,
)
if not isinstance(result, DiffusionOutput):
raise RuntimeError(f"Unexpected response type: {type(result)!r}")
runner_outputs.append(
RunnerOutput(
request_id=new_req.request_id,
step_index=None,
finished=True,
result=result,
if isinstance(result, AsyncDiffusionOutput) and result.kind == AsyncOutputKind.COMPUTE_DONE:
runner_outputs.append(
RunnerOutput(
request_id=new_req.request_id,
step_index=None,
finished=True,
result=None,
async_output_id=result.async_output_id,
)
)
)
elif isinstance(result, DiffusionOutput):
runner_outputs.append(
RunnerOutput(
request_id=new_req.request_id,
step_index=None,
finished=True,
result=result,
)
)
else:
raise RuntimeError(f"Unexpected response type: {type(result)!r}")
except Exception as exc:
runner_outputs.append(
RunnerOutput(
@@ -361,8 +405,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
A scheduler wave with one request is the conservative serial case and
uses the single-request worker RPC. Waves with multiple requests use the
fused request-batch RPC and require pipeline request-batch support.
When async output is enabled, per-request async_output_ids are
propagated through RunnerOutput so the engine waits in
step_streaming() instead of blocking the busy loop here.
"""
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput
self._ensure_open()
if len(scheduler_output.scheduled_new_reqs) <= 1:
@@ -374,6 +422,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
unique_reply_rank=0,
exec_all_ranks=True,
)
if isinstance(result, AsyncDiffusionOutput) and result.kind == AsyncOutputKind.COMPUTE_DONE:
# Propagate async_output_id to per-request RunnerOutputs so the
# engine waits in step_streaming() instead of blocking here.
batch_id = result.async_output_id
per_req_map: dict[str, str] = {}
runner_outputs: list[RunnerOutput] = []
for new_req in scheduler_output.scheduled_new_reqs:
per_req_id = f"{batch_id}/{new_req.request_id}"
per_req_map[per_req_id] = new_req.request_id
runner_outputs.append(
RunnerOutput(
request_id=new_req.request_id,
step_index=None,
finished=True,
result=None,
async_output_id=per_req_id,
)
)
with self._futures_lock:
self._batch_split_map[batch_id] = per_req_map
return BatchRunnerOutput.from_list(runner_outputs)
if not isinstance(result, BatchRunnerOutput):
raise RuntimeError(f"Unexpected response type for execute_batch: {type(result)!r}")
return result
@@ -408,11 +477,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
deadline = None if timeout is None else time.monotonic() + timeout
kwargs = kwargs or {}
# Prepare RPC request message. When unique_reply_rank is None, all
# workers must execute the RPC but only rank 0 can reply (it's the
# only one with a result_mq). Collect detailed rank statuses only for
# this control-plane all-rank path; forward-path exec_all_ranks RPCs
# avoid the per-step host object gather.
execute_all_ranks = unique_reply_rank is None or exec_all_ranks
collect_rank_status = unique_reply_rank is None
rpc_request = {
@@ -425,11 +489,32 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
"collect_rank_status": collect_rank_status,
}
# ── Path 1: async execute_model / execute_model_batch ──
# Request-mode methods use the compute_done/output_ready split
# so the D2H copy runs on a side stream in the worker background
# thread while the default stream is free for the next forward.
# pump routes the result back to a Future via rpc_id.
if not self.od_config.step_execution and method in ("execute_model", "execute_model_batch"):
rpc_id = self._next_rpc_id()
rpc_request["rpc_id"] = rpc_id
fut: concurrent.futures.Future = concurrent.futures.Future()
with self._futures_lock:
self._rpc_futures[rpc_id] = fut
self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
try:
return fut.result(timeout=timeout)
except concurrent.futures.TimeoutError:
with self._futures_lock:
self._rpc_futures.pop(rpc_id, None)
raise TimeoutError(f"RPC call to {method} timed out.")
# ── Path 2: step-execution or other non-execute_model RPCs ──
# _dequeue_one_with_failure_polling reads from _sync_result_buffer
# when pump is active (request-mode), or result_mq directly otherwise.
try:
# Broadcast RPC request to all workers via unified message queue
self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
# Only rank 0 has a result_mq, so we always expect exactly 1 response
num_responses = 1
responses = []
@@ -450,6 +535,123 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
logger.error(f"RPC call failed: {e}")
raise
# ------------------------------------------------------------------
# Async output: result pump
# ------------------------------------------------------------------
def _start_result_pump(self) -> None:
self._pump_running = True
self._pump_stop.clear()
self._result_pump_thread = threading.Thread(target=self._result_pump, daemon=True, name="DiffusionResultPump")
self._result_pump_thread.start()
logger.info("Async result pump started")
def _result_pump(self) -> None:
"""Sole reader of result_mq when async output is enabled.
Dispatches AsyncDiffusionOutput messages to the appropriate future:
* RPC_RESULT / COMPUTE_DONE → _rpc_futures[rpc_id]
* OUTPUT_READY → _output_futures[async_output_id]
"""
while not self._pump_stop.is_set():
try:
msg = self._result_mq.dequeue(timeout=1.0)
except TimeoutError:
if self._is_failed:
break
continue
except Exception:
logger.exception("Result pump dequeue failed")
if self._is_failed:
break
continue
if not isinstance(msg, AsyncDiffusionOutput):
# Non-async message: place into the sync buffer for
# collective_rpc() to consume via Path 2.
self._sync_result_buffer.put(msg)
continue
if msg.kind in (AsyncOutputKind.RPC_RESULT, AsyncOutputKind.COMPUTE_DONE):
with self._futures_lock:
fut = self._rpc_futures.pop(msg.rpc_id, None) if msg.rpc_id else None
if fut is not None and not fut.done():
if msg.error:
fut.set_exception(RuntimeError(msg.error))
else:
fut.set_result(msg)
elif msg.kind == AsyncOutputKind.OUTPUT_READY:
batch_id = msg.async_output_id
with self._futures_lock:
per_req_map = self._batch_split_map.pop(batch_id, None) if batch_id else None
if per_req_map is not None:
# Batch result: split into per-request DiffusionOutputs.
try:
unpack_diffusion_output_shm(msg.output)
except Exception:
logger.exception("SHM unpack failed for batch %s", batch_id)
batch_output = msg.output
for per_req_id, req_id in per_req_map.items():
req_output = batch_output.get_request_output(req_id)
per_req_result: DiffusionOutput
if req_output is not None and req_output.result is not None:
per_req_result = req_output.result
elif msg.error:
per_req_result = DiffusionOutput(error=msg.error)
else:
per_req_result = DiffusionOutput(error="No output result for batch request")
fut: concurrent.futures.Future = concurrent.futures.Future()
fut.set_result(per_req_result)
with self._futures_lock:
pending = self._output_futures.pop(per_req_id, None)
if pending is not None and not pending.done():
pending.set_result(per_req_result)
else:
self._completed_outputs[per_req_id] = fut
else:
with self._futures_lock:
fut = self._output_futures.pop(batch_id, None) if batch_id else None
if fut is not None and not fut.done():
if msg.error:
fut.set_exception(RuntimeError(msg.error))
else:
try:
unpack_diffusion_output_shm(msg.output)
except Exception as e:
logger.exception("SHM unpack failed in result pump")
fut.set_exception(e)
continue
fut.set_result(msg.output)
elif batch_id:
fut = concurrent.futures.Future()
if msg.error:
fut.set_exception(RuntimeError(msg.error))
else:
try:
unpack_diffusion_output_shm(msg.output)
except Exception as e:
logger.exception("SHM unpack failed in result pump (cached)")
fut.set_exception(e)
else:
fut.set_result(msg.output)
with self._futures_lock:
self._completed_outputs[batch_id] = fut
def _next_rpc_id(self) -> str:
with self._rpc_id_lock:
self._rpc_id_counter += 1
return str(self._rpc_id_counter)
def wait_output_ready(self, async_output_id: str) -> concurrent.futures.Future[DiffusionOutput]:
"""Return a Future that resolves when the async output is ready."""
with self._futures_lock:
cached = self._completed_outputs.pop(async_output_id, None)
if cached is not None:
return cached
fut: concurrent.futures.Future = concurrent.futures.Future()
self._output_futures[async_output_id] = fut
return fut
def check_health(self) -> None:
if self._is_failed:
raise EngineDeadError()
@@ -461,10 +663,21 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
def shutdown(self) -> None:
self._closed = True
self._pump_stop.set()
try:
self._finalizer()
finally:
self._broadcast_mq = None
self._result_mq = None
with self._futures_lock:
for fut in self._rpc_futures.values():
if not fut.done():
fut.set_exception(RuntimeError("Executor shut down"))
for fut in self._output_futures.values():
if not fut.done():
fut.set_exception(RuntimeError("Executor shut down"))
self._rpc_futures.clear()
self._output_futures.clear()
self._batch_split_map.clear()
self._shutdown_cleaner = None
self._processes = []
+59 -24
View File
@@ -21,23 +21,43 @@ _SHM_TENSOR_THRESHOLD = 1_000_000 # 1 MB
DIFFUSION_RPC_RESULT_ENVELOPE = "diffusion_rpc_result"
def _tensor_to_shm(tensor: torch.Tensor) -> dict[str, Any]:
def _tensor_to_shm(
tensor: torch.Tensor,
d2h_stream: torch.Stream | None = None,
) -> dict[str, Any]:
"""Copy a tensor into POSIX shared memory and return a metadata handle.
The shared memory segment remains alive after this call (the local fd is
closed, but the segment persists until ``_tensor_from_shm`` unlinks it).
If *d2h_stream* is provided, the D2H copy uses ``copy_()`` with
``pin_memory=True`` on that stream instead of the synchronous ``.cpu()``
path. The caller must synchronize *d2h_stream* after all tensors are
packed.
"""
from multiprocessing import shared_memory
import numpy as np
tensor = tensor.detach().cpu().contiguous()
original_dtype = tensor.dtype
# NumPy does not support bfloat16; promote to float32 for the SHM
# transfer and record the original dtype so _tensor_from_shm can
# convert back. The round-trip is lossless for bfloat16 values.
if original_dtype == torch.bfloat16:
tensor = tensor.to(torch.float32)
if d2h_stream is not None:
# Non-blocking D2H: copy on side stream to pinned CPU memory.
old_stream = torch.accelerator.current_stream()
torch.accelerator.set_stream(d2h_stream)
try:
t = tensor.detach()
if original_dtype == torch.bfloat16:
t = t.to(torch.float32)
cpu = torch.empty(t.shape, dtype=t.dtype, pin_memory=True)
cpu.copy_(t, non_blocking=True)
finally:
torch.accelerator.set_stream(old_stream)
d2h_stream.synchronize()
tensor = cpu
else:
tensor = tensor.detach().cpu().contiguous()
if original_dtype == torch.bfloat16:
tensor = tensor.to(torch.float32)
arr = tensor.numpy()
nbytes = arr.nbytes
shm = shared_memory.SharedMemory(create=True, size=nbytes)
@@ -79,14 +99,20 @@ def _tensor_from_shm(handle: dict[str, Any]) -> torch.Tensor:
return tensor
def _pack_tensor_if_large(val: torch.Tensor) -> torch.Tensor | dict:
def _pack_tensor_if_large(
val: torch.Tensor,
d2h_stream: torch.Stream | None = None,
) -> torch.Tensor | dict:
"""Replace a tensor with an SHM handle if it exceeds the threshold."""
if val.nelement() * val.element_size() > _SHM_TENSOR_THRESHOLD:
return _tensor_to_shm(val)
return _tensor_to_shm(val, d2h_stream=d2h_stream)
return val
def _pack_value_if_large(val: object) -> object:
def _pack_value_if_large(
val: object,
d2h_stream: torch.Stream | None = None,
) -> object:
"""Recursively replace large tensors with SHM handles.
Walks the container shapes pipelines return as ``DiffusionOutput.output``:
@@ -96,13 +122,13 @@ def _pack_value_if_large(val: object) -> object:
the two in sync.
"""
if isinstance(val, torch.Tensor):
return _pack_tensor_if_large(val)
return _pack_tensor_if_large(val, d2h_stream=d2h_stream)
if isinstance(val, dict):
return {key: _pack_value_if_large(value) for key, value in val.items()}
return {key: _pack_value_if_large(value, d2h_stream=d2h_stream) for key, value in val.items()}
if isinstance(val, list):
return [_pack_value_if_large(item) for item in val]
return [_pack_value_if_large(item, d2h_stream=d2h_stream) for item in val]
if isinstance(val, tuple):
return tuple(_pack_value_if_large(item) for item in val)
return tuple(_pack_value_if_large(item, d2h_stream=d2h_stream) for item in val)
return val
@@ -119,15 +145,18 @@ def _unpack_if_shm_handle(val: object) -> object:
return val
def _pack_diffusion_fields(output: DiffusionOutput) -> DiffusionOutput:
def _pack_diffusion_fields(
output: DiffusionOutput,
d2h_stream: torch.Stream | None = None,
) -> DiffusionOutput:
if output.output is not None:
output.output = _pack_value_if_large(output.output)
output.output = _pack_value_if_large(output.output, d2h_stream=d2h_stream)
if output.trajectory_latents is not None and isinstance(output.trajectory_latents, torch.Tensor):
output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents)
output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents, d2h_stream=d2h_stream)
if output.trajectory_timesteps is not None and isinstance(output.trajectory_timesteps, torch.Tensor):
output.trajectory_timesteps = _pack_tensor_if_large(output.trajectory_timesteps)
output.trajectory_timesteps = _pack_tensor_if_large(output.trajectory_timesteps, d2h_stream=d2h_stream)
if output.trajectory_log_probs is not None and isinstance(output.trajectory_log_probs, torch.Tensor):
output.trajectory_log_probs = _pack_tensor_if_large(output.trajectory_log_probs)
output.trajectory_log_probs = _pack_tensor_if_large(output.trajectory_log_probs, d2h_stream=d2h_stream)
return output
@@ -135,31 +164,37 @@ def _is_rpc_result_envelope(output: object) -> bool:
return isinstance(output, dict) and output.get("type") == DIFFUSION_RPC_RESULT_ENVELOPE
def pack_diffusion_output_shm(output: object) -> object:
def pack_diffusion_output_shm(
output: object,
d2h_stream: torch.Stream | None = None,
) -> object:
"""Replace large tensors in diffusion worker outputs with SHM handles.
Supports a bare ``DiffusionOutput``, a wrapper object carrying one in
``.result`` (for example ``RunnerOutput``), an RPC result envelope carrying
the diffusion output in ``["result"]``, or a batch wrapper carrying
``RunnerOutput`` objects in ``.runner_outputs``.
If *d2h_stream* is provided, D2H copies use that stream (non-blocking on
the default stream). The caller must synchronize *d2h_stream* afterward.
"""
if isinstance(output, DiffusionOutput):
return _pack_diffusion_fields(output)
return _pack_diffusion_fields(output, d2h_stream=d2h_stream)
if _is_rpc_result_envelope(output):
result = output.get("result")
if isinstance(result, DiffusionOutput):
output["result"] = _pack_diffusion_fields(result)
output["result"] = _pack_diffusion_fields(result, d2h_stream=d2h_stream)
return output
result = getattr(output, "result", None)
if isinstance(result, DiffusionOutput):
output.result = _pack_diffusion_fields(result)
output.result = _pack_diffusion_fields(result, d2h_stream=d2h_stream)
runner_outputs = getattr(output, "runner_outputs", None)
if isinstance(runner_outputs, list):
for runner_output in runner_outputs:
pack_diffusion_output_shm(runner_output)
pack_diffusion_output_shm(runner_output, d2h_stream=d2h_stream)
return output
@@ -3233,8 +3233,7 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
assert image.shape[2] == 1, "image should have shape [B, C, T, H, W] and T should be 1"
image = image.squeeze(2)
do_denormalize = [True] * image.shape[0]
image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
# Denormalize + PIL moved to engine post_process_func for overlap with next request.
if not return_dict:
return (image,)
@@ -71,6 +71,8 @@ _STEP_COT_TEXT_LIST = "hunyuan_cot_text_list"
_STEP_AR_KV = "hunyuan_ar_kv"
_STEP_PROMPT_KV = "hunyuan_prompt_kv"
_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil"
def default(val, d):
return val if val is not None else d
@@ -333,6 +335,41 @@ def get_hunyuan_image_3_pre_process_func(od_config: OmniDiffusionConfig):
return pre_process_func
def get_hunyuan_image3_post_process_func(od_config: OmniDiffusionConfig):
"""GPU tensor → PIL, runs outside pipeline_forward to overlap with next request."""
from diffusers.image_processor import VaeImageProcessor
image_processor = VaeImageProcessor()
def post_process_func(images: torch.Tensor | dict):
# Handle dict envelope format (upstream): {"payload": {"image": ...}, "metadata": ...}
if isinstance(images, dict) and "payload" in images:
tensor = images["payload"].get("image")
if tensor is None:
return images
if tensor.dim() == 3:
tensor = tensor.unsqueeze(0)
do_denormalize = [True] * tensor.shape[0]
images["payload"]["image"] = image_processor.postprocess(
tensor,
output_type=_HUNYUAN_DEFAULT_OUTPUT_TYPE,
do_denormalize=do_denormalize,
)
return images
# Legacy raw tensor format
if images.dim() == 3:
images = images.unsqueeze(0)
do_denormalize = [True] * images.shape[0]
return image_processor.postprocess(
images,
output_type=_HUNYUAN_DEFAULT_OUTPUT_TYPE,
do_denormalize=do_denormalize,
)
return post_process_func
class HunyuanImage3Pipeline(
HunyuanImage3PreTrainedModel,
GenerationMixin,
@@ -2224,11 +2261,7 @@ class HunyuanImage3Pipeline(
if hasattr(self.vae, "ffactor_temporal"):
assert image.shape[2] == 1, "image should have shape [B, C, T, H, W] and T should be 1"
image = image.squeeze(2)
image = self.pipeline.image_processor.postprocess(
image,
output_type=output_type,
do_denormalize=[True] * image.shape[0],
)
# Postprocess deferred to engine post_process_func for overlap with next request.
cot_text_list = state.extra.get(_STEP_COT_TEXT_LIST) or []
metadata = {}
+2
View File
@@ -530,6 +530,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
"Flux2Pipeline": "get_flux2_post_process_func",
"HunyuanVideo15Pipeline": "get_hunyuan_video_15_post_process_func",
"HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_post_process_func",
"HunyuanImage3Pipeline": "get_hunyuan_image3_post_process_func",
"LingBotVideoPipeline": "get_lingbot_video_post_process_func",
"MagiHumanPipeline": "get_magi_human_post_process_func",
"OmniVoicePipeline": "get_omnivoice_post_process_func",
@@ -540,6 +541,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
"HiDreamImagePipeline": "get_hidream_image_post_process_func",
"StableDiffusionXLPipeline": "get_sdxl_image_post_process_func",
"Krea2Pipeline": "get_krea2_post_process_func",
"HunyuanImage3ForCausalMM": "get_hunyuan_image3_post_process_func",
}
_DIFFUSION_IR_OP_PRIORITY_FUNCS = {
@@ -50,8 +50,14 @@ class RequestScheduler(BaseScheduler):
req_output = output.get_request_output(request_id)
result = req_output.result if req_output is not None else None
if result is None:
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ERROR
terminal_errors[request_id] = "No output result"
# Async mode: result=None with async_output_id means compute done,
# final output will arrive later via wait_output_ready.
if req_output is not None and req_output.async_output_id is not None:
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_COMPLETED
terminal_errors[request_id] = None
else:
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ERROR
terminal_errors[request_id] = "No output result"
elif result.aborted:
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ABORTED
terminal_errors[request_id] = None
+98 -12
View File
@@ -11,8 +11,11 @@ to DiffusionModelRunner.
import gc
import multiprocessing as mp
import os
import queue
import signal
import threading
import traceback
import uuid
from collections.abc import Iterator
from contextlib import AbstractContextManager, contextmanager, nullcontext
from dataclasses import dataclass
@@ -30,6 +33,8 @@ from vllm.utils.mem_utils import GiB_bytes
from vllm.v1.worker.workspace import init_workspace_manager
from vllm_omni.diffusion.data import (
AsyncDiffusionOutput,
AsyncOutputKind,
DiffusionOutput,
OmniACK,
OmniDiffusionConfig,
@@ -794,6 +799,21 @@ class WorkerProc:
self.worker = self._create_worker(gpu_id, od_config, worker_extension_cls, custom_pipeline_args)
self._running = True
self._async_output_queue: queue.Queue | None = None
self._async_output_thread: threading.Thread | None = None
if not self.od_config.step_execution:
self._async_output_queue = queue.Queue()
self._async_output_thread = threading.Thread(
target=self._async_output_loop,
daemon=True,
name="DiffusionAsyncOutput",
)
self._async_output_thread.start()
@staticmethod
def _generate_async_output_id() -> str:
return uuid.uuid4().hex
def _create_worker(
self,
gpu_id: int,
@@ -813,18 +833,72 @@ class WorkerProc:
)
return wrapper
def _return_result(self, output: Any) -> None:
def _return_result(self, output: Any, rpc_id: str | None = None) -> None:
"""Reply to client, only on rank 0."""
if self.result_mq is not None:
if isinstance(output, OmniACK):
self.result_mq.enqueue(output)
return
try:
pack_diffusion_output_shm(output)
except Exception as e:
if hasattr(output, "output"):
logger.warning("SHM pack failed for model output: %s", e)
if self.result_mq is None:
return
if isinstance(output, OmniACK):
self.result_mq.enqueue(output)
return
# Async path: enqueue compute_done immediately, bg thread does D2H+SHM.
if not self.od_config.step_execution and isinstance(output, (DiffusionOutput, BatchRunnerOutput)):
async_output_id = WorkerProc._generate_async_output_id()
gpu_event = current_omni_platform.record_device_event()
self._async_output_queue.put((output, async_output_id, gpu_event))
msg = AsyncDiffusionOutput(
kind=AsyncOutputKind.COMPUTE_DONE,
rpc_id=rpc_id,
async_output_id=async_output_id,
)
self.result_mq.enqueue(msg)
return
# Sync path (original, or async fallback).
try:
pack_diffusion_output_shm(output)
except Exception as e:
if hasattr(output, "output"):
logger.warning("SHM pack failed for model output: %s", e)
self.result_mq.enqueue(output)
def _async_output_loop(self):
"""Background thread: D2H + SHM packing for async diffusion output.
Uses a side stream so the D2H transfer does not block the default
stream where the next forward runs.
"""
device = torch.device(torch.accelerator.current_accelerator().type, self.gpu_id)
d2h_stream = torch.Stream(device=device)
while self._running:
output, async_output_id, gpu_event = self._async_output_queue.get()
try:
# Cross-stream ordering: wait for default stream to finish
# writing the output tensors before the side stream reads.
if gpu_event is not None:
d2h_stream.wait_event(gpu_event)
pack_diffusion_output_shm(output, d2h_stream=d2h_stream)
d2h_stream.synchronize()
self.result_mq.enqueue(
AsyncDiffusionOutput(
kind=AsyncOutputKind.OUTPUT_READY,
async_output_id=async_output_id,
output=output,
)
)
except Exception:
logger.exception(
"Async output packing failed for id '%s'; sending error",
async_output_id,
)
self.result_mq.enqueue(
AsyncDiffusionOutput(
kind=AsyncOutputKind.OUTPUT_READY,
async_output_id=async_output_id,
error="Background D2H/SHM packing failed",
)
)
def _gather_rpc_rank_statuses(self, status: dict[str, Any]) -> list[dict[str, Any]]:
if not torch.distributed.is_initialized():
@@ -936,13 +1010,25 @@ class WorkerProc:
# Route message based on type
elif isinstance(msg, dict) and msg.get("type") == "rpc":
try:
rpc_id = msg.get("rpc_id")
result, should_reply = self._execute_rpc(msg)
if should_reply:
self._return_result(result)
self._return_result(result, rpc_id=rpc_id)
except Exception as e:
logger.error(f"Error processing RPC: {e}", exc_info=True)
if self.result_mq is not None:
self._return_result({"status": "error", "error": str(e)})
if rpc_id is not None:
# Async RPC: must complete the executor''s pending
# future so collective_rpc() doesn''t hang.
self.result_mq.enqueue(
AsyncDiffusionOutput(
kind=AsyncOutputKind.RPC_RESULT,
rpc_id=rpc_id,
error=str(e),
)
)
else:
self._return_result({"status": "error", "error": str(e)})
elif isinstance(msg, dict) and msg.get("type") == "shutdown":
logger.info("Worker %s: Received shutdown message", self.gpu_id)
+1
View File
@@ -183,6 +183,7 @@ class RunnerOutput(BaseRunnerOutput):
step_index: int | None = None
finished: bool = False
result: DiffusionOutput | None = None
async_output_id: str | None = None
def get_request_output(self, request_id: str) -> RunnerOutput | None:
return self if self.request_id == request_id else None