Enable Diffusion Image models asynchronous output (#4981)
Signed-off-by: zzh <943967662@qq.com> Co-authored-by: dengyunyang <584797741@qq.com>
This commit is contained in:
1 parent
f2d9eb1630
commit
dba05c3517
22 files changed
+1440
-72
No files matched your search
@@ -1273,6 +1273,7 @@ class DiffusionOutput:
|
||||
trajectory_latents: torch.Tensor | dict[str, Any] | None = None
|
||||
trajectory_log_probs: torch.Tensor | dict[str, Any] | None = None
|
||||
trajectory_decoded: list[Image.Image] | None = None
|
||||
async_output_id: str | None = None
|
||||
error: str | None = None
|
||||
error_status_code: int | None = None
|
||||
error_type: str | None = None
|
||||
@@ -1333,6 +1334,37 @@ class DiffusionOutput:
|
||||
)
|
||||
|
||||
|
||||
class AsyncOutputKind(Enum):
|
||||
"""Message kind for ``AsyncDiffusionOutput`` — routes async results.
|
||||
|
||||
* ``RPC_RESULT`` — ordinary RPC return (sleep, wake, profile, etc.)
|
||||
* ``COMPUTE_DONE`` — worker forward finished, GPU can start next request
|
||||
* ``OUTPUT_READY`` — background D2H/SHM packing finished, final output
|
||||
is available via ``async_output_id``
|
||||
"""
|
||||
|
||||
RPC_RESULT = "rpc_result"
|
||||
COMPUTE_DONE = "compute_done"
|
||||
OUTPUT_READY = "output_ready"
|
||||
|
||||
|
||||
@dataclass
|
||||
class AsyncDiffusionOutput:
|
||||
"""Async protocol envelope for ``result_mq`` messages.
|
||||
|
||||
In request-mode (``step_execution=False``), all ``result_mq``
|
||||
messages use this envelope. The ``kind`` field routes the message
|
||||
to the correct consumer.
|
||||
"""
|
||||
|
||||
kind: AsyncOutputKind
|
||||
rpc_id: str | None = None
|
||||
async_output_id: str | None = None
|
||||
result: Any | None = None
|
||||
output: DiffusionOutput | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class DiffusionRequestAbortedError(RuntimeError):
|
||||
"""Raised when a diffusion request ends via user-visible abort."""
|
||||
|
||||
|
||||
@@ -55,6 +55,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_ASYNC_OUTPUT_TIMEOUT = 30.0 # seconds
|
||||
|
||||
__all__ = [
|
||||
"DiffusionEngine",
|
||||
"DiffusionExecutionMode",
|
||||
@@ -288,6 +290,10 @@ class DiffusionEngine:
|
||||
generator = self.async_add_req_and_stream_response(request)
|
||||
async for output in generator:
|
||||
exec_total_time = time.perf_counter() - exec_start_time
|
||||
# Async mode: wait for background D2H/SHM to complete.
|
||||
if output.async_output_id:
|
||||
fut = self.executor.wait_output_ready(output.async_output_id)
|
||||
output = await asyncio.wait_for(asyncio.wrap_future(fut), timeout=_ASYNC_OUTPUT_TIMEOUT)
|
||||
postprocess_start_time = time.perf_counter()
|
||||
formatted_outputs = self.postprocess_output(request, output)
|
||||
postprocess_time = time.perf_counter() - postprocess_start_time
|
||||
@@ -744,11 +750,15 @@ class DiffusionEngine:
|
||||
raise ValueError("Sync func should receive one result at one time")
|
||||
if target_request_id in finished_req_ids:
|
||||
req_output = runner_output.get_request_output(target_request_id)
|
||||
return self._finalize_finished_request(
|
||||
output = self._finalize_finished_request(
|
||||
target_request_id,
|
||||
runner_output=req_output,
|
||||
missing_result_error="Diffusion execution finished without a final output.",
|
||||
)
|
||||
if output.async_output_id:
|
||||
fut = self.executor.wait_output_ready(output.async_output_id)
|
||||
output = fut.result(timeout=_ASYNC_OUTPUT_TIMEOUT)
|
||||
return output
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None:
|
||||
"""Start or stop profiling on all diffusion workers.
|
||||
@@ -1036,4 +1046,7 @@ class DiffusionEngine:
|
||||
if runner_output is not None and runner_output.result is not None:
|
||||
return runner_output.result
|
||||
|
||||
if runner_output is not None and runner_output.async_output_id is not None:
|
||||
return DiffusionOutput(async_output_id=runner_output.async_output_id)
|
||||
|
||||
return DiffusionOutput(error=missing_result_error)
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import multiprocessing as mp
|
||||
import multiprocessing.connection
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
@@ -15,7 +17,7 @@ from vllm.distributed.device_communicators.shm_broadcast import Handle, MessageQ
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput
|
||||
from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, AsyncDiffusionOutput, AsyncOutputKind, DiffusionOutput
|
||||
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
|
||||
from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, unpack_diffusion_output_shm
|
||||
from vllm_omni.diffusion.worker import WorkerProc
|
||||
@@ -93,6 +95,23 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
# Async output: result pump thread for dispatching AsyncDiffusionOutput
|
||||
# messages in request-mode (step_execution=False).
|
||||
self._rpc_id_counter = 0
|
||||
self._rpc_id_lock = threading.Lock()
|
||||
self._rpc_futures: dict[str, concurrent.futures.Future[AsyncDiffusionOutput]] = {}
|
||||
self._output_futures: dict[str, concurrent.futures.Future[DiffusionOutput]] = {}
|
||||
self._completed_outputs: dict[str, concurrent.futures.Future[DiffusionOutput]] = {}
|
||||
self._batch_split_map: dict[str, dict[str, str]] = {} # batch_id -> {per_req_id: request_id}
|
||||
self._futures_lock = threading.RLock()
|
||||
self._pump_running = False
|
||||
self._pump_stop = threading.Event()
|
||||
# When pump is active it is the sole reader of result_mq; non-async
|
||||
# messages are placed here for collective_rpc() to consume.
|
||||
self._sync_result_buffer: queue.Queue = queue.Queue()
|
||||
if not self.od_config.step_execution:
|
||||
self._start_result_pump()
|
||||
|
||||
self._start_worker_monitor()
|
||||
|
||||
def _init_broadcast_queue(self, num_workers: int) -> MessageQueue:
|
||||
@@ -116,7 +135,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
raise RuntimeError("Broadcast queue is closed")
|
||||
|
||||
def _dequeue_one_with_failure_polling(self, deadline: float | None, method: str) -> Any:
|
||||
"""Block until one result message, polling ``_is_failed`` between chunk timeouts."""
|
||||
"""Block until one result message, polling ``_is_failed`` between chunk timeouts.
|
||||
|
||||
When async output is enabled, pump is the sole reader of result_mq;
|
||||
non-async messages are placed in _sync_result_buffer, so this method
|
||||
reads from there instead of result_mq directly.
|
||||
"""
|
||||
while True:
|
||||
if deadline is None:
|
||||
chunk_timeout = _DEQUEUE_TIMEOUT_S
|
||||
@@ -125,12 +149,20 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
if remaining <= 0:
|
||||
raise TimeoutError(f"RPC call to {method} timed out.")
|
||||
chunk_timeout = min(_DEQUEUE_TIMEOUT_S, remaining)
|
||||
try:
|
||||
return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
|
||||
except (TimeoutError, zmq.error.Again):
|
||||
if self._is_failed:
|
||||
raise EngineDeadError()
|
||||
continue
|
||||
if not self.od_config.step_execution:
|
||||
try:
|
||||
return self._sync_result_buffer.get(timeout=chunk_timeout)
|
||||
except queue.Empty:
|
||||
if self._is_failed or self._closed:
|
||||
raise EngineDeadError()
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
|
||||
except (TimeoutError, zmq.error.Again):
|
||||
if self._is_failed:
|
||||
raise EngineDeadError()
|
||||
continue
|
||||
|
||||
@staticmethod
|
||||
def _raise_for_rpc_error_dict(response: Any) -> None:
|
||||
@@ -318,6 +350,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
"""Adapt request-mode scheduler output to worker execute_model RPCs.
|
||||
|
||||
Returns a BatchRunnerOutput with one RunnerOutput per scheduled request.
|
||||
In async mode the result may carry async_output_id instead of the final output.
|
||||
"""
|
||||
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput
|
||||
|
||||
@@ -333,16 +366,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
unique_reply_rank=0,
|
||||
exec_all_ranks=True,
|
||||
)
|
||||
if not isinstance(result, DiffusionOutput):
|
||||
raise RuntimeError(f"Unexpected response type: {type(result)!r}")
|
||||
runner_outputs.append(
|
||||
RunnerOutput(
|
||||
request_id=new_req.request_id,
|
||||
step_index=None,
|
||||
finished=True,
|
||||
result=result,
|
||||
if isinstance(result, AsyncDiffusionOutput) and result.kind == AsyncOutputKind.COMPUTE_DONE:
|
||||
runner_outputs.append(
|
||||
RunnerOutput(
|
||||
request_id=new_req.request_id,
|
||||
step_index=None,
|
||||
finished=True,
|
||||
result=None,
|
||||
async_output_id=result.async_output_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
elif isinstance(result, DiffusionOutput):
|
||||
runner_outputs.append(
|
||||
RunnerOutput(
|
||||
request_id=new_req.request_id,
|
||||
step_index=None,
|
||||
finished=True,
|
||||
result=result,
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(f"Unexpected response type: {type(result)!r}")
|
||||
except Exception as exc:
|
||||
runner_outputs.append(
|
||||
RunnerOutput(
|
||||
@@ -361,8 +405,12 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
A scheduler wave with one request is the conservative serial case and
|
||||
uses the single-request worker RPC. Waves with multiple requests use the
|
||||
fused request-batch RPC and require pipeline request-batch support.
|
||||
|
||||
When async output is enabled, per-request async_output_ids are
|
||||
propagated through RunnerOutput so the engine waits in
|
||||
step_streaming() instead of blocking the busy loop here.
|
||||
"""
|
||||
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput
|
||||
from vllm_omni.diffusion.worker.utils import BatchRunnerOutput, RunnerOutput
|
||||
|
||||
self._ensure_open()
|
||||
if len(scheduler_output.scheduled_new_reqs) <= 1:
|
||||
@@ -374,6 +422,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
unique_reply_rank=0,
|
||||
exec_all_ranks=True,
|
||||
)
|
||||
if isinstance(result, AsyncDiffusionOutput) and result.kind == AsyncOutputKind.COMPUTE_DONE:
|
||||
# Propagate async_output_id to per-request RunnerOutputs so the
|
||||
# engine waits in step_streaming() instead of blocking here.
|
||||
batch_id = result.async_output_id
|
||||
per_req_map: dict[str, str] = {}
|
||||
runner_outputs: list[RunnerOutput] = []
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
per_req_id = f"{batch_id}/{new_req.request_id}"
|
||||
per_req_map[per_req_id] = new_req.request_id
|
||||
runner_outputs.append(
|
||||
RunnerOutput(
|
||||
request_id=new_req.request_id,
|
||||
step_index=None,
|
||||
finished=True,
|
||||
result=None,
|
||||
async_output_id=per_req_id,
|
||||
)
|
||||
)
|
||||
with self._futures_lock:
|
||||
self._batch_split_map[batch_id] = per_req_map
|
||||
return BatchRunnerOutput.from_list(runner_outputs)
|
||||
if not isinstance(result, BatchRunnerOutput):
|
||||
raise RuntimeError(f"Unexpected response type for execute_batch: {type(result)!r}")
|
||||
return result
|
||||
@@ -408,11 +477,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# Prepare RPC request message. When unique_reply_rank is None, all
|
||||
# workers must execute the RPC but only rank 0 can reply (it's the
|
||||
# only one with a result_mq). Collect detailed rank statuses only for
|
||||
# this control-plane all-rank path; forward-path exec_all_ranks RPCs
|
||||
# avoid the per-step host object gather.
|
||||
execute_all_ranks = unique_reply_rank is None or exec_all_ranks
|
||||
collect_rank_status = unique_reply_rank is None
|
||||
rpc_request = {
|
||||
@@ -425,11 +489,32 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
"collect_rank_status": collect_rank_status,
|
||||
}
|
||||
|
||||
# ── Path 1: async execute_model / execute_model_batch ──
|
||||
# Request-mode methods use the compute_done/output_ready split
|
||||
# so the D2H copy runs on a side stream in the worker background
|
||||
# thread while the default stream is free for the next forward.
|
||||
# pump routes the result back to a Future via rpc_id.
|
||||
if not self.od_config.step_execution and method in ("execute_model", "execute_model_batch"):
|
||||
rpc_id = self._next_rpc_id()
|
||||
rpc_request["rpc_id"] = rpc_id
|
||||
fut: concurrent.futures.Future = concurrent.futures.Future()
|
||||
with self._futures_lock:
|
||||
self._rpc_futures[rpc_id] = fut
|
||||
self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
|
||||
try:
|
||||
return fut.result(timeout=timeout)
|
||||
except concurrent.futures.TimeoutError:
|
||||
with self._futures_lock:
|
||||
self._rpc_futures.pop(rpc_id, None)
|
||||
raise TimeoutError(f"RPC call to {method} timed out.")
|
||||
|
||||
# ── Path 2: step-execution or other non-execute_model RPCs ──
|
||||
# _dequeue_one_with_failure_polling reads from _sync_result_buffer
|
||||
# when pump is active (request-mode), or result_mq directly otherwise.
|
||||
try:
|
||||
# Broadcast RPC request to all workers via unified message queue
|
||||
self._broadcast_mq.enqueue(rpc_request) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
|
||||
|
||||
# Only rank 0 has a result_mq, so we always expect exactly 1 response
|
||||
num_responses = 1
|
||||
|
||||
responses = []
|
||||
@@ -450,6 +535,123 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
logger.error(f"RPC call failed: {e}")
|
||||
raise
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Async output: result pump
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _start_result_pump(self) -> None:
|
||||
self._pump_running = True
|
||||
self._pump_stop.clear()
|
||||
self._result_pump_thread = threading.Thread(target=self._result_pump, daemon=True, name="DiffusionResultPump")
|
||||
self._result_pump_thread.start()
|
||||
logger.info("Async result pump started")
|
||||
|
||||
def _result_pump(self) -> None:
|
||||
"""Sole reader of result_mq when async output is enabled.
|
||||
|
||||
Dispatches AsyncDiffusionOutput messages to the appropriate future:
|
||||
* RPC_RESULT / COMPUTE_DONE → _rpc_futures[rpc_id]
|
||||
* OUTPUT_READY → _output_futures[async_output_id]
|
||||
"""
|
||||
while not self._pump_stop.is_set():
|
||||
try:
|
||||
msg = self._result_mq.dequeue(timeout=1.0)
|
||||
except TimeoutError:
|
||||
if self._is_failed:
|
||||
break
|
||||
continue
|
||||
except Exception:
|
||||
logger.exception("Result pump dequeue failed")
|
||||
if self._is_failed:
|
||||
break
|
||||
continue
|
||||
|
||||
if not isinstance(msg, AsyncDiffusionOutput):
|
||||
# Non-async message: place into the sync buffer for
|
||||
# collective_rpc() to consume via Path 2.
|
||||
self._sync_result_buffer.put(msg)
|
||||
continue
|
||||
|
||||
if msg.kind in (AsyncOutputKind.RPC_RESULT, AsyncOutputKind.COMPUTE_DONE):
|
||||
with self._futures_lock:
|
||||
fut = self._rpc_futures.pop(msg.rpc_id, None) if msg.rpc_id else None
|
||||
if fut is not None and not fut.done():
|
||||
if msg.error:
|
||||
fut.set_exception(RuntimeError(msg.error))
|
||||
else:
|
||||
fut.set_result(msg)
|
||||
elif msg.kind == AsyncOutputKind.OUTPUT_READY:
|
||||
batch_id = msg.async_output_id
|
||||
with self._futures_lock:
|
||||
per_req_map = self._batch_split_map.pop(batch_id, None) if batch_id else None
|
||||
if per_req_map is not None:
|
||||
# Batch result: split into per-request DiffusionOutputs.
|
||||
try:
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
except Exception:
|
||||
logger.exception("SHM unpack failed for batch %s", batch_id)
|
||||
batch_output = msg.output
|
||||
for per_req_id, req_id in per_req_map.items():
|
||||
req_output = batch_output.get_request_output(req_id)
|
||||
per_req_result: DiffusionOutput
|
||||
if req_output is not None and req_output.result is not None:
|
||||
per_req_result = req_output.result
|
||||
elif msg.error:
|
||||
per_req_result = DiffusionOutput(error=msg.error)
|
||||
else:
|
||||
per_req_result = DiffusionOutput(error="No output result for batch request")
|
||||
fut: concurrent.futures.Future = concurrent.futures.Future()
|
||||
fut.set_result(per_req_result)
|
||||
with self._futures_lock:
|
||||
pending = self._output_futures.pop(per_req_id, None)
|
||||
if pending is not None and not pending.done():
|
||||
pending.set_result(per_req_result)
|
||||
else:
|
||||
self._completed_outputs[per_req_id] = fut
|
||||
else:
|
||||
with self._futures_lock:
|
||||
fut = self._output_futures.pop(batch_id, None) if batch_id else None
|
||||
if fut is not None and not fut.done():
|
||||
if msg.error:
|
||||
fut.set_exception(RuntimeError(msg.error))
|
||||
else:
|
||||
try:
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
except Exception as e:
|
||||
logger.exception("SHM unpack failed in result pump")
|
||||
fut.set_exception(e)
|
||||
continue
|
||||
fut.set_result(msg.output)
|
||||
elif batch_id:
|
||||
fut = concurrent.futures.Future()
|
||||
if msg.error:
|
||||
fut.set_exception(RuntimeError(msg.error))
|
||||
else:
|
||||
try:
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
except Exception as e:
|
||||
logger.exception("SHM unpack failed in result pump (cached)")
|
||||
fut.set_exception(e)
|
||||
else:
|
||||
fut.set_result(msg.output)
|
||||
with self._futures_lock:
|
||||
self._completed_outputs[batch_id] = fut
|
||||
|
||||
def _next_rpc_id(self) -> str:
|
||||
with self._rpc_id_lock:
|
||||
self._rpc_id_counter += 1
|
||||
return str(self._rpc_id_counter)
|
||||
|
||||
def wait_output_ready(self, async_output_id: str) -> concurrent.futures.Future[DiffusionOutput]:
|
||||
"""Return a Future that resolves when the async output is ready."""
|
||||
with self._futures_lock:
|
||||
cached = self._completed_outputs.pop(async_output_id, None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
fut: concurrent.futures.Future = concurrent.futures.Future()
|
||||
self._output_futures[async_output_id] = fut
|
||||
return fut
|
||||
|
||||
def check_health(self) -> None:
|
||||
if self._is_failed:
|
||||
raise EngineDeadError()
|
||||
@@ -461,10 +663,21 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._closed = True
|
||||
self._pump_stop.set()
|
||||
try:
|
||||
self._finalizer()
|
||||
finally:
|
||||
self._broadcast_mq = None
|
||||
self._result_mq = None
|
||||
with self._futures_lock:
|
||||
for fut in self._rpc_futures.values():
|
||||
if not fut.done():
|
||||
fut.set_exception(RuntimeError("Executor shut down"))
|
||||
for fut in self._output_futures.values():
|
||||
if not fut.done():
|
||||
fut.set_exception(RuntimeError("Executor shut down"))
|
||||
self._rpc_futures.clear()
|
||||
self._output_futures.clear()
|
||||
self._batch_split_map.clear()
|
||||
self._shutdown_cleaner = None
|
||||
self._processes = []
|
||||
+59
-24
@@ -21,23 +21,43 @@ _SHM_TENSOR_THRESHOLD = 1_000_000 # 1 MB
|
||||
DIFFUSION_RPC_RESULT_ENVELOPE = "diffusion_rpc_result"
|
||||
|
||||
|
||||
def _tensor_to_shm(tensor: torch.Tensor) -> dict[str, Any]:
|
||||
def _tensor_to_shm(
|
||||
tensor: torch.Tensor,
|
||||
d2h_stream: torch.Stream | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy a tensor into POSIX shared memory and return a metadata handle.
|
||||
|
||||
The shared memory segment remains alive after this call (the local fd is
|
||||
closed, but the segment persists until ``_tensor_from_shm`` unlinks it).
|
||||
|
||||
If *d2h_stream* is provided, the D2H copy uses ``copy_()`` with
|
||||
``pin_memory=True`` on that stream instead of the synchronous ``.cpu()``
|
||||
path. The caller must synchronize *d2h_stream* after all tensors are
|
||||
packed.
|
||||
"""
|
||||
from multiprocessing import shared_memory
|
||||
|
||||
import numpy as np
|
||||
|
||||
tensor = tensor.detach().cpu().contiguous()
|
||||
original_dtype = tensor.dtype
|
||||
# NumPy does not support bfloat16; promote to float32 for the SHM
|
||||
# transfer and record the original dtype so _tensor_from_shm can
|
||||
# convert back. The round-trip is lossless for bfloat16 values.
|
||||
if original_dtype == torch.bfloat16:
|
||||
tensor = tensor.to(torch.float32)
|
||||
if d2h_stream is not None:
|
||||
# Non-blocking D2H: copy on side stream to pinned CPU memory.
|
||||
old_stream = torch.accelerator.current_stream()
|
||||
torch.accelerator.set_stream(d2h_stream)
|
||||
try:
|
||||
t = tensor.detach()
|
||||
if original_dtype == torch.bfloat16:
|
||||
t = t.to(torch.float32)
|
||||
cpu = torch.empty(t.shape, dtype=t.dtype, pin_memory=True)
|
||||
cpu.copy_(t, non_blocking=True)
|
||||
finally:
|
||||
torch.accelerator.set_stream(old_stream)
|
||||
d2h_stream.synchronize()
|
||||
tensor = cpu
|
||||
else:
|
||||
tensor = tensor.detach().cpu().contiguous()
|
||||
if original_dtype == torch.bfloat16:
|
||||
tensor = tensor.to(torch.float32)
|
||||
arr = tensor.numpy()
|
||||
nbytes = arr.nbytes
|
||||
shm = shared_memory.SharedMemory(create=True, size=nbytes)
|
||||
@@ -79,14 +99,20 @@ def _tensor_from_shm(handle: dict[str, Any]) -> torch.Tensor:
|
||||
return tensor
|
||||
|
||||
|
||||
def _pack_tensor_if_large(val: torch.Tensor) -> torch.Tensor | dict:
|
||||
def _pack_tensor_if_large(
|
||||
val: torch.Tensor,
|
||||
d2h_stream: torch.Stream | None = None,
|
||||
) -> torch.Tensor | dict:
|
||||
"""Replace a tensor with an SHM handle if it exceeds the threshold."""
|
||||
if val.nelement() * val.element_size() > _SHM_TENSOR_THRESHOLD:
|
||||
return _tensor_to_shm(val)
|
||||
return _tensor_to_shm(val, d2h_stream=d2h_stream)
|
||||
return val
|
||||
|
||||
|
||||
def _pack_value_if_large(val: object) -> object:
|
||||
def _pack_value_if_large(
|
||||
val: object,
|
||||
d2h_stream: torch.Stream | None = None,
|
||||
) -> object:
|
||||
"""Recursively replace large tensors with SHM handles.
|
||||
|
||||
Walks the container shapes pipelines return as ``DiffusionOutput.output``:
|
||||
@@ -96,13 +122,13 @@ def _pack_value_if_large(val: object) -> object:
|
||||
the two in sync.
|
||||
"""
|
||||
if isinstance(val, torch.Tensor):
|
||||
return _pack_tensor_if_large(val)
|
||||
return _pack_tensor_if_large(val, d2h_stream=d2h_stream)
|
||||
if isinstance(val, dict):
|
||||
return {key: _pack_value_if_large(value) for key, value in val.items()}
|
||||
return {key: _pack_value_if_large(value, d2h_stream=d2h_stream) for key, value in val.items()}
|
||||
if isinstance(val, list):
|
||||
return [_pack_value_if_large(item) for item in val]
|
||||
return [_pack_value_if_large(item, d2h_stream=d2h_stream) for item in val]
|
||||
if isinstance(val, tuple):
|
||||
return tuple(_pack_value_if_large(item) for item in val)
|
||||
return tuple(_pack_value_if_large(item, d2h_stream=d2h_stream) for item in val)
|
||||
return val
|
||||
|
||||
|
||||
@@ -119,15 +145,18 @@ def _unpack_if_shm_handle(val: object) -> object:
|
||||
return val
|
||||
|
||||
|
||||
def _pack_diffusion_fields(output: DiffusionOutput) -> DiffusionOutput:
|
||||
def _pack_diffusion_fields(
|
||||
output: DiffusionOutput,
|
||||
d2h_stream: torch.Stream | None = None,
|
||||
) -> DiffusionOutput:
|
||||
if output.output is not None:
|
||||
output.output = _pack_value_if_large(output.output)
|
||||
output.output = _pack_value_if_large(output.output, d2h_stream=d2h_stream)
|
||||
if output.trajectory_latents is not None and isinstance(output.trajectory_latents, torch.Tensor):
|
||||
output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents)
|
||||
output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents, d2h_stream=d2h_stream)
|
||||
if output.trajectory_timesteps is not None and isinstance(output.trajectory_timesteps, torch.Tensor):
|
||||
output.trajectory_timesteps = _pack_tensor_if_large(output.trajectory_timesteps)
|
||||
output.trajectory_timesteps = _pack_tensor_if_large(output.trajectory_timesteps, d2h_stream=d2h_stream)
|
||||
if output.trajectory_log_probs is not None and isinstance(output.trajectory_log_probs, torch.Tensor):
|
||||
output.trajectory_log_probs = _pack_tensor_if_large(output.trajectory_log_probs)
|
||||
output.trajectory_log_probs = _pack_tensor_if_large(output.trajectory_log_probs, d2h_stream=d2h_stream)
|
||||
return output
|
||||
|
||||
|
||||
@@ -135,31 +164,37 @@ def _is_rpc_result_envelope(output: object) -> bool:
|
||||
return isinstance(output, dict) and output.get("type") == DIFFUSION_RPC_RESULT_ENVELOPE
|
||||
|
||||
|
||||
def pack_diffusion_output_shm(output: object) -> object:
|
||||
def pack_diffusion_output_shm(
|
||||
output: object,
|
||||
d2h_stream: torch.Stream | None = None,
|
||||
) -> object:
|
||||
"""Replace large tensors in diffusion worker outputs with SHM handles.
|
||||
|
||||
Supports a bare ``DiffusionOutput``, a wrapper object carrying one in
|
||||
``.result`` (for example ``RunnerOutput``), an RPC result envelope carrying
|
||||
the diffusion output in ``["result"]``, or a batch wrapper carrying
|
||||
``RunnerOutput`` objects in ``.runner_outputs``.
|
||||
|
||||
If *d2h_stream* is provided, D2H copies use that stream (non-blocking on
|
||||
the default stream). The caller must synchronize *d2h_stream* afterward.
|
||||
"""
|
||||
if isinstance(output, DiffusionOutput):
|
||||
return _pack_diffusion_fields(output)
|
||||
return _pack_diffusion_fields(output, d2h_stream=d2h_stream)
|
||||
|
||||
if _is_rpc_result_envelope(output):
|
||||
result = output.get("result")
|
||||
if isinstance(result, DiffusionOutput):
|
||||
output["result"] = _pack_diffusion_fields(result)
|
||||
output["result"] = _pack_diffusion_fields(result, d2h_stream=d2h_stream)
|
||||
return output
|
||||
|
||||
result = getattr(output, "result", None)
|
||||
if isinstance(result, DiffusionOutput):
|
||||
output.result = _pack_diffusion_fields(result)
|
||||
output.result = _pack_diffusion_fields(result, d2h_stream=d2h_stream)
|
||||
|
||||
runner_outputs = getattr(output, "runner_outputs", None)
|
||||
if isinstance(runner_outputs, list):
|
||||
for runner_output in runner_outputs:
|
||||
pack_diffusion_output_shm(runner_output)
|
||||
pack_diffusion_output_shm(runner_output, d2h_stream=d2h_stream)
|
||||
return output
|
||||
|
||||
|
||||
|
||||
@@ -3233,8 +3233,7 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
assert image.shape[2] == 1, "image should have shape [B, C, T, H, W] and T should be 1"
|
||||
image = image.squeeze(2)
|
||||
|
||||
do_denormalize = [True] * image.shape[0]
|
||||
image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
|
||||
# Denormalize + PIL moved to engine post_process_func for overlap with next request.
|
||||
|
||||
if not return_dict:
|
||||
return (image,)
|
||||
|
||||
@@ -71,6 +71,8 @@ _STEP_COT_TEXT_LIST = "hunyuan_cot_text_list"
|
||||
_STEP_AR_KV = "hunyuan_ar_kv"
|
||||
_STEP_PROMPT_KV = "hunyuan_prompt_kv"
|
||||
|
||||
_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil"
|
||||
|
||||
|
||||
def default(val, d):
|
||||
return val if val is not None else d
|
||||
@@ -333,6 +335,41 @@ def get_hunyuan_image_3_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
return pre_process_func
|
||||
|
||||
|
||||
def get_hunyuan_image3_post_process_func(od_config: OmniDiffusionConfig):
|
||||
"""GPU tensor → PIL, runs outside pipeline_forward to overlap with next request."""
|
||||
from diffusers.image_processor import VaeImageProcessor
|
||||
|
||||
image_processor = VaeImageProcessor()
|
||||
|
||||
def post_process_func(images: torch.Tensor | dict):
|
||||
# Handle dict envelope format (upstream): {"payload": {"image": ...}, "metadata": ...}
|
||||
if isinstance(images, dict) and "payload" in images:
|
||||
tensor = images["payload"].get("image")
|
||||
if tensor is None:
|
||||
return images
|
||||
if tensor.dim() == 3:
|
||||
tensor = tensor.unsqueeze(0)
|
||||
do_denormalize = [True] * tensor.shape[0]
|
||||
images["payload"]["image"] = image_processor.postprocess(
|
||||
tensor,
|
||||
output_type=_HUNYUAN_DEFAULT_OUTPUT_TYPE,
|
||||
do_denormalize=do_denormalize,
|
||||
)
|
||||
return images
|
||||
|
||||
# Legacy raw tensor format
|
||||
if images.dim() == 3:
|
||||
images = images.unsqueeze(0)
|
||||
do_denormalize = [True] * images.shape[0]
|
||||
return image_processor.postprocess(
|
||||
images,
|
||||
output_type=_HUNYUAN_DEFAULT_OUTPUT_TYPE,
|
||||
do_denormalize=do_denormalize,
|
||||
)
|
||||
|
||||
return post_process_func
|
||||
|
||||
|
||||
class HunyuanImage3Pipeline(
|
||||
HunyuanImage3PreTrainedModel,
|
||||
GenerationMixin,
|
||||
@@ -2224,11 +2261,7 @@ class HunyuanImage3Pipeline(
|
||||
if hasattr(self.vae, "ffactor_temporal"):
|
||||
assert image.shape[2] == 1, "image should have shape [B, C, T, H, W] and T should be 1"
|
||||
image = image.squeeze(2)
|
||||
image = self.pipeline.image_processor.postprocess(
|
||||
image,
|
||||
output_type=output_type,
|
||||
do_denormalize=[True] * image.shape[0],
|
||||
)
|
||||
# Postprocess deferred to engine post_process_func for overlap with next request.
|
||||
|
||||
cot_text_list = state.extra.get(_STEP_COT_TEXT_LIST) or []
|
||||
metadata = {}
|
||||
|
||||
@@ -530,6 +530,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"Flux2Pipeline": "get_flux2_post_process_func",
|
||||
"HunyuanVideo15Pipeline": "get_hunyuan_video_15_post_process_func",
|
||||
"HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_post_process_func",
|
||||
"HunyuanImage3Pipeline": "get_hunyuan_image3_post_process_func",
|
||||
"LingBotVideoPipeline": "get_lingbot_video_post_process_func",
|
||||
"MagiHumanPipeline": "get_magi_human_post_process_func",
|
||||
"OmniVoicePipeline": "get_omnivoice_post_process_func",
|
||||
@@ -540,6 +541,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"HiDreamImagePipeline": "get_hidream_image_post_process_func",
|
||||
"StableDiffusionXLPipeline": "get_sdxl_image_post_process_func",
|
||||
"Krea2Pipeline": "get_krea2_post_process_func",
|
||||
"HunyuanImage3ForCausalMM": "get_hunyuan_image3_post_process_func",
|
||||
}
|
||||
|
||||
_DIFFUSION_IR_OP_PRIORITY_FUNCS = {
|
||||
|
||||
@@ -50,8 +50,14 @@ class RequestScheduler(BaseScheduler):
|
||||
req_output = output.get_request_output(request_id)
|
||||
result = req_output.result if req_output is not None else None
|
||||
if result is None:
|
||||
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ERROR
|
||||
terminal_errors[request_id] = "No output result"
|
||||
# Async mode: result=None with async_output_id means compute done,
|
||||
# final output will arrive later via wait_output_ready.
|
||||
if req_output is not None and req_output.async_output_id is not None:
|
||||
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_COMPLETED
|
||||
terminal_errors[request_id] = None
|
||||
else:
|
||||
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ERROR
|
||||
terminal_errors[request_id] = "No output result"
|
||||
elif result.aborted:
|
||||
terminal_statuses[request_id] = DiffusionRequestStatus.FINISHED_ABORTED
|
||||
terminal_errors[request_id] = None
|
||||
|
||||
@@ -11,8 +11,11 @@ to DiffusionModelRunner.
|
||||
import gc
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import threading
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
@@ -30,6 +33,8 @@ from vllm.utils.mem_utils import GiB_bytes
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from vllm_omni.diffusion.data import (
|
||||
AsyncDiffusionOutput,
|
||||
AsyncOutputKind,
|
||||
DiffusionOutput,
|
||||
OmniACK,
|
||||
OmniDiffusionConfig,
|
||||
@@ -794,6 +799,21 @@ class WorkerProc:
|
||||
self.worker = self._create_worker(gpu_id, od_config, worker_extension_cls, custom_pipeline_args)
|
||||
self._running = True
|
||||
|
||||
self._async_output_queue: queue.Queue | None = None
|
||||
self._async_output_thread: threading.Thread | None = None
|
||||
if not self.od_config.step_execution:
|
||||
self._async_output_queue = queue.Queue()
|
||||
self._async_output_thread = threading.Thread(
|
||||
target=self._async_output_loop,
|
||||
daemon=True,
|
||||
name="DiffusionAsyncOutput",
|
||||
)
|
||||
self._async_output_thread.start()
|
||||
|
||||
@staticmethod
|
||||
def _generate_async_output_id() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
def _create_worker(
|
||||
self,
|
||||
gpu_id: int,
|
||||
@@ -813,18 +833,72 @@ class WorkerProc:
|
||||
)
|
||||
return wrapper
|
||||
|
||||
def _return_result(self, output: Any) -> None:
|
||||
def _return_result(self, output: Any, rpc_id: str | None = None) -> None:
|
||||
"""Reply to client, only on rank 0."""
|
||||
if self.result_mq is not None:
|
||||
if isinstance(output, OmniACK):
|
||||
self.result_mq.enqueue(output)
|
||||
return
|
||||
try:
|
||||
pack_diffusion_output_shm(output)
|
||||
except Exception as e:
|
||||
if hasattr(output, "output"):
|
||||
logger.warning("SHM pack failed for model output: %s", e)
|
||||
if self.result_mq is None:
|
||||
return
|
||||
if isinstance(output, OmniACK):
|
||||
self.result_mq.enqueue(output)
|
||||
return
|
||||
|
||||
# Async path: enqueue compute_done immediately, bg thread does D2H+SHM.
|
||||
if not self.od_config.step_execution and isinstance(output, (DiffusionOutput, BatchRunnerOutput)):
|
||||
async_output_id = WorkerProc._generate_async_output_id()
|
||||
gpu_event = current_omni_platform.record_device_event()
|
||||
self._async_output_queue.put((output, async_output_id, gpu_event))
|
||||
msg = AsyncDiffusionOutput(
|
||||
kind=AsyncOutputKind.COMPUTE_DONE,
|
||||
rpc_id=rpc_id,
|
||||
async_output_id=async_output_id,
|
||||
)
|
||||
self.result_mq.enqueue(msg)
|
||||
return
|
||||
|
||||
# Sync path (original, or async fallback).
|
||||
try:
|
||||
pack_diffusion_output_shm(output)
|
||||
except Exception as e:
|
||||
if hasattr(output, "output"):
|
||||
logger.warning("SHM pack failed for model output: %s", e)
|
||||
self.result_mq.enqueue(output)
|
||||
|
||||
def _async_output_loop(self):
|
||||
"""Background thread: D2H + SHM packing for async diffusion output.
|
||||
|
||||
Uses a side stream so the D2H transfer does not block the default
|
||||
stream where the next forward runs.
|
||||
"""
|
||||
device = torch.device(torch.accelerator.current_accelerator().type, self.gpu_id)
|
||||
d2h_stream = torch.Stream(device=device)
|
||||
while self._running:
|
||||
output, async_output_id, gpu_event = self._async_output_queue.get()
|
||||
try:
|
||||
# Cross-stream ordering: wait for default stream to finish
|
||||
# writing the output tensors before the side stream reads.
|
||||
if gpu_event is not None:
|
||||
d2h_stream.wait_event(gpu_event)
|
||||
pack_diffusion_output_shm(output, d2h_stream=d2h_stream)
|
||||
d2h_stream.synchronize()
|
||||
|
||||
self.result_mq.enqueue(
|
||||
AsyncDiffusionOutput(
|
||||
kind=AsyncOutputKind.OUTPUT_READY,
|
||||
async_output_id=async_output_id,
|
||||
output=output,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Async output packing failed for id '%s'; sending error",
|
||||
async_output_id,
|
||||
)
|
||||
self.result_mq.enqueue(
|
||||
AsyncDiffusionOutput(
|
||||
kind=AsyncOutputKind.OUTPUT_READY,
|
||||
async_output_id=async_output_id,
|
||||
error="Background D2H/SHM packing failed",
|
||||
)
|
||||
)
|
||||
|
||||
def _gather_rpc_rank_statuses(self, status: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if not torch.distributed.is_initialized():
|
||||
@@ -936,13 +1010,25 @@ class WorkerProc:
|
||||
# Route message based on type
|
||||
elif isinstance(msg, dict) and msg.get("type") == "rpc":
|
||||
try:
|
||||
rpc_id = msg.get("rpc_id")
|
||||
result, should_reply = self._execute_rpc(msg)
|
||||
if should_reply:
|
||||
self._return_result(result)
|
||||
self._return_result(result, rpc_id=rpc_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing RPC: {e}", exc_info=True)
|
||||
if self.result_mq is not None:
|
||||
self._return_result({"status": "error", "error": str(e)})
|
||||
if rpc_id is not None:
|
||||
# Async RPC: must complete the executor''s pending
|
||||
# future so collective_rpc() doesn''t hang.
|
||||
self.result_mq.enqueue(
|
||||
AsyncDiffusionOutput(
|
||||
kind=AsyncOutputKind.RPC_RESULT,
|
||||
rpc_id=rpc_id,
|
||||
error=str(e),
|
||||
)
|
||||
)
|
||||
else:
|
||||
self._return_result({"status": "error", "error": str(e)})
|
||||
|
||||
elif isinstance(msg, dict) and msg.get("type") == "shutdown":
|
||||
logger.info("Worker %s: Received shutdown message", self.gpu_id)
|
||||
|
||||
@@ -183,6 +183,7 @@ class RunnerOutput(BaseRunnerOutput):
|
||||
step_index: int | None = None
|
||||
finished: bool = False
|
||||
result: DiffusionOutput | None = None
|
||||
async_output_id: str | None = None
|
||||
|
||||
def get_request_output(self, request_id: str) -> RunnerOutput | None:
|
||||
return self if self.request_id == request_id else None
|
||||
|
||||
Reference in new issue
Block a user