[Diffusion] Fix DLO DP concurrent request execution (#5864)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
b15d4e6a64
commit
44e38d7b38
14 files changed
+1147
-127
No files matched your search
@@ -4,6 +4,7 @@ import concurrent.futures
|
||||
import json
|
||||
import multiprocessing as mp
|
||||
import multiprocessing.connection
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
@@ -21,6 +22,7 @@ from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, AsyncDiffusionOutput, AsyncOutputKind, DiffusionOutput
|
||||
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
|
||||
from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, unpack_diffusion_output_shm
|
||||
from vllm_omni.diffusion.sched.request_scheduler import build_request_batch_sampling_params_key
|
||||
from vllm_omni.diffusion.worker import WorkerProc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -30,6 +32,21 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DEQUEUE_TIMEOUT_S = 5.0
|
||||
_DLO_DP_WAVE_TIMEOUT_S = float(os.environ.get("VLLM_OMNI_DLO_DP_WAVE_TIMEOUT", 600.0))
|
||||
_WORKER_SHUTDOWN_GRACE_S = 15.0
|
||||
_WORKER_TERMINATE_GRACE_S = 5.0
|
||||
_RESULT_PUMP_JOIN_TIMEOUT_S = 2.0
|
||||
|
||||
|
||||
def _is_empty_dp_prompt(prompt: object) -> bool:
|
||||
"""Return whether a DP request has no usable text prompt."""
|
||||
if prompt is None:
|
||||
return True
|
||||
if isinstance(prompt, str):
|
||||
return not prompt
|
||||
if isinstance(prompt, dict) and "prompt" in prompt:
|
||||
return not prompt["prompt"]
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -52,14 +69,20 @@ class _ExecutorShutdownCleaner:
|
||||
logger.warning("Failed to send shutdown signal: %s", exc)
|
||||
|
||||
if self.processes:
|
||||
join_deadline = time.monotonic() + _WORKER_SHUTDOWN_GRACE_S
|
||||
for proc in self.processes:
|
||||
if not proc.is_alive():
|
||||
continue
|
||||
proc.join(5)
|
||||
if proc.is_alive():
|
||||
logger.warning("Terminating diffusion worker %s after timeout", proc.name)
|
||||
proc.terminate()
|
||||
proc.join(5)
|
||||
proc.join(max(0.0, join_deadline - time.monotonic()))
|
||||
|
||||
alive = [proc for proc in self.processes if proc.is_alive()]
|
||||
for proc in alive:
|
||||
logger.warning("Terminating diffusion worker %s after timeout", proc.name)
|
||||
proc.terminate()
|
||||
|
||||
terminate_deadline = time.monotonic() + _WORKER_TERMINATE_GRACE_S
|
||||
for proc in alive:
|
||||
proc.join(max(0.0, terminate_deadline - time.monotonic()))
|
||||
|
||||
|
||||
class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
@@ -75,6 +98,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
self._is_failed = False
|
||||
self._failure_callbacks: list[Callable[[], None]] = []
|
||||
self._result_mq: MessageQueue | None = None
|
||||
self._result_mqs: list[MessageQueue] = []
|
||||
self._rpc_wave_id: int = 0
|
||||
|
||||
num_workers = cast(int, self.od_config.num_gpus)
|
||||
@@ -84,7 +108,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
broadcast_handle = self._broadcast_mq.export_handle()
|
||||
|
||||
# Launch workers
|
||||
processes, result_handle = self._launch_workers(broadcast_handle, self.wake_events)
|
||||
processes, result_handles = self._launch_workers(broadcast_handle, self.wake_events)
|
||||
self._processes = processes
|
||||
|
||||
shutdown_cleaner = _ExecutorShutdownCleaner(
|
||||
@@ -96,7 +120,8 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
self._finalizer = weakref.finalize(self, shutdown_cleaner)
|
||||
|
||||
try:
|
||||
self._result_mq = self._init_result_queue(result_handle)
|
||||
self._result_mqs = [self._init_result_queue(handle) for handle in result_handles]
|
||||
self._result_mq = self._result_mqs[0]
|
||||
except Exception:
|
||||
self.shutdown()
|
||||
raise
|
||||
@@ -112,8 +137,8 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
self._futures_lock = threading.RLock()
|
||||
self._pump_running = False
|
||||
self._pump_stop = threading.Event()
|
||||
# When pump is active it is the sole reader of result_mq; non-async
|
||||
# messages are placed here for collective_rpc() to consume.
|
||||
# When pumps are active they are the sole readers of the worker result
|
||||
# queues; non-async messages are placed here for collective_rpc().
|
||||
self._sync_result_buffer: queue.Queue = queue.Queue()
|
||||
if not self.od_config.step_execution:
|
||||
self._start_result_pump()
|
||||
@@ -140,12 +165,17 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
if self._broadcast_mq is None:
|
||||
raise RuntimeError("Broadcast queue is closed")
|
||||
|
||||
def _dequeue_one_with_failure_polling(self, deadline: float | None, method: str) -> Any:
|
||||
def _dequeue_one_with_failure_polling(
|
||||
self,
|
||||
deadline: float | None,
|
||||
method: str,
|
||||
result_mq: MessageQueue | None = None,
|
||||
) -> Any:
|
||||
"""Block until one result message, polling ``_is_failed`` between chunk timeouts.
|
||||
|
||||
When async output is enabled, pump is the sole reader of result_mq;
|
||||
non-async messages are placed in _sync_result_buffer, so this method
|
||||
reads from there instead of result_mq directly.
|
||||
When async output is enabled, the pumps are the sole readers of the
|
||||
worker result queues; non-async messages are placed in
|
||||
_sync_result_buffer, so this method reads from there instead.
|
||||
"""
|
||||
while True:
|
||||
if deadline is None:
|
||||
@@ -163,8 +193,11 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
raise EngineDeadError()
|
||||
continue
|
||||
else:
|
||||
queue_to_read = result_mq if result_mq is not None else self._result_mq
|
||||
if queue_to_read is None:
|
||||
raise RuntimeError("Result queue is closed")
|
||||
try:
|
||||
return self._result_mq.dequeue(timeout=chunk_timeout) # pyright: ignore[reportOptionalMemberAccess] MQ is not None before shutdown
|
||||
return queue_to_read.dequeue(timeout=chunk_timeout)
|
||||
except (TimeoutError, zmq.error.Again):
|
||||
if self._is_failed:
|
||||
raise EngineDeadError()
|
||||
@@ -223,7 +256,14 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
|
||||
_MAX_STALE_DISCARDS = 16
|
||||
|
||||
def _validate_wave_id(self, response: Any, expected_wave_id: int, deadline: float | None, method: str) -> Any:
|
||||
def _validate_wave_id(
|
||||
self,
|
||||
response: Any,
|
||||
expected_wave_id: int,
|
||||
deadline: float | None,
|
||||
method: str,
|
||||
result_mq: MessageQueue | None = None,
|
||||
) -> Any:
|
||||
"""Discard stale RPC responses from a previous wave."""
|
||||
discards = 0
|
||||
while discards < self._MAX_STALE_DISCARDS:
|
||||
@@ -239,7 +279,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
method,
|
||||
)
|
||||
discards += 1
|
||||
response = self._dequeue_one_with_failure_polling(deadline, method)
|
||||
response = self._dequeue_one_with_failure_polling(deadline, method, result_mq)
|
||||
raise TimeoutError(
|
||||
f"Discarded {self._MAX_STALE_DISCARDS} stale RPC responses "
|
||||
f"without finding wave_id={expected_wave_id} for method={method}."
|
||||
@@ -249,7 +289,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
self,
|
||||
broadcast_handle: Handle,
|
||||
wake_events: list[Event],
|
||||
) -> tuple[list[mp.Process], Handle | None]:
|
||||
) -> tuple[list[mp.Process], list[Handle]]:
|
||||
od_config = self.od_config
|
||||
logger.info("Starting server...")
|
||||
|
||||
@@ -287,7 +327,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
processes.append(process)
|
||||
|
||||
# Wait for all workers to be ready
|
||||
result_handle = None
|
||||
result_handles: list[Handle] = []
|
||||
for writer in scheduler_pipe_writers:
|
||||
writer.close()
|
||||
|
||||
@@ -303,14 +343,16 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
if data["status"] != "ready":
|
||||
raise RuntimeError("Initialization failed. Please see the error messages above.")
|
||||
|
||||
if i == 0:
|
||||
result_handle = data.get("result_handle")
|
||||
result_handle = data.get("result_handle")
|
||||
if result_handle is None:
|
||||
raise RuntimeError(f"Rank {i} did not provide a result queue handle")
|
||||
result_handles.append(result_handle)
|
||||
|
||||
reader.close()
|
||||
|
||||
logger.debug("All workers are ready")
|
||||
|
||||
return processes, result_handle
|
||||
return processes, result_handles
|
||||
|
||||
@property
|
||||
def is_dead(self) -> bool:
|
||||
@@ -369,6 +411,27 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
t = threading.Thread(target=_monitor, daemon=True, name="diffusion-worker-monitor")
|
||||
t.start()
|
||||
|
||||
def _fail_closed_on_dp_wave_timeout(self, exc: TimeoutError) -> None:
|
||||
"""Shut down every rank after a partial DP wave times out.
|
||||
|
||||
Once one rank is stuck in a DLO AllGather, that process group cannot be
|
||||
reused safely. Shutting down the executor converts an otherwise
|
||||
permanent request hang into a bounded engine failure and lets the
|
||||
caller restart.
|
||||
"""
|
||||
logger.error(
|
||||
"DLO DP collective wave timed out after %.1fs; shutting down the worker group: %s",
|
||||
_DLO_DP_WAVE_TIMEOUT_S,
|
||||
exc,
|
||||
)
|
||||
self._is_failed = True
|
||||
self.shutdown()
|
||||
for callback in self._failure_callbacks:
|
||||
try:
|
||||
callback()
|
||||
except Exception:
|
||||
logger.exception("failure_callback raised")
|
||||
|
||||
def register_failure_callback(
|
||||
self,
|
||||
callback: Callable[[], None],
|
||||
@@ -398,21 +461,17 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
and getattr(self.od_config, "enable_distributed_layerwise_offload", False)
|
||||
and getattr(self.od_config, "dlo_use_allgather", True)
|
||||
):
|
||||
# Validate: all concurrent requests must share the same
|
||||
# num_inference_steps and identical extra_args, because
|
||||
# AllGather is a collective that requires every rank to
|
||||
# participate at each step.
|
||||
step_counts = {
|
||||
nr.req.sampling_params.num_inference_steps
|
||||
for nr in new_reqs
|
||||
if nr.req.sampling_params.num_inference_steps is not None
|
||||
}
|
||||
has_none = any(nr.req.sampling_params.num_inference_steps is None for nr in new_reqs)
|
||||
if (len(step_counts) > 1) or has_none:
|
||||
# Reuse the request scheduler's complete compatibility key. DLO
|
||||
# AllGather requires every DP rank to execute the same collective
|
||||
# schedule, including shape, CFG, denoise steps, output count,
|
||||
# and LoRA settings. Two default (None) step counts are valid and
|
||||
# resolve identically inside the same pipeline.
|
||||
compatibility_keys = [build_request_batch_sampling_params_key(nr.req) for nr in new_reqs]
|
||||
if any(key != compatibility_keys[0] for key in compatibility_keys[1:]):
|
||||
raise ValueError(
|
||||
"DP multi-concurrency requires all concurrent requests to have "
|
||||
"the same explicit num_inference_steps (None is not allowed), got "
|
||||
f"{[nr.req.sampling_params.num_inference_steps for nr in new_reqs]}."
|
||||
"DLO DP multi-concurrency requires compatible shape, CFG, "
|
||||
"denoise schedule, output count, and LoRA settings for all "
|
||||
"requests in one collective wave."
|
||||
)
|
||||
extra_args_signatures: set = set()
|
||||
for nr in new_reqs:
|
||||
@@ -424,11 +483,18 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
"share identical extra_args. Different extra_args can change "
|
||||
"the forward schedule and cause AllGather deadlock."
|
||||
)
|
||||
empty_prompt_ids = [nr.request_id for nr in new_reqs if _is_empty_dp_prompt(nr.req.prompt)]
|
||||
if empty_prompt_ids:
|
||||
raise ValueError(
|
||||
"DP multi-concurrency requires a non-empty prompt for every request; "
|
||||
f"empty prompt request IDs: {empty_prompt_ids}."
|
||||
)
|
||||
|
||||
reqs_list = [nr.req for nr in new_reqs]
|
||||
try:
|
||||
results = self.collective_rpc(
|
||||
"execute_model",
|
||||
timeout=_DLO_DP_WAVE_TIMEOUT_S,
|
||||
args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job),
|
||||
unique_reply_rank=None,
|
||||
exec_all_ranks=True,
|
||||
@@ -448,6 +514,8 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
else:
|
||||
raise RuntimeError(f"Unexpected response type [{i}]: {type(res)!r}")
|
||||
except Exception as exc:
|
||||
if isinstance(exc, TimeoutError):
|
||||
self._fail_closed_on_dp_wave_timeout(exc)
|
||||
for new_req in new_reqs:
|
||||
runner_outputs.append(
|
||||
RunnerOutput(
|
||||
@@ -591,14 +659,18 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
kwargs = kwargs or {}
|
||||
|
||||
multi_rank_reply = unique_reply_rank is None and exec_all_ranks
|
||||
execute_all_ranks = unique_reply_rank is None or exec_all_ranks
|
||||
collect_rank_status = unique_reply_rank is None
|
||||
# Status aggregation is for control-plane RPCs, where rank 0 sends
|
||||
# one envelope representing every rank. DP request concurrency needs
|
||||
# one independent reply per DP primary instead.
|
||||
collect_rank_status = unique_reply_rank is None and not multi_rank_reply
|
||||
rpc_request = {
|
||||
"type": "rpc",
|
||||
"method": method,
|
||||
"args": args,
|
||||
"kwargs": kwargs,
|
||||
"output_rank": unique_reply_rank if unique_reply_rank is not None else 0,
|
||||
"output_rank": None if multi_rank_reply else (unique_reply_rank if unique_reply_rank is not None else 0),
|
||||
"exec_all_ranks": execute_all_ranks,
|
||||
"collect_rank_status": collect_rank_status,
|
||||
}
|
||||
@@ -608,7 +680,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
# so the D2H copy runs on a side stream in the worker background
|
||||
# thread while the default stream is free for the next forward.
|
||||
# pump routes the result back to a Future via rpc_id.
|
||||
multi_rank_reply = unique_reply_rank is None and exec_all_ranks
|
||||
if (
|
||||
not self.od_config.step_execution
|
||||
and method in ("execute_model", "execute_model_batch")
|
||||
@@ -651,11 +722,28 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
responses: list = []
|
||||
if unique_reply_rank is None and exec_all_ranks and num_responses > 1:
|
||||
# DP multi-concurrency: collect num_responses replies, sort by dp_rank.
|
||||
result_mqs: list[MessageQueue | None] = [None] * num_responses
|
||||
if self.od_config.step_execution:
|
||||
parallel_config = self.od_config.parallel_config
|
||||
replica_size = (
|
||||
getattr(parallel_config, "tensor_parallel_size", 1)
|
||||
* getattr(parallel_config, "sequence_parallel_size", 1)
|
||||
* getattr(parallel_config, "pipeline_parallel_size", 1)
|
||||
* getattr(parallel_config, "cfg_parallel_size", 1)
|
||||
)
|
||||
primary_ranks = [dp_rank * replica_size for dp_rank in range(num_responses)]
|
||||
all_result_mqs = getattr(self, "_result_mqs", [])
|
||||
if not all_result_mqs or primary_ranks[-1] >= len(all_result_mqs):
|
||||
raise RuntimeError(
|
||||
"Missing result queues for DP primary ranks "
|
||||
f"{primary_ranks}; available queues: {len(all_result_mqs)}"
|
||||
)
|
||||
result_mqs = [all_result_mqs[rank] for rank in primary_ranks]
|
||||
tagged: list[tuple[int, Any]] = []
|
||||
collected_errors: list[str] = []
|
||||
for _ in range(num_responses):
|
||||
response = self._dequeue_one_with_failure_polling(deadline, method)
|
||||
response = self._validate_wave_id(response, wave_id, deadline, method)
|
||||
for result_mq in result_mqs:
|
||||
response = self._dequeue_one_with_failure_polling(deadline, method, result_mq)
|
||||
response = self._validate_wave_id(response, wave_id, deadline, method, result_mq)
|
||||
try:
|
||||
unpack_diffusion_output_shm(response)
|
||||
except Exception as e:
|
||||
@@ -698,20 +786,35 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
def _start_result_pump(self) -> None:
|
||||
self._pump_running = True
|
||||
self._pump_stop.clear()
|
||||
self._result_pump_thread = threading.Thread(target=self._result_pump, daemon=True, name="DiffusionResultPump")
|
||||
self._result_pump_thread.start()
|
||||
logger.info("Async result pump started")
|
||||
result_mqs = self._result_mqs or ([self._result_mq] if self._result_mq is not None else [])
|
||||
self._result_pump_threads = [
|
||||
threading.Thread(
|
||||
target=self._result_pump,
|
||||
args=(result_mq,),
|
||||
daemon=True,
|
||||
name=f"DiffusionResultPump-{rank}",
|
||||
)
|
||||
for rank, result_mq in enumerate(result_mqs)
|
||||
]
|
||||
for thread in self._result_pump_threads:
|
||||
thread.start()
|
||||
self._result_pump_thread = self._result_pump_threads[0] if self._result_pump_threads else None
|
||||
logger.info("Async result pump started for %d worker queue(s)", len(self._result_pump_threads))
|
||||
|
||||
def _result_pump(self) -> None:
|
||||
"""Sole reader of result_mq when async output is enabled.
|
||||
def _result_pump(self, result_mq: MessageQueue | None = None) -> None:
|
||||
"""Sole reader of one worker result queue when async output is enabled.
|
||||
|
||||
Dispatches AsyncDiffusionOutput messages to the appropriate future:
|
||||
* RPC_RESULT / COMPUTE_DONE → _rpc_futures[rpc_id]
|
||||
* OUTPUT_READY → _output_futures[async_output_id]
|
||||
"""
|
||||
result_mq = result_mq or self._result_mq
|
||||
if result_mq is None:
|
||||
return
|
||||
|
||||
while not self._pump_stop.is_set():
|
||||
try:
|
||||
msg = self._result_mq.dequeue(timeout=1.0)
|
||||
msg = result_mq.dequeue(timeout=1.0)
|
||||
except TimeoutError:
|
||||
if self._is_failed:
|
||||
break
|
||||
@@ -823,8 +926,18 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
try:
|
||||
self._finalizer()
|
||||
finally:
|
||||
pump_threads = getattr(self, "_result_pump_threads", [])
|
||||
for thread in pump_threads:
|
||||
if thread is threading.current_thread():
|
||||
continue
|
||||
thread.join(timeout=_RESULT_PUMP_JOIN_TIMEOUT_S)
|
||||
if thread.is_alive():
|
||||
logger.warning("Result pump thread %s did not stop before shutdown", thread.name)
|
||||
self._pump_running = False
|
||||
self._broadcast_mq = None
|
||||
self._result_mq = None
|
||||
self._result_mqs = []
|
||||
self._result_pump_threads = []
|
||||
with self._futures_lock:
|
||||
for fut in self._rpc_futures.values():
|
||||
if not fut.done():
|
||||
|
||||
Reference in new issue
Block a user