[Bugfix] Race condition in MultiprocExecutor when concurent access to Scheduler (#1448)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
knlnguyen1802andHongsheng Liu authored and GitHub committed 2026-02-26 21:03:38 +08:00
1 parent 03c01e0415
commit 5361353be5
4 files changed
+504 -45

No files matched your search

@@ -160,33 +160,43 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
}
try:
# Broadcast RPC request to all workers via unified message queue
self.scheduler.mq.enqueue(rpc_request)
# Acquire lock with timeout awareness so that a stalled add_req
# (holding the lock while blocked on dequeue) does not prevent
# this RPC from honouring its own timeout.
lock_timeout = None if deadline is None else max(0, deadline - time.monotonic())
acquired = self.scheduler._lock.acquire(timeout=lock_timeout if lock_timeout is not None else -1)
if not acquired:
raise TimeoutError(f"RPC call to {method} timed out waiting for scheduler lock.")
try:
# Broadcast RPC request to all workers via unified message queue
self.scheduler.mq.enqueue(rpc_request)
# Determine which workers we expect responses from
num_responses = 1 if unique_reply_rank is not None else self.od_config.num_gpus
# Determine which workers we expect responses from
num_responses = 1 if unique_reply_rank is not None else self.od_config.num_gpus
responses = []
for _ in range(num_responses):
dequeue_timeout = None if deadline is None else (deadline - time.monotonic())
try:
if self.scheduler.result_mq is None:
raise RuntimeError("Result queue not initialized")
responses = []
for _ in range(num_responses):
dequeue_timeout = None if deadline is None else max(0, deadline - time.monotonic())
try:
if self.scheduler.result_mq is None:
raise RuntimeError("Result queue not initialized")
response = self.scheduler.result_mq.dequeue(timeout=dequeue_timeout)
response = self.scheduler.result_mq.dequeue(timeout=dequeue_timeout)
# Check if response indicates an error
if isinstance(response, dict) and response.get("status") == "error":
raise RuntimeError(
f"Worker failed with error '{response.get('error')}', "
"please check the stack trace above for the root cause"
)
# Check if response indicates an error
if isinstance(response, dict) and response.get("status") == "error":
raise RuntimeError(
f"Worker failed with error '{response.get('error')}', "
"please check the stack trace above for the root cause"
)
responses.append(response)
except TimeoutError as e:
raise TimeoutError(f"RPC call to {method} timed out.") from e
responses.append(response)
except TimeoutError as e:
raise TimeoutError(f"RPC call to {method} timed out.") from e
return responses[0] if unique_reply_rank is not None else responses
return responses[0] if unique_reply_rank is not None else responses
finally:
self.scheduler._lock.release()
except Exception as e:
logger.error(f"RPC call failed: {e}")
+27 -23
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import zmq
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
from vllm.logger import init_logger
@@ -20,6 +22,7 @@ class Scheduler:
self.num_workers = od_config.num_gpus
self.od_config = od_config
self._lock = threading.Lock()
# Initialize single MessageQueue for all message types (generation & RPC)
# Assuming all readers are local for now as per current launch_engine implementation
@@ -42,32 +45,33 @@ class Scheduler:
def add_req(self, request: OmniDiffusionRequest) -> DiffusionOutput:
"""Sends a request to the scheduler and waits for the response."""
try:
# Prepare RPC request for generation
rpc_request = {
"type": "rpc",
"method": "generate",
"args": (request,),
"kwargs": {},
"output_rank": 0,
"exec_all_ranks": True,
}
with self._lock:
try:
# Prepare RPC request for generation
rpc_request = {
"type": "rpc",
"method": "generate",
"args": (request,),
"kwargs": {},
"output_rank": 0,
"exec_all_ranks": True,
}
# Broadcast RPC request to all workers
self.mq.enqueue(rpc_request)
# Wait for result from Rank 0 (or whoever sends it)
# Broadcast RPC request to all workers
self.mq.enqueue(rpc_request)
# Wait for result from Rank 0 (or whoever sends it)
if self.result_mq is None:
raise RuntimeError("Result queue not initialized")
if self.result_mq is None:
raise RuntimeError("Result queue not initialized")
output = self.result_mq.dequeue()
# {"status": "error", "error": str(e)}
if isinstance(output, dict) and output.get("status") == "error":
raise RuntimeError("worker error")
return output
except zmq.error.Again:
logger.error("Timeout waiting for response from scheduler.")
raise TimeoutError("Scheduler did not respond in time.")
output = self.result_mq.dequeue()
# {"status": "error", "error": str(e)}
if isinstance(output, dict) and output.get("status") == "error":
raise RuntimeError("worker error")
return output
except zmq.error.Again:
logger.error("Timeout waiting for response from scheduler.")
raise TimeoutError("Scheduler did not respond in time.")
def close(self):
"""Closes the socket and terminates the context."""
@@ -143,7 +143,10 @@ class DiffusionWorker:
self.rank,
process_memory / GiB_bytes,
)
assert self.model_runner.pipeline is not None
# When load_format is "dummy", pipeline will init with custom pipeline later
if load_format != "dummy":
assert self.model_runner.pipeline is not None
def init_lora_manager(self) -> None:
"""Initialize the LoRA manager for this worker."""