[Bugfix] Race condition in MultiprocExecutor when concurent access to Scheduler (#1448)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
03c01e0415
commit
5361353be5
4 files changed
+504
-45
No files matched your search
@@ -160,33 +160,43 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
}
|
||||
|
||||
try:
|
||||
# Broadcast RPC request to all workers via unified message queue
|
||||
self.scheduler.mq.enqueue(rpc_request)
|
||||
# Acquire lock with timeout awareness so that a stalled add_req
|
||||
# (holding the lock while blocked on dequeue) does not prevent
|
||||
# this RPC from honouring its own timeout.
|
||||
lock_timeout = None if deadline is None else max(0, deadline - time.monotonic())
|
||||
acquired = self.scheduler._lock.acquire(timeout=lock_timeout if lock_timeout is not None else -1)
|
||||
if not acquired:
|
||||
raise TimeoutError(f"RPC call to {method} timed out waiting for scheduler lock.")
|
||||
try:
|
||||
# Broadcast RPC request to all workers via unified message queue
|
||||
self.scheduler.mq.enqueue(rpc_request)
|
||||
|
||||
# Determine which workers we expect responses from
|
||||
num_responses = 1 if unique_reply_rank is not None else self.od_config.num_gpus
|
||||
# Determine which workers we expect responses from
|
||||
num_responses = 1 if unique_reply_rank is not None else self.od_config.num_gpus
|
||||
|
||||
responses = []
|
||||
for _ in range(num_responses):
|
||||
dequeue_timeout = None if deadline is None else (deadline - time.monotonic())
|
||||
try:
|
||||
if self.scheduler.result_mq is None:
|
||||
raise RuntimeError("Result queue not initialized")
|
||||
responses = []
|
||||
for _ in range(num_responses):
|
||||
dequeue_timeout = None if deadline is None else max(0, deadline - time.monotonic())
|
||||
try:
|
||||
if self.scheduler.result_mq is None:
|
||||
raise RuntimeError("Result queue not initialized")
|
||||
|
||||
response = self.scheduler.result_mq.dequeue(timeout=dequeue_timeout)
|
||||
response = self.scheduler.result_mq.dequeue(timeout=dequeue_timeout)
|
||||
|
||||
# Check if response indicates an error
|
||||
if isinstance(response, dict) and response.get("status") == "error":
|
||||
raise RuntimeError(
|
||||
f"Worker failed with error '{response.get('error')}', "
|
||||
"please check the stack trace above for the root cause"
|
||||
)
|
||||
# Check if response indicates an error
|
||||
if isinstance(response, dict) and response.get("status") == "error":
|
||||
raise RuntimeError(
|
||||
f"Worker failed with error '{response.get('error')}', "
|
||||
"please check the stack trace above for the root cause"
|
||||
)
|
||||
|
||||
responses.append(response)
|
||||
except TimeoutError as e:
|
||||
raise TimeoutError(f"RPC call to {method} timed out.") from e
|
||||
responses.append(response)
|
||||
except TimeoutError as e:
|
||||
raise TimeoutError(f"RPC call to {method} timed out.") from e
|
||||
|
||||
return responses[0] if unique_reply_rank is not None else responses
|
||||
return responses[0] if unique_reply_rank is not None else responses
|
||||
finally:
|
||||
self.scheduler._lock.release()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"RPC call failed: {e}")
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
|
||||
import zmq
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
from vllm.logger import init_logger
|
||||
@@ -20,6 +22,7 @@ class Scheduler:
|
||||
|
||||
self.num_workers = od_config.num_gpus
|
||||
self.od_config = od_config
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Initialize single MessageQueue for all message types (generation & RPC)
|
||||
# Assuming all readers are local for now as per current launch_engine implementation
|
||||
@@ -42,32 +45,33 @@ class Scheduler:
|
||||
|
||||
def add_req(self, request: OmniDiffusionRequest) -> DiffusionOutput:
|
||||
"""Sends a request to the scheduler and waits for the response."""
|
||||
try:
|
||||
# Prepare RPC request for generation
|
||||
rpc_request = {
|
||||
"type": "rpc",
|
||||
"method": "generate",
|
||||
"args": (request,),
|
||||
"kwargs": {},
|
||||
"output_rank": 0,
|
||||
"exec_all_ranks": True,
|
||||
}
|
||||
with self._lock:
|
||||
try:
|
||||
# Prepare RPC request for generation
|
||||
rpc_request = {
|
||||
"type": "rpc",
|
||||
"method": "generate",
|
||||
"args": (request,),
|
||||
"kwargs": {},
|
||||
"output_rank": 0,
|
||||
"exec_all_ranks": True,
|
||||
}
|
||||
|
||||
# Broadcast RPC request to all workers
|
||||
self.mq.enqueue(rpc_request)
|
||||
# Wait for result from Rank 0 (or whoever sends it)
|
||||
# Broadcast RPC request to all workers
|
||||
self.mq.enqueue(rpc_request)
|
||||
# Wait for result from Rank 0 (or whoever sends it)
|
||||
|
||||
if self.result_mq is None:
|
||||
raise RuntimeError("Result queue not initialized")
|
||||
if self.result_mq is None:
|
||||
raise RuntimeError("Result queue not initialized")
|
||||
|
||||
output = self.result_mq.dequeue()
|
||||
# {"status": "error", "error": str(e)}
|
||||
if isinstance(output, dict) and output.get("status") == "error":
|
||||
raise RuntimeError("worker error")
|
||||
return output
|
||||
except zmq.error.Again:
|
||||
logger.error("Timeout waiting for response from scheduler.")
|
||||
raise TimeoutError("Scheduler did not respond in time.")
|
||||
output = self.result_mq.dequeue()
|
||||
# {"status": "error", "error": str(e)}
|
||||
if isinstance(output, dict) and output.get("status") == "error":
|
||||
raise RuntimeError("worker error")
|
||||
return output
|
||||
except zmq.error.Again:
|
||||
logger.error("Timeout waiting for response from scheduler.")
|
||||
raise TimeoutError("Scheduler did not respond in time.")
|
||||
|
||||
def close(self):
|
||||
"""Closes the socket and terminates the context."""
|
||||
|
||||
@@ -143,7 +143,10 @@ class DiffusionWorker:
|
||||
self.rank,
|
||||
process_memory / GiB_bytes,
|
||||
)
|
||||
assert self.model_runner.pipeline is not None
|
||||
|
||||
# When load_format is "dummy", pipeline will init with custom pipeline later
|
||||
if load_format != "dummy":
|
||||
assert self.model_runner.pipeline is not None
|
||||
|
||||
def init_lora_manager(self) -> None:
|
||||
"""Initialize the LoRA manager for this worker."""
|
||||
|
||||
Reference in new issue
Block a user