[Bugfix] Release GPU memory after diffusion execution failures (#6385)
Signed-off-by: nagisa <1434936049@qq.com> Signed-off-by: nagisa-kun <1434936049@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
bd4f9acfd3
commit
855719e331
3 files changed
+84
-7
No files matched your search
@@ -71,6 +71,45 @@ def test_recv_message_waits_indefinitely():
|
||||
worker_proc.mq.dequeue.assert_called_once_with(indefinite=True)
|
||||
|
||||
|
||||
def test_busy_loop_releases_failed_rpc_traceback_and_device_cache(monkeypatch):
|
||||
original = RuntimeError("model execution failed")
|
||||
worker_proc = WorkerProc.__new__(WorkerProc)
|
||||
worker_proc.gpu_id = 0
|
||||
worker_proc.mq = Mock()
|
||||
worker_proc.mq.dequeue.side_effect = [
|
||||
{
|
||||
"type": "rpc",
|
||||
"method": "execute_model",
|
||||
"args": (),
|
||||
"kwargs": {},
|
||||
"output_rank": 0,
|
||||
"exec_all_ranks": False,
|
||||
"collect_rank_status": False,
|
||||
},
|
||||
{"type": "shutdown"},
|
||||
]
|
||||
worker_proc.worker = Mock()
|
||||
worker_proc.worker.execute_method.side_effect = original
|
||||
worker_proc.result_mq = object()
|
||||
worker_proc._return_result = Mock()
|
||||
worker_proc.wake_event = None
|
||||
worker_proc._running = True
|
||||
|
||||
gc_collect = Mock()
|
||||
mock_platform = Mock()
|
||||
monkeypatch.setattr(diffusion_worker_module.gc, "collect", gc_collect)
|
||||
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", mock_platform)
|
||||
|
||||
worker_proc._worker_busy_loop()
|
||||
|
||||
worker_proc._return_result.assert_called_once_with(
|
||||
{"status": "error", "error": "model execution failed", "wave_id": None}
|
||||
)
|
||||
assert original.__traceback__ is None
|
||||
gc_collect.assert_called_once_with()
|
||||
mock_platform.empty_cache.assert_called_once_with()
|
||||
|
||||
|
||||
def test_busy_loop_preserves_order_when_overflow_payload_is_delayed(monkeypatch):
|
||||
"""A consumed overflow marker must not be abandoned on a socket timeout."""
|
||||
writer = MessageQueue(
|
||||
|
||||
Reference in new issue
Block a user