[Bugfix] Release GPU memory after diffusion execution failures (#6385)

Signed-off-by: nagisa <1434936049@qq.com>
Signed-off-by: nagisa-kun <1434936049@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
hurukawaandHongsheng Liu authored and GitHub committed 2026-08-22 18:32:48 +00:00
1 parent bd4f9acfd3
commit 855719e331
3 files changed
+84 -7

No files matched your search

@@ -71,6 +71,45 @@ def test_recv_message_waits_indefinitely():
worker_proc.mq.dequeue.assert_called_once_with(indefinite=True)
def test_busy_loop_releases_failed_rpc_traceback_and_device_cache(monkeypatch):
original = RuntimeError("model execution failed")
worker_proc = WorkerProc.__new__(WorkerProc)
worker_proc.gpu_id = 0
worker_proc.mq = Mock()
worker_proc.mq.dequeue.side_effect = [
{
"type": "rpc",
"method": "execute_model",
"args": (),
"kwargs": {},
"output_rank": 0,
"exec_all_ranks": False,
"collect_rank_status": False,
},
{"type": "shutdown"},
]
worker_proc.worker = Mock()
worker_proc.worker.execute_method.side_effect = original
worker_proc.result_mq = object()
worker_proc._return_result = Mock()
worker_proc.wake_event = None
worker_proc._running = True
gc_collect = Mock()
mock_platform = Mock()
monkeypatch.setattr(diffusion_worker_module.gc, "collect", gc_collect)
monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", mock_platform)
worker_proc._worker_busy_loop()
worker_proc._return_result.assert_called_once_with(
{"status": "error", "error": "model execution failed", "wave_id": None}
)
assert original.__traceback__ is None
gc_collect.assert_called_once_with()
mock_platform.empty_cache.assert_called_once_with()
def test_busy_loop_preserves_order_when_overflow_payload_is_delayed(monkeypatch):
"""A consumed overflow marker must not be abandoned on a socket timeout."""
writer = MessageQueue(