[Bugfix] graceful shutdown for multi-stage engine processes (#3001)

Signed-off-by: Lancer <maruixiang6688@gmail.com>
This commit is contained in:
Lancer authored and GitHub committed 2026-04-24 20:01:01 +08:00
1 parent 0fe76857ac
commit b33d536f63
3 files changed
+149 -18

No files matched your search

+94
View File
@@ -10,11 +10,14 @@ from types import SimpleNamespace
from typing import Any
import janus
import psutil
import pytest
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.core_client import AsyncMPClient
from vllm_omni.engine.orchestrator import Orchestrator
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -104,6 +107,42 @@ class FakeOutputProcessor:
return None
class _FakeProc:
pid = 1234
def __init__(self):
self.terminated = False
self.killed = False
self.join_calls = []
def is_alive(self):
return not self.terminated and not self.killed
def terminate(self):
self.terminated = True
def kill(self):
self.killed = True
def join(self, timeout=None):
self.join_calls.append(timeout)
class _FakeChildProc:
def __init__(self):
self.terminated = False
self.killed = False
def is_running(self):
return not self.terminated and not self.killed
def terminate(self):
self.terminated = True
def kill(self):
self.killed = True
def _sampling_params(max_tokens: int = 4) -> SamplingParams:
return SamplingParams(max_tokens=max_tokens)
@@ -269,6 +308,61 @@ async def _enqueue_abort_request(orchestrator_fixture: OrchestratorFixture, requ
)
def test_stage_engine_core_client_shutdown_cleans_children_if_base_shutdown_fails(monkeypatch):
fake_proc = _FakeProc()
fake_child = _FakeChildProc()
class FakePsutilProcess:
def __init__(self, pid):
assert pid == fake_proc.pid
def children(self, recursive=True):
assert recursive
return [fake_child]
def fail_base_shutdown(self):
raise RuntimeError("base shutdown failed")
monkeypatch.setattr(psutil, "Process", FakePsutilProcess)
monkeypatch.setattr(psutil, "wait_procs", lambda procs, timeout: (list(procs), []))
monkeypatch.setattr(AsyncMPClient, "shutdown", fail_base_shutdown)
client = object.__new__(StageEngineCoreClient)
client._proc = fake_proc
with pytest.raises(RuntimeError, match="base shutdown failed"):
client.shutdown()
assert fake_proc.terminated
assert fake_proc.join_calls == [5]
assert fake_child.terminated
def test_stage_engine_core_client_shutdown_kills_stubborn_children(monkeypatch):
fake_proc = _FakeProc()
fake_child = _FakeChildProc()
class FakePsutilProcess:
def __init__(self, pid):
assert pid == fake_proc.pid
def children(self, recursive=True):
assert recursive
return [fake_child]
monkeypatch.setattr(psutil, "Process", FakePsutilProcess)
monkeypatch.setattr(psutil, "wait_procs", lambda procs, timeout: ([], list(procs)))
monkeypatch.setattr(AsyncMPClient, "shutdown", lambda self: None)
client = object.__new__(StageEngineCoreClient)
client._proc = fake_proc
client.shutdown()
assert fake_child.terminated
assert fake_child.killed
@pytest.fixture
def orchestrator_factory():
fixtures: list[OrchestratorFixture] = []