Parallel stage initialization (admission + SH/EX device locks) (Worker refactor 1/N) (#5224)

Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk>
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Zhou TaichangandClaude Fable 5 authored and GitHub committed 2026-09-05 22:22:41 +08:00
1 parent 5d7fdf9350
commit 142151fef5
15 files changed
+1888 -194

No files matched your search

@@ -205,6 +205,7 @@ def test_async_omni_engine_initialize_stages_passes_log_stats_to_runtime(monkeyp
engine._omni_lb_policy = "random"
engine.request_queue = types.SimpleNamespace()
engine._log_stats = True
engine._parallel_stage_init = False
captured: dict[str, object] = {}
runtime = types.SimpleNamespace(stage_pools=[], initialize=lambda: None)
+842
View File
@@ -0,0 +1,842 @@
"""Unit tests for parallel stage initialization (CPU-only).
Covers the pure/lock logic added for ``parallel_stage_init``:
* init-group device keying (resolved physical set; parallel unique keys),
* admission control arithmetic + graph reserve,
* SH/EX device phase locks (real flock on a temp dir),
* the phase-locked executor wrapper ordering,
* a regression guard that the NVML process-scope path stayed removed.
The GPU-only same-device concurrency (actual overlapping init) requires a
hardware soak and is out of scope for these unit tests.
"""
from __future__ import annotations
import contextlib
import inspect
import os
import subprocess
import threading
import types
from pathlib import Path
import pytest
from vllm_omni.engine.stage_admission import (
ADMISSION_EXEMPT,
DeviceLedger,
StageAdmissionError,
StageDemand,
check_admission,
evaluate,
graph_reserve_bytes,
)
from vllm_omni.engine.stage_engine_core_proc import _install_phase_locks
from vllm_omni.engine.stage_init_utils import (
LogicalStageInitPlan,
ReplicaInitPlan,
device_init_lock_path,
open_device_lock_file,
)
from vllm_omni.engine.stage_phase_lock import (
DeviceLockTimeoutError,
DevicePhaseLock,
resolve_driver_device_ids,
wrap_executor_with_phase_locks,
)
from vllm_omni.engine.stage_runtime import StageRuntime
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# --------------------------------------------------------------------------- #
# helpers
# --------------------------------------------------------------------------- #
def _llm_replica(stage_id: int, replica_id: int, devices: str | None, *, vllm_config=None) -> ReplicaInitPlan:
metadata = types.SimpleNamespace(
stage_id=stage_id,
stage_type="llm",
runtime_cfg={"devices": devices} if devices is not None else {},
replica_id=replica_id,
)
stage_cfg = types.SimpleNamespace(
stage_id=stage_id,
stage_type="llm",
runtime=types.SimpleNamespace(devices=devices),
engine_args={},
)
return ReplicaInitPlan(
replica_id=replica_id,
num_replicas=1,
launch_mode="local",
stage_cfg=stage_cfg,
metadata=metadata,
stage_connector_spec={},
omni_kv_connector=(None, None, None),
stage_vllm_config=vllm_config,
executor_class=object,
)
def _runtime(parallel_stage_init: bool = False) -> StageRuntime:
return StageRuntime(
stage_configs=[],
model="dummy",
config_path="dummy",
stage_init_timeout=5,
diffusion_batch_size=1,
async_chunk=False,
parallel_stage_init=parallel_stage_init,
)
def _fake_vllm_config(util: float, *, model="m", dtype="bf16", cudagraph_mode="FULL", capture_sizes=(1, 2, 4)):
return types.SimpleNamespace(
cache_config=types.SimpleNamespace(gpu_memory_utilization=util),
model_config=types.SimpleNamespace(model=model, dtype=dtype, enforce_eager=False),
compilation_config=types.SimpleNamespace(
cudagraph_mode=cudagraph_mode,
cudagraph_capture_sizes=list(capture_sizes),
),
)
# --------------------------------------------------------------------------- #
# A1: init-group device key
# --------------------------------------------------------------------------- #
def test_init_group_key_resolves_physical_devices_serial():
runtime = _runtime(parallel_stage_init=False)
runtime._init_visible_devices_baseline = None # no CUDA_VISIBLE_DEVICES mapping
r0 = _llm_replica(0, 0, "0")
r1 = _llm_replica(1, 0, "1")
r0b = _llm_replica(2, 0, "0")
k0 = runtime._replica_init_group_key(r0)
k1 = runtime._replica_init_group_key(r1)
k0b = runtime._replica_init_group_key(r0b)
assert k0 == "device:0"
assert k1 == "device:1"
# Different physical devices -> different groups (parallel).
assert k0 != k1
# Same physical device -> same group (serialized under LOCK_EX).
assert k0 == k0b
def test_init_group_key_parallel_is_unique_per_replica():
runtime = _runtime(parallel_stage_init=True)
runtime._init_visible_devices_baseline = None
# Both on the SAME device, but parallel mode must give distinct keys so they
# run concurrently, coordinated by the child SH/EX locks + admission.
keys = [
runtime._replica_init_group_key(_llm_replica(0, 0, "0")),
runtime._replica_init_group_key(_llm_replica(0, 1, "0")),
runtime._replica_init_group_key(_llm_replica(1, 0, "0")),
]
assert keys == ["parallel:0:0", "parallel:0:1", "parallel:1:0"]
assert len(set(keys)) == 3
def test_init_group_key_remote_and_diffusion_unchanged():
runtime = _runtime(parallel_stage_init=True)
remote = _llm_replica(1, 0, "0")
remote.launch_mode = "remote"
remote.metadata.runtime_cfg = None
assert runtime._replica_init_group_key(remote) == "remote:1:0"
diffusion = _llm_replica(2, 0, "0")
diffusion.metadata.stage_type = "diffusion"
assert runtime._replica_init_group_key(diffusion) == "inline:diffusion"
# --------------------------------------------------------------------------- #
# B1: admission
# --------------------------------------------------------------------------- #
def test_evaluate_fits():
cap = 40 * 1024**3
demands = [StageDemand(0, 0, [0], utilization=0.5, graph_reserve_bytes=1 * 1024**3)]
ledgers = evaluate(demands, {0: cap}, external_reserve_bytes=0, safety_margin_bytes=0)
assert ledgers[0].fits
assert ledgers[0].kv_budget_bytes == int(cap * 0.5)
def test_evaluate_oversubscription_raises():
cap = 40 * 1024**3
demands = [
StageDemand(0, 0, [0], utilization=0.7, graph_reserve_bytes=0),
StageDemand(1, 0, [0], utilization=0.7, graph_reserve_bytes=0),
]
with pytest.raises(StageAdmissionError):
evaluate(demands, {0: cap}, external_reserve_bytes=0, safety_margin_bytes=0)
def test_evaluate_unknown_capacity_raises():
with pytest.raises(StageAdmissionError):
evaluate([StageDemand(0, 0, [3], 0.5, 0)], {0: 1})
def test_graph_reserve_positive_when_enabled_zero_when_disabled():
# cudagraph enabled -> conservative positive reserve
reserve = graph_reserve_bytes(_fake_vllm_config(0.9, cudagraph_mode="FULL"))
assert reserve > 0
# cudagraph disabled -> exactly 0
assert graph_reserve_bytes(_fake_vllm_config(0.9, cudagraph_mode="NONE")) == 0
eager = _fake_vllm_config(0.9)
eager.model_config.enforce_eager = True
assert graph_reserve_bytes(eager) == 0
def test_check_admission_multi_device_and_grouping():
cap = 80 * 1024**3
plan_a = LogicalStageInitPlan(
stage_idx=0,
stage_id=0,
replicas=[_llm_replica(0, 0, "0,1", vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))],
)
plan_b = LogicalStageInitPlan(
stage_idx=1,
stage_id=1,
replicas=[_llm_replica(1, 0, "1", vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))],
)
ledgers = check_admission(
[plan_a, plan_b],
resolve_physical_devices=lambda r: [int(x) for x in r.metadata.runtime_cfg["devices"].split(",")],
device_total_memory=lambda _d: cap,
external_reserve_bytes=0,
safety_margin_bytes=0,
)
# device 0: only stage 0 (0.4). device 1: stage 0 + stage 1 (0.8).
assert ledgers[0].kv_budget_bytes == int(cap * 0.4)
assert ledgers[1].kv_budget_bytes == int(cap * 0.4) + int(cap * 0.4)
assert ledgers[1].fits
def _local_diffusion_replica(stage_id: int, replica_id: int, devices: str, *, util: float | None) -> ReplicaInitPlan:
replica = _llm_replica(stage_id, replica_id, devices)
replica.metadata.stage_type = "diffusion"
replica.stage_vllm_config = None
replica.stage_cfg.engine_args = {} if util is None else {"gpu_memory_utilization": util}
return replica
def test_check_admission_admits_diffusion_with_explicit_util():
cap = 80 * 1024**3
diff = _local_diffusion_replica(0, 0, "0", util=0.3)
plan = LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[diff])
ledgers = check_admission(
[plan],
resolve_physical_devices=lambda _r: [0],
device_total_memory=lambda _d: cap,
external_reserve_bytes=0,
safety_margin_bytes=0,
)
assert set(ledgers) == {0}
assert ledgers[0].kv_budget_bytes == int(cap * 0.3)
def test_check_admission_unresolved_local_raises():
"""A local replica invisible to the ledger must fail admission, not skip it:
it would otherwise initialize concurrently with unbounded demand."""
unresolved = _llm_replica(1, 0, None, vllm_config=_fake_vllm_config(0.9, cudagraph_mode="NONE"))
plan = LogicalStageInitPlan(stage_idx=0, stage_id=1, replicas=[unresolved])
with pytest.raises(StageAdmissionError, match="stage1/replica0.*unresolved"):
check_admission(
[plan],
resolve_physical_devices=lambda _r: None,
device_total_memory=lambda _d: 80 * 1024**3,
)
def test_check_admission_local_diffusion_without_util_raises():
"""The reviewer's repro: local LLM + local diffusion with no explicit util
used to leave the diffusion stage out of the ledger silently."""
llm = _llm_replica(0, 0, "0", vllm_config=_fake_vllm_config(0.45, cudagraph_mode="NONE"))
diff = _local_diffusion_replica(1, 0, "0", util=None)
plans = [
LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[llm]),
LogicalStageInitPlan(stage_idx=1, stage_id=1, replicas=[diff]),
]
with pytest.raises(StageAdmissionError, match="diffusion, no gpu_memory_utilization"):
check_admission(
plans,
resolve_physical_devices=lambda _r: [0],
device_total_memory=lambda _d: 80 * 1024**3,
)
def test_check_admission_exempt_replicas_skip_without_error():
"""Only ADMISSION_EXEMPT (remote/operator-isolated) may skip the ledger."""
cap = 80 * 1024**3
local = _llm_replica(0, 0, "0", vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))
remote = _llm_replica(1, 0, None, vllm_config=_fake_vllm_config(0.9, cudagraph_mode="NONE"))
remote.launch_mode = "remote"
plan = LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[local, remote])
ledgers = check_admission(
[plan],
resolve_physical_devices=lambda r: ADMISSION_EXEMPT if r.launch_mode == "remote" else [0],
device_total_memory=lambda _d: cap,
external_reserve_bytes=0,
safety_margin_bytes=0,
)
assert set(ledgers) == {0}
assert ledgers[0].contributors == ["stage0/replica0"]
assert ledgers[0].kv_budget_bytes == int(cap * 0.4)
def test_run_stage_admission_excludes_remote_replicas(monkeypatch):
"""Remote replicas consume a remote node's memory — never the local ledger.
Remote LLM plans already carry runtime_cfg=None, but remote diffusion
replicas keep their cfg + utilization, so without an explicit launch_mode
guard they would be counted against local devices. They must resolve to
ADMISSION_EXEMPT (not None): None now means an unaccountable LOCAL replica
and fails admission.
"""
import vllm_omni.engine.stage_admission as admission_mod
runtime = _runtime(parallel_stage_init=True)
runtime._init_visible_devices_baseline = None
local = _llm_replica(0, 0, "0", vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))
remote_diff = _llm_replica(1, 0, "0")
remote_diff.launch_mode = "remote"
remote_diff.metadata.stage_type = "diffusion"
remote_diff.stage_vllm_config = None
remote_diff.stage_cfg.engine_args = {"gpu_memory_utilization": 0.3}
captured = {}
def _fake_check_admission(stage_plans, *, resolve_physical_devices, device_total_memory, **kw):
captured["resolve"] = resolve_physical_devices
return {}
monkeypatch.setattr(admission_mod, "check_admission", _fake_check_admission)
runtime._run_stage_admission([LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[local, remote_diff])])
assert captured["resolve"](local) == [0]
assert captured["resolve"](remote_diff) is ADMISSION_EXEMPT
# --------------------------------------------------------------------------- #
# B1b: admission capacity is queried in the visible-ordinal namespace
# --------------------------------------------------------------------------- #
def _capture_admission(monkeypatch, runtime, replicas):
"""Run _run_stage_admission with check_admission stubbed; return its callables."""
import vllm_omni.engine.stage_admission as admission_mod
captured: dict = {}
def _fake_check_admission(stage_plans, *, resolve_physical_devices, device_total_memory, **kw):
captured["resolve"] = resolve_physical_devices
captured["capacity"] = device_total_memory
return {}
monkeypatch.setattr(admission_mod, "check_admission", _fake_check_admission)
runtime._run_stage_admission([LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=replicas)])
return captured
def _stub_platform(monkeypatch, seen: list[int], total: int = 80 * 1024**3):
import vllm_omni.engine.stage_runtime as runtime_mod
monkeypatch.setattr(
runtime_mod,
"current_omni_platform",
types.SimpleNamespace(
device_control_env_var="CUDA_VISIBLE_DEVICES",
get_device_total_memory=lambda d: (seen.append(d), total)[1],
),
)
@pytest.mark.parametrize(
("baseline", "devices", "physical", "ordinal"),
[
("3,4,5", "1", 4, 1), # restricted visibility
("1,0", "0", 1, 0), # reordered visibility
("5,GPU-deadbeef", "0", 5, 0), # mixed identifiers, numeric first
("GPU-deadbeef,5", "1", 5, 1), # mixed identifiers, numeric second
("0,1,2,3", "2", 2, 2), # identity
],
ids=["restricted", "reordered", "mixed-numeric-first", "mixed-numeric-second", "identity"],
)
def test_admission_capacity_uses_visible_ordinal(monkeypatch, baseline, devices, physical, ordinal):
"""The ledger stays keyed by physical id, but the capacity query is translated.
``get_device_total_memory``/``torch.cuda.get_device_properties`` read their
argument as an ordinal in the *current* process's visible-device namespace.
With CUDA_VISIBLE_DEVICES=3,4,5 the orchestrator only has ordinals 0..2, so
passing physical id 4 straight through would raise — or, when visibility is
reordered, silently measure a different GPU.
"""
runtime = _runtime(parallel_stage_init=True)
runtime._init_visible_devices_baseline = baseline
replica = _llm_replica(0, 0, devices, vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))
seen: list[int] = []
_stub_platform(monkeypatch, seen)
captured = _capture_admission(monkeypatch, runtime, [replica])
assert captured["resolve"](replica) == [physical], "ledger key must stay physical"
assert captured["capacity"](physical) == 80 * 1024**3
assert seen == [ordinal], f"expected the capacity query on ordinal {ordinal}, got {seen}"
def test_admission_capacity_rejects_device_outside_baseline(monkeypatch):
"""A physical id absent from the captured baseline cannot be measured here."""
runtime = _runtime(parallel_stage_init=True)
runtime._init_visible_devices_baseline = "3,4,5"
replica = _llm_replica(0, 0, "0", vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))
seen: list[int] = []
_stub_platform(monkeypatch, seen)
captured = _capture_admission(monkeypatch, runtime, [replica])
with pytest.raises(StageAdmissionError, match="not in the visibility baseline"):
captured["capacity"](7)
assert seen == [], "must not query the platform with an unmappable id"
def test_admission_capacity_identity_when_no_baseline(monkeypatch):
"""With no device-control env captured, ordinals and physical ids coincide."""
runtime = _runtime(parallel_stage_init=True)
runtime._init_visible_devices_baseline = None
replica = _llm_replica(0, 0, "2", vllm_config=_fake_vllm_config(0.4, cudagraph_mode="NONE"))
seen: list[int] = []
_stub_platform(monkeypatch, seen)
captured = _capture_admission(monkeypatch, runtime, [replica])
assert captured["capacity"](2) == 80 * 1024**3
assert seen == [2]
# --------------------------------------------------------------------------- #
# B1c: fail closed for executors the phase locks cannot guard
# --------------------------------------------------------------------------- #
class _RayBackedExecutor:
uses_ray = True
class _LocalExecutor:
uses_ray = False
def test_parallel_init_refuses_ray_backed_executor():
"""Phase locks are node-local flocks and the ledger only sees this host.
A Ray executor may place the workers that load weights, profile, allocate KV
and capture graphs on other nodes, which participate in neither mechanism —
and the parent skips its own device lock when parallel_stage_init is on.
"""
runtime = _runtime(parallel_stage_init=True)
replica = _llm_replica(0, 0, "0")
replica.executor_class = _RayBackedExecutor
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[replica])]
with pytest.raises(RuntimeError, match="cannot guard Ray-backed or multi-node executors"):
runtime._reject_unguardable_executors(plans)
@pytest.mark.parametrize(
("use_ray", "nnodes", "reason"),
[
(True, 1, "Ray-backed"),
(False, 2, r"multi-node \(nnodes=2\)"),
],
ids=["parallel-config-ray", "multi-node-mp"],
)
def test_parallel_init_refuses_unguardable_parallel_config(use_ray, nnodes, reason):
"""The config guard also covers non-Ray executors spread across nodes."""
runtime = _runtime(parallel_stage_init=True)
vllm_config = types.SimpleNamespace(
parallel_config=types.SimpleNamespace(use_ray=use_ray, nnodes=nnodes),
)
replica = _llm_replica(0, 0, "0", vllm_config=vllm_config)
replica.executor_class = _LocalExecutor
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[replica])]
with pytest.raises(RuntimeError, match=reason):
runtime._reject_unguardable_executors(plans)
def test_parallel_init_allows_non_ray_executor():
runtime = _runtime(parallel_stage_init=True)
replicas = [_llm_replica(0, 0, "0"), _llm_replica(1, 0, "1")]
replicas[0].executor_class = _LocalExecutor
replicas[1].executor_class = object # no uses_ray attribute at all
runtime._reject_unguardable_executors([LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=replicas)])
def test_parallel_init_ignores_remote_ray_replicas():
"""Remote replicas are launched by the runtime owning their node."""
runtime = _runtime(parallel_stage_init=True)
replica = _llm_replica(0, 0, "0")
replica.executor_class = _RayBackedExecutor
replica.launch_mode = "remote"
runtime._reject_unguardable_executors([LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[replica])])
# --------------------------------------------------------------------------- #
# B2: SH/EX device phase locks (real flock)
# --------------------------------------------------------------------------- #
def _lock(tmp: Path, **kw) -> DevicePhaseLock:
return DevicePhaseLock([0], lock_dir=str(tmp), timeout_s=kw.pop("timeout_s", 0.3), **kw)
def test_shared_locks_coexist(tmp_path):
a, b = _lock(tmp_path), _lock(tmp_path)
with a.shared():
with b.shared():
pass # no timeout
def test_exclusive_excludes_shared_fail_closed(tmp_path):
a, b = _lock(tmp_path, timeout_s=0.2), _lock(tmp_path, timeout_s=0.2)
with a.exclusive():
with pytest.raises(DeviceLockTimeoutError):
with b.shared():
pass
def test_exclusive_excludes_exclusive(tmp_path):
a, b = _lock(tmp_path, timeout_s=0.2), _lock(tmp_path, timeout_s=0.2)
with a.exclusive():
with pytest.raises(DeviceLockTimeoutError):
with b.exclusive():
pass
def test_empty_device_set_is_noop(tmp_path):
lock = DevicePhaseLock([], lock_dir=str(tmp_path))
with lock.shared():
pass
with lock.exclusive():
pass
def _dead_pid() -> int:
"""A PID that has certainly exited and been reaped."""
proc = subprocess.Popen(["/bin/true"])
proc.wait()
return proc.pid
def test_legacy_device_lock_never_unlinks_the_path():
"""The stale-PID unlink path is gone from the legacy acquirer.
``flock`` is released by the kernel when the holder exits (SIGKILL included),
so a dead holder never keeps the lock and unlink-based cleanup is never
needed. It was also unsafe: the cleanup only ran on ``BlockingIOError``, i.e.
only while the lock *was* held, so every case it fired in was a case where
unlinking broke a live holder.
"""
import vllm_omni.engine.stage_init_utils as siu
assert not hasattr(siu, "_cleanup_stale_lock_if_dead")
assert "unlink" not in inspect.getsource(siu.acquire_device_locks)
# Test cleanup runs in many autouse/runtime fixtures and must not unlink the
# shared production lock path either. Persistent unlocked flock files are
# harmless; deleting one can split live holders across different inodes.
helper_source = (Path(__file__).parents[1] / "helpers" / "clean.py").read_text()
assert "_cleanup_stale_device_locks" not in helper_source
assert "vllm_omni_device_*_init.lock" not in helper_source
def test_legacy_lock_cannot_break_a_live_shared_holder(tmp_path, monkeypatch):
"""A live LOCK_SH holder must keep excluding the legacy LOCK_EX acquirer
even when the PID recorded in the lock file belongs to a dead process.
Several LOCK_SH holders overwrite that PID, so the recorded value can name a
process that has exited while another holder still owns the inode. Unlinking
on that signal let the legacy acquirer create a fresh file under the same
name and take LOCK_EX on a *different* inode — conflicting with nobody.
"""
import vllm_omni.engine.stage_init_utils as siu
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0")
monkeypatch.setattr(
siu,
"device_init_lock_path",
lambda device_id, lock_dir="/tmp": device_init_lock_path(device_id, str(tmp_path)),
)
path = Path(device_init_lock_path(0, str(tmp_path)))
holder = DevicePhaseLock([0], lock_dir=str(tmp_path), timeout_s=5)
with holder.shared():
path.write_text(f"{_dead_pid()}\n") # peer SH holder exited after recording its PID
inode_before = path.stat().st_ino
lock_fds = siu.acquire_device_locks(0, {"tensor_parallel_size": 1}, stage_init_timeout=1)
assert lock_fds == [], "legacy LOCK_EX must not be granted while a shared holder is live"
assert path.exists(), "the lock path must never be unlinked"
assert path.stat().st_ino == inode_before, "the lock inode must stay stable"
siu.release_device_locks(lock_fds)
def test_phase_lock_reuses_the_existing_lock_inode(tmp_path):
"""Repeated acquisitions coordinate on one stable inode per device."""
path = Path(device_init_lock_path(0, str(tmp_path)))
lock = DevicePhaseLock([0], lock_dir=str(tmp_path), timeout_s=1)
with lock.shared():
first = path.stat().st_ino
with lock.exclusive():
assert path.stat().st_ino == first
assert os.path.exists(path)
def test_device_lock_file_never_visible_at_restrictive_mode(tmp_path):
"""Concurrent creators must never observe the lock at a umask-filtered mode.
Creating in place with O_EXCL and widening afterwards publishes the file at
0600 under a restrictive umask; a second user opening it in that window is
locked out and parallel init aborts. Publishing atomically closes the window,
so every concurrent creator either wins the link or opens a readable file.
"""
path = device_init_lock_path(0, str(tmp_path))
results: list[tuple[int, bool]] = []
errors: list[BaseException] = []
modes: list[int] = []
barrier = threading.Barrier(8)
def _worker() -> None:
barrier.wait()
try:
fd, writable = open_device_lock_file(path)
except BaseException as exc: # noqa: BLE001 - surfaced by the assert below
errors.append(exc)
return
try:
modes.append(os.stat(path).st_mode & 0o777)
results.append((fd, writable))
finally:
os.close(fd)
previous = os.umask(0o077)
try:
threads = [threading.Thread(target=_worker) for _ in range(8)]
for t in threads:
t.start()
for t in threads:
t.join()
finally:
os.umask(previous)
assert not errors, f"concurrent creators were locked out: {errors!r}"
assert len(results) == 8
assert all(m & 0o044 == 0o044 for m in modes), f"lock was visible at {[oct(m) for m in modes]}"
assert len({os.stat(path).st_ino}) == 1
@pytest.mark.parametrize("umask_value", [0o022, 0o027, 0o077], ids=["022", "027", "077"])
def test_device_lock_file_is_readable_by_other_users(tmp_path, umask_value):
"""The lock file must stay group/other readable whatever the creator's umask.
These files coordinate GPU init *across* users, so a second user has to be
able to open the first user's file. The mode passed to ``os.open`` is filtered
by umask, so a restrictive umask (0027, 0077) would produce 0640 or 0600 and
lock every other user out -- fatal for parallel_stage_init, and a silent loss
of serialization in the legacy path.
"""
path = device_init_lock_path(0, str(tmp_path))
previous = os.umask(umask_value)
try:
fd, writable = open_device_lock_file(path)
finally:
os.umask(previous)
try:
assert writable is True
mode = os.stat(path).st_mode & 0o777
assert mode & 0o044 == 0o044, f"lock file mode {mode:04o} is not readable by other users"
finally:
os.close(fd)
@pytest.mark.skipif(os.geteuid() == 0, reason="root bypasses file permission bits")
def test_phase_lock_tolerates_lock_file_owned_by_another_user(tmp_path):
"""A lock file created by a different user must not break locking.
These files coordinate *across* users -- two people on the same physical GPU
must contend on the same path -- so whoever creates one owns it at 0644 and
every later user's O_RDWR open fails with EACCES. That made parallel_stage_init
die at startup on any shared machine. flock attaches to the open file
description and needs no write access, so the read-only fallback keeps full
mutual exclusion and only forfeits the diagnostic PID write.
"""
path = Path(device_init_lock_path(0, str(tmp_path)))
path.write_text("999999\n")
path.chmod(0o444)
fd, writable = open_device_lock_file(str(path))
try:
assert writable is False, "expected the read-only fallback on an unwritable file"
finally:
os.close(fd)
holder = DevicePhaseLock([0], lock_dir=str(tmp_path), timeout_s=0.3)
contender = DevicePhaseLock([0], lock_dir=str(tmp_path), timeout_s=0.2)
with holder.exclusive():
with pytest.raises(DeviceLockTimeoutError):
with contender.shared():
pass
assert path.read_text() == "999999\n", "must not clobber a file it cannot own"
def test_resolve_driver_device_ids_dp_slice(monkeypatch):
from vllm_omni.platforms import current_omni_platform
env_var = current_omni_platform.device_control_env_var
monkeypatch.setenv(env_var, "4,5,6,7")
pc = types.SimpleNamespace(
tensor_parallel_size=2,
pipeline_parallel_size=1,
prefill_context_parallel_size=1,
sequence_parallel_size=1,
cfg_parallel_size=1,
)
vc = types.SimpleNamespace(parallel_config=pc)
assert resolve_driver_device_ids(vc, local_dp_rank=0) == [4, 5]
assert resolve_driver_device_ids(vc, local_dp_rank=1) == [6, 7]
# --------------------------------------------------------------------------- #
# B2: executor wrapper ordering + retry
# --------------------------------------------------------------------------- #
class _RecordingLocker:
def __init__(self):
self.events: list[str] = []
@contextlib.contextmanager
def shared(self):
self.events.append("SH-enter")
try:
yield
finally:
self.events.append("SH-exit")
@contextlib.contextmanager
def exclusive(self):
self.events.append("EX-enter")
try:
yield
finally:
self.events.append("EX-exit")
class _FakeExecutor:
instances: list[object] = []
def __init__(self, vllm_config):
self.vllm_config = vllm_config
self.dam_calls = 0
_FakeExecutor.instances.append(self)
def determine_available_memory(self):
self.dam_calls += 1
return 123
def initialize_from_config(self, kv_cache_configs):
return "ok"
def compile_or_warm_up_model(self):
return "warm"
def test_wrapper_phase_ordering():
locker = _RecordingLocker()
wrapped_cls = wrap_executor_with_phase_locks(_FakeExecutor, locker)
exec_ = wrapped_cls(vllm_config=object())
assert exec_.determine_available_memory() == 123
assert exec_.initialize_from_config([]) == "ok"
assert exec_.compile_or_warm_up_model() == "warm"
assert locker.events == [
"SH-enter",
"SH-exit", # load
"EX-enter",
"EX-exit", # profile
"SH-enter",
"SH-exit", # KV cache allocation
"SH-enter",
"SH-exit", # kernel warmup + CUDA-graph capture
]
def test_wrapper_locks_warmup_and_capture_separately():
"""Regression guard for the upstream split.
``Executor.initialize_from_config`` used to fuse ``compile_or_warm_up_model``
into itself; since vLLM v0.20.0 ``EngineCore._initialize_kv_caches`` calls the
two separately. If the wrapper only brackets ``initialize_from_config``, one
stage captures CUDA graphs — mutating device memory — while a peer holds the
exclusive lock for its profiling measurement.
"""
locker = _RecordingLocker()
wrapped_cls = wrap_executor_with_phase_locks(_FakeExecutor, locker)
exec_ = wrapped_cls(vllm_config=object())
locker.events.clear()
assert exec_.compile_or_warm_up_model() == "warm"
assert locker.events == ["SH-enter", "SH-exit"], "compile_or_warm_up_model must run under the shared device lock"
assert "compile_or_warm_up_model" in vars(wrapped_cls), (
"the wrapper must override compile_or_warm_up_model, not inherit it unguarded"
)
@pytest.mark.parametrize(
"kwargs",
[{}, {"vllm_config": object()}, {"executor_class": object}],
ids=["both-missing", "executor-missing", "config-missing"],
)
def test_install_phase_locks_fails_closed_on_missing_kwargs(kwargs):
"""parallel_stage_init must never proceed with the phase-lock guard
uninstalled — a missing vllm_config/executor_class aborts the child."""
with pytest.raises(RuntimeError, match="phase-lock guard cannot be installed"):
_install_phase_locks(kwargs, local_dp_rank=0)
def test_install_phase_locks_wraps_executor(monkeypatch):
import vllm_omni.engine.stage_phase_lock as phase_lock_mod
locker = _RecordingLocker()
locker.device_ids = [0]
monkeypatch.setattr(
phase_lock_mod.DevicePhaseLock,
"from_child",
classmethod(lambda _cls, _cfg, _rank: locker),
)
kwargs = {"vllm_config": object(), "executor_class": _FakeExecutor}
_install_phase_locks(kwargs, local_dp_rank=0)
exec_ = kwargs["executor_class"](vllm_config=object())
assert exec_.determine_available_memory() == 123
assert locker.events[:2] == ["SH-enter", "SH-exit"]
# --------------------------------------------------------------------------- #
# A2: NVML process-scope removal regression guard
# --------------------------------------------------------------------------- #
def test_base_worker_dropped_nvml_process_scope():
import vllm_omni.worker.base as base
src = Path(base.__file__).read_text()
assert "get_process_gpu_memory" not in src
assert "is_process_scoped_memory_available" not in src
def test_device_ledger_required_and_fits():
led = DeviceLedger(device_id=0, capacity_bytes=100, kv_budget_bytes=40, graph_reserve_bytes=20)
led.external_reserve_bytes = 10
led.safety_margin_bytes = 10
assert led.required_bytes == 80
assert led.fits
led.kv_budget_bytes = 90
assert not led.fits