[Edge][CPU] Admit CPU stages against host memory so parallel init can run
`parallel_stage_init` was structurally impossible on a CPU deployment.
`resolve_stage_physical_devices` returns None for `devices: "cpu"`, and the
admission resolver fails closed on None -- correctly, for a GPU replica whose
devices cannot be accounted -- so every CPU pipeline raised
StageAdmissionError and fell back to sequential stage startup. The edge
profile sets parallel_stage_init: true, so that profile was broken on CPU
rather than merely slower.
Measured on Qwen3-TTS 0.6B (Xeon 8480C, 24 threads, init only):
sequential 64.57 s
parallel 30.54 s 2.1x
The fix is not to skip admission. The per-device ledger is vacuous on CPU --
there is no device to overcommit -- but the resource concurrent CPU init
contends for is host memory, and two stages allocating KV caches at once can
exhaust it exactly as two GPU stages exhaust a card. So there is now a host
ledger with the same discipline: sum the declared KV budgets, compare against
available RAM with a margin, refuse to start if it does not fit, and fail
closed on a stage that declares no budget at all. The arithmetic is pure
(`evaluate_host`) so it is testable without a device, matching the existing
design.
Which ledger applies is decided by where the stages are *placed*, not by what
platform the process reports: a first version keyed on
`current_platform.is_cpu()` and broke five existing GPU-path tests, because a
CPU-only test host still has to admit GPU-shaped plans correctly. It now
routes on the replicas' own `devices`.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bc7f8237e3
commit
a51461c131
3 files changed
+223
No files matched your search
@@ -839,3 +839,95 @@ def test_device_ledger_required_and_fits():
|
||||
assert led.fits
|
||||
led.kv_budget_bytes = 90
|
||||
assert not led.fits
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Host-memory admission: parallel init on a CPU platform
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _cpu_vllm_config(kv_bytes: int | None):
|
||||
return types.SimpleNamespace(
|
||||
cache_config=types.SimpleNamespace(kv_cache_memory_bytes=kv_bytes),
|
||||
compilation_config=types.SimpleNamespace(cudagraph_capture_sizes=[]),
|
||||
)
|
||||
|
||||
|
||||
def test_host_ledger_arithmetic():
|
||||
from vllm_omni.engine.stage_admission import evaluate_host
|
||||
|
||||
led = evaluate_host([("s0", 4 << 30), ("s1", 2 << 30)],
|
||||
available_bytes=16 << 30, safety_margin_bytes=1 << 30)
|
||||
assert led.kv_bytes == 6 << 30
|
||||
assert led.required_bytes == 7 << 30
|
||||
assert led.fits
|
||||
assert evaluate_host([("s0", 20 << 30)], available_bytes=16 << 30).fits is False
|
||||
|
||||
|
||||
def test_cpu_stages_are_admitted_against_host_memory():
|
||||
"""The per-device ledger is vacuous on CPU; host RAM is the real resource."""
|
||||
from vllm_omni.engine.stage_admission import check_host_admission
|
||||
|
||||
plans = [
|
||||
LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
|
||||
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(1 << 30))]),
|
||||
LogicalStageInitPlan(stage_idx=1, stage_id=1, replicas=[
|
||||
_llm_replica(1, 0, "cpu", vllm_config=_cpu_vllm_config(2 << 30))]),
|
||||
]
|
||||
led = check_host_admission(plans, available_memory=lambda: 32 << 30)
|
||||
assert led.kv_bytes == 3 << 30
|
||||
assert led.contributors == ["stage0/replica0", "stage1/replica0"]
|
||||
|
||||
|
||||
def test_cpu_admission_fails_fast_when_host_memory_is_short():
|
||||
from vllm_omni.engine.stage_admission import check_host_admission
|
||||
|
||||
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
|
||||
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(30 << 30))])]
|
||||
with pytest.raises(StageAdmissionError, match="host memory"):
|
||||
check_host_admission(plans, available_memory=lambda: 8 << 30)
|
||||
|
||||
|
||||
def test_cpu_admission_fails_closed_on_an_undeclared_budget():
|
||||
"""Same discipline as the device ledger: an unbounded stage cannot be
|
||||
admitted to initialize concurrently with others."""
|
||||
from vllm_omni.engine.stage_admission import check_host_admission
|
||||
|
||||
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
|
||||
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(None))])]
|
||||
with pytest.raises(StageAdmissionError, match="cannot account for CPU replicas"):
|
||||
check_host_admission(plans, available_memory=lambda: 64 << 30)
|
||||
|
||||
|
||||
def test_cpu_diffusion_stage_allocates_no_kv_and_is_skipped():
|
||||
from vllm_omni.engine.stage_admission import check_host_admission
|
||||
|
||||
replica = _llm_replica(0, 0, "cpu", vllm_config=None)
|
||||
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[replica])]
|
||||
led = check_host_admission(plans, available_memory=lambda: 16 << 30)
|
||||
assert led.kv_bytes == 0 and led.contributors == []
|
||||
|
||||
|
||||
def test_runtime_routes_cpu_platforms_to_the_host_ledger(monkeypatch):
|
||||
"""Before this, `devices: "cpu"` resolved to no physical ids and the device
|
||||
resolver failed closed, so parallel_stage_init could never run on CPU."""
|
||||
import vllm_omni.engine.stage_admission as admission_mod
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
def _fake_host(stage_plans, *, available_memory, **kw):
|
||||
seen["plans"] = stage_plans
|
||||
seen["available"] = available_memory()
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(admission_mod, "check_host_admission", _fake_host)
|
||||
|
||||
def _explode(*a, **k):
|
||||
raise AssertionError("device ledger must not run on a CPU platform")
|
||||
|
||||
monkeypatch.setattr(admission_mod, "check_admission", _explode)
|
||||
|
||||
runtime = _runtime(parallel_stage_init=True)
|
||||
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
|
||||
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(1 << 30))])]
|
||||
runtime._run_stage_admission(plans)
|
||||
assert seen["plans"] is plans
|
||||
assert seen["available"] > 0
|
||||
@@ -189,6 +189,102 @@ def evaluate(
|
||||
# ---- plan walking ----------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class HostLedger:
|
||||
"""What concurrent CPU stage init will ask of host memory."""
|
||||
|
||||
available_bytes: int
|
||||
kv_bytes: int
|
||||
safety_margin_bytes: int
|
||||
contributors: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def required_bytes(self) -> int:
|
||||
return self.kv_bytes + self.safety_margin_bytes
|
||||
|
||||
@property
|
||||
def fits(self) -> bool:
|
||||
return self.required_bytes <= self.available_bytes
|
||||
|
||||
|
||||
def evaluate_host(
|
||||
kv_demands: Sequence[tuple[str, int]],
|
||||
available_bytes: int,
|
||||
safety_margin_bytes: int = _DEFAULT_SAFETY_MARGIN_BYTES,
|
||||
) -> HostLedger:
|
||||
"""Pure arithmetic for the host-memory ledger, so it is unit-testable."""
|
||||
return HostLedger(
|
||||
available_bytes=available_bytes,
|
||||
kv_bytes=sum(kv for _, kv in kv_demands),
|
||||
safety_margin_bytes=safety_margin_bytes,
|
||||
contributors=[label for label, _ in kv_demands],
|
||||
)
|
||||
|
||||
|
||||
def check_host_admission(
|
||||
stage_plans: Sequence[Any],
|
||||
*,
|
||||
available_memory: Callable[[], int],
|
||||
safety_margin_bytes: int = _DEFAULT_SAFETY_MARGIN_BYTES,
|
||||
) -> HostLedger:
|
||||
"""Admit concurrent CPU stage init against host RAM.
|
||||
|
||||
On a CPU platform the per-device ledger is vacuous -- there is no device to
|
||||
overcommit -- but the resource parallel init actually contends for is host
|
||||
memory, and two stages allocating their KV caches at once can exhaust it
|
||||
just as two GPU stages can exhaust a card. So the same fail-fast discipline
|
||||
applies to a different resource.
|
||||
|
||||
Weights are not counted: they are already resident by the time a stage
|
||||
allocates its cache, so they are inside ``available_memory()``. What this
|
||||
bounds is the *new* allocation the stages are about to make concurrently.
|
||||
"""
|
||||
kv_demands: list[tuple[str, int]] = []
|
||||
unaccounted: list[str] = []
|
||||
for plan in stage_plans:
|
||||
for replica in getattr(plan, "replicas", []):
|
||||
metadata = replica.metadata
|
||||
label = f"stage{metadata.stage_id}/replica{replica.replica_id}"
|
||||
vllm_config = replica.stage_vllm_config
|
||||
if vllm_config is None:
|
||||
# A CPU diffusion stage allocates no KV cache.
|
||||
continue
|
||||
kv = getattr(vllm_config.cache_config, "kv_cache_memory_bytes", None)
|
||||
if not kv:
|
||||
unaccounted.append(label)
|
||||
continue
|
||||
kv_demands.append((label, int(kv)))
|
||||
|
||||
if unaccounted:
|
||||
raise StageAdmissionError(
|
||||
"parallel_stage_init admission cannot account for CPU replicas: "
|
||||
f"{unaccounted}. Each must declare a KV budget -- set "
|
||||
"kv_cache_memory_bytes in the stage's engine_args, or "
|
||||
"VLLM_CPU_KVCACHE_SPACE -- otherwise it would allocate "
|
||||
"concurrently without bounding host memory."
|
||||
)
|
||||
|
||||
ledger = evaluate_host(kv_demands, available_memory(), safety_margin_bytes)
|
||||
logger.info(
|
||||
"[admission] host memory: available=%s required=%s (kv=%s margin=%s) "
|
||||
"contributors=%s",
|
||||
format_gib(ledger.available_bytes),
|
||||
format_gib(ledger.required_bytes),
|
||||
format_gib(ledger.kv_bytes),
|
||||
format_gib(ledger.safety_margin_bytes),
|
||||
ledger.contributors,
|
||||
)
|
||||
if not ledger.fits:
|
||||
raise StageAdmissionError(
|
||||
"parallel_stage_init admission failed — concurrent CPU stage init "
|
||||
f"needs {format_gib(ledger.required_bytes)} GiB of host memory but "
|
||||
f"only {format_gib(ledger.available_bytes)} GiB is available "
|
||||
f"(contributors {ledger.contributors}). Lower the per-stage KV "
|
||||
"budgets or disable parallel_stage_init."
|
||||
)
|
||||
return ledger
|
||||
|
||||
|
||||
def check_admission(
|
||||
stage_plans: Sequence[Any],
|
||||
*,
|
||||
|
||||
@@ -578,8 +578,32 @@ class StageRuntime:
|
||||
AdmissionExempt,
|
||||
StageAdmissionError,
|
||||
check_admission,
|
||||
check_host_admission,
|
||||
)
|
||||
|
||||
# Which ledger applies is a property of where the stages are *placed*,
|
||||
# not of what platform this process reports: a CPU-only host still has
|
||||
# to admit GPU-shaped plans correctly.
|
||||
#
|
||||
# `devices: "cpu"` resolves to no physical ids, and the device resolver
|
||||
# fails closed on that, so parallel_stage_init was structurally
|
||||
# unusable for CPU deployments -- they silently fell back to sequential
|
||||
# init, which on Qwen3-TTS is ~28 s of a 66 s start. The resource
|
||||
# concurrent CPU init actually contends for is host memory, so admit
|
||||
# against that rather than skipping admission.
|
||||
local = [
|
||||
replica
|
||||
for plan in stage_plans
|
||||
for replica in getattr(plan, "replicas", [])
|
||||
if getattr(replica, "launch_mode", None) == "local"
|
||||
]
|
||||
if local and all(self._cpu_placed(r) for r in local):
|
||||
import psutil
|
||||
|
||||
self._run_host_admission(stage_plans, check_host_admission,
|
||||
lambda: int(psutil.virtual_memory().available))
|
||||
return
|
||||
|
||||
def _resolve(replica: ReplicaInitPlan) -> list[int] | AdmissionExempt | None:
|
||||
if replica.launch_mode == "remote":
|
||||
# Remote replicas consume a remote node's memory and the ledger
|
||||
@@ -657,6 +681,17 @@ class StageRuntime:
|
||||
device_total_memory=_total_memory,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cpu_placed(replica: Any) -> bool:
|
||||
"""True when this replica is explicitly placed on the CPU."""
|
||||
cfg = getattr(replica.metadata, "runtime_cfg", None) or {}
|
||||
devices = cfg.get("devices") if hasattr(cfg, "get") else getattr(cfg, "devices", None)
|
||||
return isinstance(devices, str) and devices.strip().lower() == "cpu"
|
||||
|
||||
def _run_host_admission(self, stage_plans, check_host_admission, available_memory) -> None:
|
||||
"""Bound concurrent CPU stage init against host RAM (see stage_admission)."""
|
||||
check_host_admission(stage_plans, available_memory=available_memory)
|
||||
|
||||
def _replica_init_group_key(self, replica: ReplicaInitPlan) -> str:
|
||||
"""Return the scheduling group used during replica initialization.
|
||||
|
||||
|
||||
Reference in new issue
Block a user