[Edge][CPU] Admit CPU stages against host memory so parallel init can run

`parallel_stage_init` was structurally impossible on a CPU deployment.
`resolve_stage_physical_devices` returns None for `devices: "cpu"`, and the
admission resolver fails closed on None -- correctly, for a GPU replica whose
devices cannot be accounted -- so every CPU pipeline raised
StageAdmissionError and fell back to sequential stage startup. The edge
profile sets parallel_stage_init: true, so that profile was broken on CPU
rather than merely slower.

Measured on Qwen3-TTS 0.6B (Xeon 8480C, 24 threads, init only):

    sequential   64.57 s
    parallel     30.54 s      2.1x

The fix is not to skip admission. The per-device ledger is vacuous on CPU --
there is no device to overcommit -- but the resource concurrent CPU init
contends for is host memory, and two stages allocating KV caches at once can
exhaust it exactly as two GPU stages exhaust a card. So there is now a host
ledger with the same discipline: sum the declared KV budgets, compare against
available RAM with a margin, refuse to start if it does not fit, and fail
closed on a stage that declares no budget at all. The arithmetic is pure
(`evaluate_host`) so it is testable without a device, matching the existing
design.

Which ledger applies is decided by where the stages are *placed*, not by what
platform the process reports: a first version keyed on
`current_platform.is_cpu()` and broke five existing GPU-path tests, because a
CPU-only test host still has to admit GPU-shaped plans correctly. It now
routes on the replicas' own `devices`.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
tzhouamandClaude Opus 5 committed 2026-09-12 21:16:11 +08:00
1 parent bc7f8237e3
commit a51461c131
3 files changed
+223

No files matched your search

+92
View File
@@ -839,3 +839,95 @@ def test_device_ledger_required_and_fits():
assert led.fits
led.kv_budget_bytes = 90
assert not led.fits
# --------------------------------------------------------------------------- #
# Host-memory admission: parallel init on a CPU platform
# --------------------------------------------------------------------------- #
def _cpu_vllm_config(kv_bytes: int | None):
return types.SimpleNamespace(
cache_config=types.SimpleNamespace(kv_cache_memory_bytes=kv_bytes),
compilation_config=types.SimpleNamespace(cudagraph_capture_sizes=[]),
)
def test_host_ledger_arithmetic():
from vllm_omni.engine.stage_admission import evaluate_host
led = evaluate_host([("s0", 4 << 30), ("s1", 2 << 30)],
available_bytes=16 << 30, safety_margin_bytes=1 << 30)
assert led.kv_bytes == 6 << 30
assert led.required_bytes == 7 << 30
assert led.fits
assert evaluate_host([("s0", 20 << 30)], available_bytes=16 << 30).fits is False
def test_cpu_stages_are_admitted_against_host_memory():
"""The per-device ledger is vacuous on CPU; host RAM is the real resource."""
from vllm_omni.engine.stage_admission import check_host_admission
plans = [
LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(1 << 30))]),
LogicalStageInitPlan(stage_idx=1, stage_id=1, replicas=[
_llm_replica(1, 0, "cpu", vllm_config=_cpu_vllm_config(2 << 30))]),
]
led = check_host_admission(plans, available_memory=lambda: 32 << 30)
assert led.kv_bytes == 3 << 30
assert led.contributors == ["stage0/replica0", "stage1/replica0"]
def test_cpu_admission_fails_fast_when_host_memory_is_short():
from vllm_omni.engine.stage_admission import check_host_admission
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(30 << 30))])]
with pytest.raises(StageAdmissionError, match="host memory"):
check_host_admission(plans, available_memory=lambda: 8 << 30)
def test_cpu_admission_fails_closed_on_an_undeclared_budget():
"""Same discipline as the device ledger: an unbounded stage cannot be
admitted to initialize concurrently with others."""
from vllm_omni.engine.stage_admission import check_host_admission
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(None))])]
with pytest.raises(StageAdmissionError, match="cannot account for CPU replicas"):
check_host_admission(plans, available_memory=lambda: 64 << 30)
def test_cpu_diffusion_stage_allocates_no_kv_and_is_skipped():
from vllm_omni.engine.stage_admission import check_host_admission
replica = _llm_replica(0, 0, "cpu", vllm_config=None)
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[replica])]
led = check_host_admission(plans, available_memory=lambda: 16 << 30)
assert led.kv_bytes == 0 and led.contributors == []
def test_runtime_routes_cpu_platforms_to_the_host_ledger(monkeypatch):
"""Before this, `devices: "cpu"` resolved to no physical ids and the device
resolver failed closed, so parallel_stage_init could never run on CPU."""
import vllm_omni.engine.stage_admission as admission_mod
seen: dict = {}
def _fake_host(stage_plans, *, available_memory, **kw):
seen["plans"] = stage_plans
seen["available"] = available_memory()
return None
monkeypatch.setattr(admission_mod, "check_host_admission", _fake_host)
def _explode(*a, **k):
raise AssertionError("device ledger must not run on a CPU platform")
monkeypatch.setattr(admission_mod, "check_admission", _explode)
runtime = _runtime(parallel_stage_init=True)
plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[
_llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(1 << 30))])]
runtime._run_stage_admission(plans)
assert seen["plans"] is plans
assert seen["available"] > 0
+96
View File
@@ -189,6 +189,102 @@ def evaluate(
# ---- plan walking ----------------------------------------------------------
@dataclass
class HostLedger:
"""What concurrent CPU stage init will ask of host memory."""
available_bytes: int
kv_bytes: int
safety_margin_bytes: int
contributors: list[str] = field(default_factory=list)
@property
def required_bytes(self) -> int:
return self.kv_bytes + self.safety_margin_bytes
@property
def fits(self) -> bool:
return self.required_bytes <= self.available_bytes
def evaluate_host(
kv_demands: Sequence[tuple[str, int]],
available_bytes: int,
safety_margin_bytes: int = _DEFAULT_SAFETY_MARGIN_BYTES,
) -> HostLedger:
"""Pure arithmetic for the host-memory ledger, so it is unit-testable."""
return HostLedger(
available_bytes=available_bytes,
kv_bytes=sum(kv for _, kv in kv_demands),
safety_margin_bytes=safety_margin_bytes,
contributors=[label for label, _ in kv_demands],
)
def check_host_admission(
stage_plans: Sequence[Any],
*,
available_memory: Callable[[], int],
safety_margin_bytes: int = _DEFAULT_SAFETY_MARGIN_BYTES,
) -> HostLedger:
"""Admit concurrent CPU stage init against host RAM.
On a CPU platform the per-device ledger is vacuous -- there is no device to
overcommit -- but the resource parallel init actually contends for is host
memory, and two stages allocating their KV caches at once can exhaust it
just as two GPU stages can exhaust a card. So the same fail-fast discipline
applies to a different resource.
Weights are not counted: they are already resident by the time a stage
allocates its cache, so they are inside ``available_memory()``. What this
bounds is the *new* allocation the stages are about to make concurrently.
"""
kv_demands: list[tuple[str, int]] = []
unaccounted: list[str] = []
for plan in stage_plans:
for replica in getattr(plan, "replicas", []):
metadata = replica.metadata
label = f"stage{metadata.stage_id}/replica{replica.replica_id}"
vllm_config = replica.stage_vllm_config
if vllm_config is None:
# A CPU diffusion stage allocates no KV cache.
continue
kv = getattr(vllm_config.cache_config, "kv_cache_memory_bytes", None)
if not kv:
unaccounted.append(label)
continue
kv_demands.append((label, int(kv)))
if unaccounted:
raise StageAdmissionError(
"parallel_stage_init admission cannot account for CPU replicas: "
f"{unaccounted}. Each must declare a KV budget -- set "
"kv_cache_memory_bytes in the stage's engine_args, or "
"VLLM_CPU_KVCACHE_SPACE -- otherwise it would allocate "
"concurrently without bounding host memory."
)
ledger = evaluate_host(kv_demands, available_memory(), safety_margin_bytes)
logger.info(
"[admission] host memory: available=%s required=%s (kv=%s margin=%s) "
"contributors=%s",
format_gib(ledger.available_bytes),
format_gib(ledger.required_bytes),
format_gib(ledger.kv_bytes),
format_gib(ledger.safety_margin_bytes),
ledger.contributors,
)
if not ledger.fits:
raise StageAdmissionError(
"parallel_stage_init admission failed — concurrent CPU stage init "
f"needs {format_gib(ledger.required_bytes)} GiB of host memory but "
f"only {format_gib(ledger.available_bytes)} GiB is available "
f"(contributors {ledger.contributors}). Lower the per-stage KV "
"budgets or disable parallel_stage_init."
)
return ledger
def check_admission(
stage_plans: Sequence[Any],
*,
+35
View File
@@ -578,8 +578,32 @@ class StageRuntime:
AdmissionExempt,
StageAdmissionError,
check_admission,
check_host_admission,
)
# Which ledger applies is a property of where the stages are *placed*,
# not of what platform this process reports: a CPU-only host still has
# to admit GPU-shaped plans correctly.
#
# `devices: "cpu"` resolves to no physical ids, and the device resolver
# fails closed on that, so parallel_stage_init was structurally
# unusable for CPU deployments -- they silently fell back to sequential
# init, which on Qwen3-TTS is ~28 s of a 66 s start. The resource
# concurrent CPU init actually contends for is host memory, so admit
# against that rather than skipping admission.
local = [
replica
for plan in stage_plans
for replica in getattr(plan, "replicas", [])
if getattr(replica, "launch_mode", None) == "local"
]
if local and all(self._cpu_placed(r) for r in local):
import psutil
self._run_host_admission(stage_plans, check_host_admission,
lambda: int(psutil.virtual_memory().available))
return
def _resolve(replica: ReplicaInitPlan) -> list[int] | AdmissionExempt | None:
if replica.launch_mode == "remote":
# Remote replicas consume a remote node's memory and the ledger
@@ -657,6 +681,17 @@ class StageRuntime:
device_total_memory=_total_memory,
)
@staticmethod
def _cpu_placed(replica: Any) -> bool:
"""True when this replica is explicitly placed on the CPU."""
cfg = getattr(replica.metadata, "runtime_cfg", None) or {}
devices = cfg.get("devices") if hasattr(cfg, "get") else getattr(cfg, "devices", None)
return isinstance(devices, str) and devices.strip().lower() == "cpu"
def _run_host_admission(self, stage_plans, check_host_admission, available_memory) -> None:
"""Bound concurrent CPU stage init against host RAM (see stage_admission)."""
check_host_admission(stage_plans, available_memory=available_memory)
def _replica_init_group_key(self, replica: ReplicaInitPlan) -> str:
"""Return the scheduling group used during replica initialization.