diff --git a/tests/engine/test_parallel_stage_init.py b/tests/engine/test_parallel_stage_init.py index 5952d710e..691c9053a 100644 --- a/tests/engine/test_parallel_stage_init.py +++ b/tests/engine/test_parallel_stage_init.py @@ -839,3 +839,95 @@ def test_device_ledger_required_and_fits(): assert led.fits led.kv_budget_bytes = 90 assert not led.fits + + +# --------------------------------------------------------------------------- # +# Host-memory admission: parallel init on a CPU platform +# --------------------------------------------------------------------------- # +def _cpu_vllm_config(kv_bytes: int | None): + return types.SimpleNamespace( + cache_config=types.SimpleNamespace(kv_cache_memory_bytes=kv_bytes), + compilation_config=types.SimpleNamespace(cudagraph_capture_sizes=[]), + ) + + +def test_host_ledger_arithmetic(): + from vllm_omni.engine.stage_admission import evaluate_host + + led = evaluate_host([("s0", 4 << 30), ("s1", 2 << 30)], + available_bytes=16 << 30, safety_margin_bytes=1 << 30) + assert led.kv_bytes == 6 << 30 + assert led.required_bytes == 7 << 30 + assert led.fits + assert evaluate_host([("s0", 20 << 30)], available_bytes=16 << 30).fits is False + + +def test_cpu_stages_are_admitted_against_host_memory(): + """The per-device ledger is vacuous on CPU; host RAM is the real resource.""" + from vllm_omni.engine.stage_admission import check_host_admission + + plans = [ + LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[ + _llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(1 << 30))]), + LogicalStageInitPlan(stage_idx=1, stage_id=1, replicas=[ + _llm_replica(1, 0, "cpu", vllm_config=_cpu_vllm_config(2 << 30))]), + ] + led = check_host_admission(plans, available_memory=lambda: 32 << 30) + assert led.kv_bytes == 3 << 30 + assert led.contributors == ["stage0/replica0", "stage1/replica0"] + + +def test_cpu_admission_fails_fast_when_host_memory_is_short(): + from vllm_omni.engine.stage_admission import check_host_admission + + plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[ + _llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(30 << 30))])] + with pytest.raises(StageAdmissionError, match="host memory"): + check_host_admission(plans, available_memory=lambda: 8 << 30) + + +def test_cpu_admission_fails_closed_on_an_undeclared_budget(): + """Same discipline as the device ledger: an unbounded stage cannot be + admitted to initialize concurrently with others.""" + from vllm_omni.engine.stage_admission import check_host_admission + + plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[ + _llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(None))])] + with pytest.raises(StageAdmissionError, match="cannot account for CPU replicas"): + check_host_admission(plans, available_memory=lambda: 64 << 30) + + +def test_cpu_diffusion_stage_allocates_no_kv_and_is_skipped(): + from vllm_omni.engine.stage_admission import check_host_admission + + replica = _llm_replica(0, 0, "cpu", vllm_config=None) + plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[replica])] + led = check_host_admission(plans, available_memory=lambda: 16 << 30) + assert led.kv_bytes == 0 and led.contributors == [] + + +def test_runtime_routes_cpu_platforms_to_the_host_ledger(monkeypatch): + """Before this, `devices: "cpu"` resolved to no physical ids and the device + resolver failed closed, so parallel_stage_init could never run on CPU.""" + import vllm_omni.engine.stage_admission as admission_mod + + seen: dict = {} + + def _fake_host(stage_plans, *, available_memory, **kw): + seen["plans"] = stage_plans + seen["available"] = available_memory() + return None + + monkeypatch.setattr(admission_mod, "check_host_admission", _fake_host) + + def _explode(*a, **k): + raise AssertionError("device ledger must not run on a CPU platform") + + monkeypatch.setattr(admission_mod, "check_admission", _explode) + + runtime = _runtime(parallel_stage_init=True) + plans = [LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[ + _llm_replica(0, 0, "cpu", vllm_config=_cpu_vllm_config(1 << 30))])] + runtime._run_stage_admission(plans) + assert seen["plans"] is plans + assert seen["available"] > 0 diff --git a/vllm_omni/engine/stage_admission.py b/vllm_omni/engine/stage_admission.py index 469787bca..2a7d1372d 100644 --- a/vllm_omni/engine/stage_admission.py +++ b/vllm_omni/engine/stage_admission.py @@ -189,6 +189,102 @@ def evaluate( # ---- plan walking ---------------------------------------------------------- +@dataclass +class HostLedger: + """What concurrent CPU stage init will ask of host memory.""" + + available_bytes: int + kv_bytes: int + safety_margin_bytes: int + contributors: list[str] = field(default_factory=list) + + @property + def required_bytes(self) -> int: + return self.kv_bytes + self.safety_margin_bytes + + @property + def fits(self) -> bool: + return self.required_bytes <= self.available_bytes + + +def evaluate_host( + kv_demands: Sequence[tuple[str, int]], + available_bytes: int, + safety_margin_bytes: int = _DEFAULT_SAFETY_MARGIN_BYTES, +) -> HostLedger: + """Pure arithmetic for the host-memory ledger, so it is unit-testable.""" + return HostLedger( + available_bytes=available_bytes, + kv_bytes=sum(kv for _, kv in kv_demands), + safety_margin_bytes=safety_margin_bytes, + contributors=[label for label, _ in kv_demands], + ) + + +def check_host_admission( + stage_plans: Sequence[Any], + *, + available_memory: Callable[[], int], + safety_margin_bytes: int = _DEFAULT_SAFETY_MARGIN_BYTES, +) -> HostLedger: + """Admit concurrent CPU stage init against host RAM. + + On a CPU platform the per-device ledger is vacuous -- there is no device to + overcommit -- but the resource parallel init actually contends for is host + memory, and two stages allocating their KV caches at once can exhaust it + just as two GPU stages can exhaust a card. So the same fail-fast discipline + applies to a different resource. + + Weights are not counted: they are already resident by the time a stage + allocates its cache, so they are inside ``available_memory()``. What this + bounds is the *new* allocation the stages are about to make concurrently. + """ + kv_demands: list[tuple[str, int]] = [] + unaccounted: list[str] = [] + for plan in stage_plans: + for replica in getattr(plan, "replicas", []): + metadata = replica.metadata + label = f"stage{metadata.stage_id}/replica{replica.replica_id}" + vllm_config = replica.stage_vllm_config + if vllm_config is None: + # A CPU diffusion stage allocates no KV cache. + continue + kv = getattr(vllm_config.cache_config, "kv_cache_memory_bytes", None) + if not kv: + unaccounted.append(label) + continue + kv_demands.append((label, int(kv))) + + if unaccounted: + raise StageAdmissionError( + "parallel_stage_init admission cannot account for CPU replicas: " + f"{unaccounted}. Each must declare a KV budget -- set " + "kv_cache_memory_bytes in the stage's engine_args, or " + "VLLM_CPU_KVCACHE_SPACE -- otherwise it would allocate " + "concurrently without bounding host memory." + ) + + ledger = evaluate_host(kv_demands, available_memory(), safety_margin_bytes) + logger.info( + "[admission] host memory: available=%s required=%s (kv=%s margin=%s) " + "contributors=%s", + format_gib(ledger.available_bytes), + format_gib(ledger.required_bytes), + format_gib(ledger.kv_bytes), + format_gib(ledger.safety_margin_bytes), + ledger.contributors, + ) + if not ledger.fits: + raise StageAdmissionError( + "parallel_stage_init admission failed — concurrent CPU stage init " + f"needs {format_gib(ledger.required_bytes)} GiB of host memory but " + f"only {format_gib(ledger.available_bytes)} GiB is available " + f"(contributors {ledger.contributors}). Lower the per-stage KV " + "budgets or disable parallel_stage_init." + ) + return ledger + + def check_admission( stage_plans: Sequence[Any], *, diff --git a/vllm_omni/engine/stage_runtime.py b/vllm_omni/engine/stage_runtime.py index 33ed4e6ec..dc2997417 100644 --- a/vllm_omni/engine/stage_runtime.py +++ b/vllm_omni/engine/stage_runtime.py @@ -578,8 +578,32 @@ class StageRuntime: AdmissionExempt, StageAdmissionError, check_admission, + check_host_admission, ) + # Which ledger applies is a property of where the stages are *placed*, + # not of what platform this process reports: a CPU-only host still has + # to admit GPU-shaped plans correctly. + # + # `devices: "cpu"` resolves to no physical ids, and the device resolver + # fails closed on that, so parallel_stage_init was structurally + # unusable for CPU deployments -- they silently fell back to sequential + # init, which on Qwen3-TTS is ~28 s of a 66 s start. The resource + # concurrent CPU init actually contends for is host memory, so admit + # against that rather than skipping admission. + local = [ + replica + for plan in stage_plans + for replica in getattr(plan, "replicas", []) + if getattr(replica, "launch_mode", None) == "local" + ] + if local and all(self._cpu_placed(r) for r in local): + import psutil + + self._run_host_admission(stage_plans, check_host_admission, + lambda: int(psutil.virtual_memory().available)) + return + def _resolve(replica: ReplicaInitPlan) -> list[int] | AdmissionExempt | None: if replica.launch_mode == "remote": # Remote replicas consume a remote node's memory and the ledger @@ -657,6 +681,17 @@ class StageRuntime: device_total_memory=_total_memory, ) + @staticmethod + def _cpu_placed(replica: Any) -> bool: + """True when this replica is explicitly placed on the CPU.""" + cfg = getattr(replica.metadata, "runtime_cfg", None) or {} + devices = cfg.get("devices") if hasattr(cfg, "get") else getattr(cfg, "devices", None) + return isinstance(devices, str) and devices.strip().lower() == "cpu" + + def _run_host_admission(self, stage_plans, check_host_admission, available_memory) -> None: + """Bound concurrent CPU stage init against host RAM (see stage_admission).""" + check_host_admission(stage_plans, available_memory=available_memory) + def _replica_init_group_key(self, replica: ReplicaInitPlan) -> str: """Return the scheduling group used during replica initialization.