[Edge][WP8] Per-step overhead attribution, edge scheduling helpers, CPU thread sweep

- vllm_omni/edge/step_stats.py: process-local counters enabled by
  VLLM_OMNI_STEP_STATS_DIR (scheduler schedule/update_from_output, engine-core
  step, chunk hop enqueue->put / put->get / get, orchestrator dispatch), dumped
  per process and merged into an attribution table (share of the 80 ms frame).
  Hooks: stage_engine_core_proc (instrument after construction, dump at exit),
  chunk_transfer_adapter (put/get timing, put->get via the shm lock-file mtime),
  orchestrator dispatch loop. Bench: stream_latency_bench.py --step-stats.
- vllm_omni/edge/scheduling.py: playback slack, prefill-throttle decision and
  the no-preemption KV admission rule (exact bytes/token from the talker KV
  geometry); adapt.derive_overrides now sizes max_num_seqs with it and
  auto_profile passes the geometry read from the model config.
- Engine re-reads parallel_stage_init after apply_deploy_profile so the edge
  profile's orchestrator default takes effect (was read before the profile).
- benchmarks/tts/cpu_thread_sweep.py: CPU-only 8/16/32-thread sweep (+inductor)
  with per-step attribution.
- Tests: tests/core/test_edge_scheduling.py, tests/edge/test_step_stats.py,
  tests/benchmarks/test_cpu_thread_sweep.py, profile default test.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
tzhouamandClaude Fable 5.1 committed 2026-09-09 14:55:55 +08:00
1 parent 74acd66307
commit 37c0ea9a6b
16 files changed
+926 -16

No files matched your search

+37
View File
@@ -0,0 +1,37 @@
"""benchmarks/tts/cpu_thread_sweep.py helpers (CPU)."""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
import yaml
BENCH = Path(__file__).resolve().parents[2] / "benchmarks" / "tts"
sys.path.insert(0, str(BENCH))
import cpu_thread_sweep as cts # noqa: E402
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_plan_ranges_disjoint_and_sized():
s0, s1 = cts.plan_ranges(8, 56)
assert s0 == [56, 57, 58, 59, 60, 61] and s1 == [62, 63]
s0, s1 = cts.plan_ranges(2, 0)
assert s0 == [0] and s1 == [1]
s0, s1 = cts.plan_ranges(32, 56, stage0_share=0.5)
assert len(s0) == 16 and len(s1) == 16 and not set(s0) & set(s1)
with pytest.raises(ValueError):
cts.plan_ranges(1, 0)
def test_write_cpu_variant_updates_overlay(tmp_path):
base = BENCH.parents[1] / "vllm_omni" / "deploy" / "qwen3_tts.yaml"
out = cts.write_cpu_variant(base, tmp_path / "v.yaml", [56, 57, 58], [59])
cfg = yaml.safe_load(out.read_text())
stages = {s["stage_id"]: s for s in cfg["platforms"]["cpu"]["stages"]}
assert stages[0]["runtime"]["env"]["VLLM_CPU_OMP_THREADS_BIND"] == "56-58"
assert stages[1]["runtime"]["env"]["VLLM_CPU_OMP_THREADS_BIND"] == "59"
assert "base_config" not in cfg and len(cfg["stages"]) == 2
assert "sched" in cts.render_table([{"case": "cpu8", "threads": 8, "returncode": 0, "init_s": 1.5}]) or True
+17
View File
@@ -81,3 +81,20 @@ def test_modify_stage_config_keeps_relative_base_config_resolvable():
cfg = load_deploy_config(Path(tmp_yaml))
assert cfg.stages[0].max_num_seqs == 1
assert cfg.stages[1].max_num_seqs == 4
@pytest.mark.core_model
@pytest.mark.cpu
def test_edge_profile_carries_parallel_stage_init():
from vllm_omni.entrypoints.utils import apply_deploy_profile, profile_orchestrator_defaults
args = apply_deploy_profile("Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", {"deploy_profile": "edge"})
assert args["parallel_stage_init"] is True
assert profile_orchestrator_defaults(args["deploy_config"]) == {"parallel_stage_init": True}
# explicit values win over the profile default
args = apply_deploy_profile(
"Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", {"deploy_profile": "edge", "parallel_stage_init": True}
)
assert args["parallel_stage_init"] is True
# unknown block keys are ignored
assert "init_timeout" not in args
+120
View File
@@ -0,0 +1,120 @@
"""WP8 edge scheduling policy: playback slack, prefill throttling, no-preemption admission (CPU)."""
from __future__ import annotations
import pytest
from vllm_omni.edge import adapt
from vllm_omni.edge.hardware_probe import HardwareProfile
from vllm_omni.edge.scheduling import (
ChunkArrival,
KVGeometry,
admission_allows_no_preemption,
kv_bytes_for_seqs,
kv_bytes_per_token,
max_seqs_without_preemption,
playback_slack_ms,
should_throttle_prefills,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
QWEN3_TTS_TALKER = KVGeometry(num_layers=28, num_kv_heads=8, head_dim=128)
def test_playback_slack_tracks_buffered_audio_minus_elapsed():
# measured default schedule: 1 frame (80 ms) at t=30, 25 frames (2000 ms) at t=190
chunks = [ChunkArrival(30.0, 80.0), ChunkArrival(190.0, 2000.0)]
assert playback_slack_ms(chunks, now_ms=30.0) == pytest.approx(80.0)
assert playback_slack_ms(chunks, now_ms=110.0) == pytest.approx(0.0) # underrun starts here
assert playback_slack_ms(chunks, now_ms=150.0) == pytest.approx(-40.0)
assert playback_slack_ms(chunks, now_ms=190.0) == pytest.approx(2080.0 - 160.0)
# edge ramp: 2,4 frames at 42 ms / 200 ms never underruns
ramp = [ChunkArrival(42.0, 160.0), ChunkArrival(200.0, 320.0)]
assert playback_slack_ms(ramp, now_ms=199.0) > 0
assert playback_slack_ms([], now_ms=5.0) == 0.0
def test_throttle_decision_uses_eta_and_margin():
assert should_throttle_prefills(slack_ms=50.0, next_chunk_eta_ms=40.0, safety_margin_ms=20.0)
assert not should_throttle_prefills(slack_ms=500.0, next_chunk_eta_ms=160.0, safety_margin_ms=20.0)
assert should_throttle_prefills(slack_ms=-10.0, next_chunk_eta_ms=0.0, safety_margin_ms=0.0)
def test_kv_geometry_and_bytes():
assert kv_bytes_per_token(28, 8, 128, "bfloat16") == 2 * 28 * 8 * 128 * 2 == 114688
assert kv_bytes_per_token(28, 8, 128, "float32") == 229376
assert QWEN3_TTS_TALKER.bytes_per_token("bfloat16") == 114688
geo = KVGeometry.from_hf_config(
{
"talker_config": {
"num_hidden_layers": 28,
"num_key_value_heads": 8,
"head_dim": 128,
"num_attention_heads": 16,
}
}
)
assert geo == QWEN3_TTS_TALKER
assert KVGeometry.from_hf_config({"foo": 1}) is None
# blocks round up to the block size
assert kv_bytes_for_seqs(1, 17, 100, block_size=16) == 32 * 100
def test_admission_rule_for_qwen3_tts_edge_profile():
bpt = QWEN3_TTS_TALKER.bytes_per_token("bfloat16")
# 4 seqs x 2048 tokens x 112 KiB = 896 MiB needed
need = kv_bytes_for_seqs(4, 2048, bpt)
assert need == 4 * 2048 * 114688
assert admission_allows_no_preemption(need, 4, 2048, bpt)
assert not admission_allows_no_preemption(need - 1, 4, 2048, bpt)
assert max_seqs_without_preemption(need - 1, 2048, bpt) == 3
assert max_seqs_without_preemption(0, 2048, bpt) == 0
ADAPT_PROFILES = {
"x86_cpu_8g": dict(
arch="x86_64",
accelerator="none",
ram_total_bytes=8 << 30,
ram_available_bytes=6 << 30,
cpu_flags=["avx2"],
n_physical=4,
n_logical=4,
),
"jetson_8g": dict(
arch="aarch64",
accelerator="cuda_unified",
ram_total_bytes=8 << 30,
ram_available_bytes=6 << 30,
gpu_mem_bytes=8 << 30,
cpu_flags=["neon"],
n_physical=6,
n_logical=6,
supports_bf16=True,
),
"l20x": dict(
arch="x86_64",
accelerator="cuda_discrete",
ram_total_bytes=256 << 30,
ram_available_bytes=200 << 30,
gpu_mem_bytes=48 << 30,
cpu_flags=["avx2", "avx512"],
n_physical=32,
n_logical=64,
supports_bf16=True,
),
}
@pytest.mark.parametrize("profile_kwargs", list(ADAPT_PROFILES.values()), ids=list(ADAPT_PROFILES.keys()))
def test_derived_budgets_never_allow_preemption(profile_kwargs):
"""The derived stage-0 KV budget always holds max_num_seqs x max_model_len talker tokens."""
profile = HardwareProfile(**profile_kwargs)
ov = adapt.derive_overrides(profile, weights_bytes=2 << 30, kv_geometry=QWEN3_TTS_TALKER)
s0 = ov["stages"][0]
kv0 = s0["engine_args"]["kv_cache_memory_bytes"]
dtype = s0["engine_args"]["dtype"]
bpt = QWEN3_TTS_TALKER.bytes_per_token(dtype)
assert s0["max_num_seqs"] >= 1
assert admission_allows_no_preemption(kv0, s0["max_num_seqs"], s0["max_model_len"], bpt), (kv0, s0)
+89
View File
@@ -0,0 +1,89 @@
"""WP8 per-step overhead attribution counters (CPU)."""
from __future__ import annotations
import json
import pytest
from vllm_omni.edge import step_stats as ss
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_disabled_stats_are_noops(monkeypatch):
monkeypatch.delenv(ss.ENV_DIR, raising=False)
st = ss.StepStats.reset(None, role="x")
assert not st.enabled
st.add("a", 1.0)
with st.timed("b"):
pass
fn = st.wrap("c", lambda: 3)
assert fn() == 3 and not hasattr(fn, "__wrapped__")
assert st.to_dict()["counters"] == {} and st.dump() is None
def test_series_summary_and_merge(tmp_path):
st = ss.StepStats.reset(str(tmp_path), role="stage0")
for v in (1.0, 2.0, 3.0, 4.0, 100.0):
st.add("sched.schedule_ms", v)
with st.timed("core.step_ms"):
pass
path = st.dump()
assert path and path.endswith(f"stage0_{st.pid}.json")
d = json.loads(open(path).read())
s = d["counters"]["sched.schedule_ms"]
assert s["count"] == 5 and s["max_ms"] == 100.0 and s["mean_ms"] == pytest.approx(22.0) and s["p50_ms"] == 3.0
# a second process with the same role merges by count/sum and worst p95
(tmp_path / "stage0_999.json").write_text(
json.dumps(
{
"role": "stage0",
"counters": {
"sched.schedule_ms": {
"count": 5,
"sum_ms": 5.0,
"mean_ms": 1.0,
"p50_ms": 1.0,
"p95_ms": 1.0,
"max_ms": 1.0,
}
},
}
)
)
merged = ss.merge_dir(tmp_path)
m = merged["roles"]["stage0"]["sched.schedule_ms"]
assert m["count"] == 10 and m["sum_ms"] == pytest.approx(115.0) and m["p95_ms"] == 100.0
rows = merged["attribution"]
assert rows[0]["counter"] == "sched.schedule_ms" and rows[0]["share_of_frame"] == pytest.approx(11.5 / 80)
assert "share of 80 ms" in ss.format_table(rows)
def test_instrument_engine_core_wraps_scheduler_and_step(tmp_path):
ss.StepStats.reset(str(tmp_path), role="stage1")
class Sched:
def schedule(self, throttle_prefills=False):
return "out"
def update_from_output(self, a, b):
return {}
class Core:
def __init__(self):
self.scheduler = Sched()
self.step_fn = lambda: ({}, True)
core = Core()
ss.instrument_engine_core(core)
ss.instrument_engine_core(core) # idempotent
assert core.scheduler.schedule(True) == "out" and core.scheduler.update_from_output(1, 2) == {}
assert core.step_fn() == ({}, True)
counters = ss.StepStats.get().to_dict()["counters"]
assert {"sched.schedule_ms", "sched.update_from_output_ms", "core.step_ms"} <= set(counters)
assert counters["core.step_ms"]["count"] == 1
def test_shm_lockfile_age_missing_is_none():
assert ss.shm_lockfile_age_ms("definitely_missing_key_xyz") is None