[Edge][WP8] Per-step overhead attribution, edge scheduling helpers, CPU thread sweep
- vllm_omni/edge/step_stats.py: process-local counters enabled by VLLM_OMNI_STEP_STATS_DIR (scheduler schedule/update_from_output, engine-core step, chunk hop enqueue->put / put->get / get, orchestrator dispatch), dumped per process and merged into an attribution table (share of the 80 ms frame). Hooks: stage_engine_core_proc (instrument after construction, dump at exit), chunk_transfer_adapter (put/get timing, put->get via the shm lock-file mtime), orchestrator dispatch loop. Bench: stream_latency_bench.py --step-stats. - vllm_omni/edge/scheduling.py: playback slack, prefill-throttle decision and the no-preemption KV admission rule (exact bytes/token from the talker KV geometry); adapt.derive_overrides now sizes max_num_seqs with it and auto_profile passes the geometry read from the model config. - Engine re-reads parallel_stage_init after apply_deploy_profile so the edge profile's orchestrator default takes effect (was read before the profile). - benchmarks/tts/cpu_thread_sweep.py: CPU-only 8/16/32-thread sweep (+inductor) with per-step attribution. - Tests: tests/core/test_edge_scheduling.py, tests/edge/test_step_stats.py, tests/benchmarks/test_cpu_thread_sweep.py, profile default test. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
74acd66307
commit
37c0ea9a6b
16 files changed
+926
-16
No files matched your search
@@ -0,0 +1,37 @@
|
||||
"""benchmarks/tts/cpu_thread_sweep.py helpers (CPU)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
BENCH = Path(__file__).resolve().parents[2] / "benchmarks" / "tts"
|
||||
sys.path.insert(0, str(BENCH))
|
||||
import cpu_thread_sweep as cts # noqa: E402
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_plan_ranges_disjoint_and_sized():
|
||||
s0, s1 = cts.plan_ranges(8, 56)
|
||||
assert s0 == [56, 57, 58, 59, 60, 61] and s1 == [62, 63]
|
||||
s0, s1 = cts.plan_ranges(2, 0)
|
||||
assert s0 == [0] and s1 == [1]
|
||||
s0, s1 = cts.plan_ranges(32, 56, stage0_share=0.5)
|
||||
assert len(s0) == 16 and len(s1) == 16 and not set(s0) & set(s1)
|
||||
with pytest.raises(ValueError):
|
||||
cts.plan_ranges(1, 0)
|
||||
|
||||
|
||||
def test_write_cpu_variant_updates_overlay(tmp_path):
|
||||
base = BENCH.parents[1] / "vllm_omni" / "deploy" / "qwen3_tts.yaml"
|
||||
out = cts.write_cpu_variant(base, tmp_path / "v.yaml", [56, 57, 58], [59])
|
||||
cfg = yaml.safe_load(out.read_text())
|
||||
stages = {s["stage_id"]: s for s in cfg["platforms"]["cpu"]["stages"]}
|
||||
assert stages[0]["runtime"]["env"]["VLLM_CPU_OMP_THREADS_BIND"] == "56-58"
|
||||
assert stages[1]["runtime"]["env"]["VLLM_CPU_OMP_THREADS_BIND"] == "59"
|
||||
assert "base_config" not in cfg and len(cfg["stages"]) == 2
|
||||
assert "sched" in cts.render_table([{"case": "cpu8", "threads": 8, "returncode": 0, "init_s": 1.5}]) or True
|
||||
@@ -81,3 +81,20 @@ def test_modify_stage_config_keeps_relative_base_config_resolvable():
|
||||
cfg = load_deploy_config(Path(tmp_yaml))
|
||||
assert cfg.stages[0].max_num_seqs == 1
|
||||
assert cfg.stages[1].max_num_seqs == 4
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_edge_profile_carries_parallel_stage_init():
|
||||
from vllm_omni.entrypoints.utils import apply_deploy_profile, profile_orchestrator_defaults
|
||||
|
||||
args = apply_deploy_profile("Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", {"deploy_profile": "edge"})
|
||||
assert args["parallel_stage_init"] is True
|
||||
assert profile_orchestrator_defaults(args["deploy_config"]) == {"parallel_stage_init": True}
|
||||
# explicit values win over the profile default
|
||||
args = apply_deploy_profile(
|
||||
"Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", {"deploy_profile": "edge", "parallel_stage_init": True}
|
||||
)
|
||||
assert args["parallel_stage_init"] is True
|
||||
# unknown block keys are ignored
|
||||
assert "init_timeout" not in args
|
||||
@@ -0,0 +1,120 @@
|
||||
"""WP8 edge scheduling policy: playback slack, prefill throttling, no-preemption admission (CPU)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge import adapt
|
||||
from vllm_omni.edge.hardware_probe import HardwareProfile
|
||||
from vllm_omni.edge.scheduling import (
|
||||
ChunkArrival,
|
||||
KVGeometry,
|
||||
admission_allows_no_preemption,
|
||||
kv_bytes_for_seqs,
|
||||
kv_bytes_per_token,
|
||||
max_seqs_without_preemption,
|
||||
playback_slack_ms,
|
||||
should_throttle_prefills,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
QWEN3_TTS_TALKER = KVGeometry(num_layers=28, num_kv_heads=8, head_dim=128)
|
||||
|
||||
|
||||
def test_playback_slack_tracks_buffered_audio_minus_elapsed():
|
||||
# measured default schedule: 1 frame (80 ms) at t=30, 25 frames (2000 ms) at t=190
|
||||
chunks = [ChunkArrival(30.0, 80.0), ChunkArrival(190.0, 2000.0)]
|
||||
assert playback_slack_ms(chunks, now_ms=30.0) == pytest.approx(80.0)
|
||||
assert playback_slack_ms(chunks, now_ms=110.0) == pytest.approx(0.0) # underrun starts here
|
||||
assert playback_slack_ms(chunks, now_ms=150.0) == pytest.approx(-40.0)
|
||||
assert playback_slack_ms(chunks, now_ms=190.0) == pytest.approx(2080.0 - 160.0)
|
||||
# edge ramp: 2,4 frames at 42 ms / 200 ms never underruns
|
||||
ramp = [ChunkArrival(42.0, 160.0), ChunkArrival(200.0, 320.0)]
|
||||
assert playback_slack_ms(ramp, now_ms=199.0) > 0
|
||||
assert playback_slack_ms([], now_ms=5.0) == 0.0
|
||||
|
||||
|
||||
def test_throttle_decision_uses_eta_and_margin():
|
||||
assert should_throttle_prefills(slack_ms=50.0, next_chunk_eta_ms=40.0, safety_margin_ms=20.0)
|
||||
assert not should_throttle_prefills(slack_ms=500.0, next_chunk_eta_ms=160.0, safety_margin_ms=20.0)
|
||||
assert should_throttle_prefills(slack_ms=-10.0, next_chunk_eta_ms=0.0, safety_margin_ms=0.0)
|
||||
|
||||
|
||||
def test_kv_geometry_and_bytes():
|
||||
assert kv_bytes_per_token(28, 8, 128, "bfloat16") == 2 * 28 * 8 * 128 * 2 == 114688
|
||||
assert kv_bytes_per_token(28, 8, 128, "float32") == 229376
|
||||
assert QWEN3_TTS_TALKER.bytes_per_token("bfloat16") == 114688
|
||||
geo = KVGeometry.from_hf_config(
|
||||
{
|
||||
"talker_config": {
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 8,
|
||||
"head_dim": 128,
|
||||
"num_attention_heads": 16,
|
||||
}
|
||||
}
|
||||
)
|
||||
assert geo == QWEN3_TTS_TALKER
|
||||
assert KVGeometry.from_hf_config({"foo": 1}) is None
|
||||
# blocks round up to the block size
|
||||
assert kv_bytes_for_seqs(1, 17, 100, block_size=16) == 32 * 100
|
||||
|
||||
|
||||
def test_admission_rule_for_qwen3_tts_edge_profile():
|
||||
bpt = QWEN3_TTS_TALKER.bytes_per_token("bfloat16")
|
||||
# 4 seqs x 2048 tokens x 112 KiB = 896 MiB needed
|
||||
need = kv_bytes_for_seqs(4, 2048, bpt)
|
||||
assert need == 4 * 2048 * 114688
|
||||
assert admission_allows_no_preemption(need, 4, 2048, bpt)
|
||||
assert not admission_allows_no_preemption(need - 1, 4, 2048, bpt)
|
||||
assert max_seqs_without_preemption(need - 1, 2048, bpt) == 3
|
||||
assert max_seqs_without_preemption(0, 2048, bpt) == 0
|
||||
|
||||
|
||||
ADAPT_PROFILES = {
|
||||
"x86_cpu_8g": dict(
|
||||
arch="x86_64",
|
||||
accelerator="none",
|
||||
ram_total_bytes=8 << 30,
|
||||
ram_available_bytes=6 << 30,
|
||||
cpu_flags=["avx2"],
|
||||
n_physical=4,
|
||||
n_logical=4,
|
||||
),
|
||||
"jetson_8g": dict(
|
||||
arch="aarch64",
|
||||
accelerator="cuda_unified",
|
||||
ram_total_bytes=8 << 30,
|
||||
ram_available_bytes=6 << 30,
|
||||
gpu_mem_bytes=8 << 30,
|
||||
cpu_flags=["neon"],
|
||||
n_physical=6,
|
||||
n_logical=6,
|
||||
supports_bf16=True,
|
||||
),
|
||||
"l20x": dict(
|
||||
arch="x86_64",
|
||||
accelerator="cuda_discrete",
|
||||
ram_total_bytes=256 << 30,
|
||||
ram_available_bytes=200 << 30,
|
||||
gpu_mem_bytes=48 << 30,
|
||||
cpu_flags=["avx2", "avx512"],
|
||||
n_physical=32,
|
||||
n_logical=64,
|
||||
supports_bf16=True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("profile_kwargs", list(ADAPT_PROFILES.values()), ids=list(ADAPT_PROFILES.keys()))
|
||||
def test_derived_budgets_never_allow_preemption(profile_kwargs):
|
||||
"""The derived stage-0 KV budget always holds max_num_seqs x max_model_len talker tokens."""
|
||||
profile = HardwareProfile(**profile_kwargs)
|
||||
ov = adapt.derive_overrides(profile, weights_bytes=2 << 30, kv_geometry=QWEN3_TTS_TALKER)
|
||||
s0 = ov["stages"][0]
|
||||
kv0 = s0["engine_args"]["kv_cache_memory_bytes"]
|
||||
dtype = s0["engine_args"]["dtype"]
|
||||
bpt = QWEN3_TTS_TALKER.bytes_per_token(dtype)
|
||||
assert s0["max_num_seqs"] >= 1
|
||||
assert admission_allows_no_preemption(kv0, s0["max_num_seqs"], s0["max_model_len"], bpt), (kv0, s0)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""WP8 per-step overhead attribution counters (CPU)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.edge import step_stats as ss
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_disabled_stats_are_noops(monkeypatch):
|
||||
monkeypatch.delenv(ss.ENV_DIR, raising=False)
|
||||
st = ss.StepStats.reset(None, role="x")
|
||||
assert not st.enabled
|
||||
st.add("a", 1.0)
|
||||
with st.timed("b"):
|
||||
pass
|
||||
fn = st.wrap("c", lambda: 3)
|
||||
assert fn() == 3 and not hasattr(fn, "__wrapped__")
|
||||
assert st.to_dict()["counters"] == {} and st.dump() is None
|
||||
|
||||
|
||||
def test_series_summary_and_merge(tmp_path):
|
||||
st = ss.StepStats.reset(str(tmp_path), role="stage0")
|
||||
for v in (1.0, 2.0, 3.0, 4.0, 100.0):
|
||||
st.add("sched.schedule_ms", v)
|
||||
with st.timed("core.step_ms"):
|
||||
pass
|
||||
path = st.dump()
|
||||
assert path and path.endswith(f"stage0_{st.pid}.json")
|
||||
d = json.loads(open(path).read())
|
||||
s = d["counters"]["sched.schedule_ms"]
|
||||
assert s["count"] == 5 and s["max_ms"] == 100.0 and s["mean_ms"] == pytest.approx(22.0) and s["p50_ms"] == 3.0
|
||||
# a second process with the same role merges by count/sum and worst p95
|
||||
(tmp_path / "stage0_999.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"role": "stage0",
|
||||
"counters": {
|
||||
"sched.schedule_ms": {
|
||||
"count": 5,
|
||||
"sum_ms": 5.0,
|
||||
"mean_ms": 1.0,
|
||||
"p50_ms": 1.0,
|
||||
"p95_ms": 1.0,
|
||||
"max_ms": 1.0,
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
merged = ss.merge_dir(tmp_path)
|
||||
m = merged["roles"]["stage0"]["sched.schedule_ms"]
|
||||
assert m["count"] == 10 and m["sum_ms"] == pytest.approx(115.0) and m["p95_ms"] == 100.0
|
||||
rows = merged["attribution"]
|
||||
assert rows[0]["counter"] == "sched.schedule_ms" and rows[0]["share_of_frame"] == pytest.approx(11.5 / 80)
|
||||
assert "share of 80 ms" in ss.format_table(rows)
|
||||
|
||||
|
||||
def test_instrument_engine_core_wraps_scheduler_and_step(tmp_path):
|
||||
ss.StepStats.reset(str(tmp_path), role="stage1")
|
||||
|
||||
class Sched:
|
||||
def schedule(self, throttle_prefills=False):
|
||||
return "out"
|
||||
|
||||
def update_from_output(self, a, b):
|
||||
return {}
|
||||
|
||||
class Core:
|
||||
def __init__(self):
|
||||
self.scheduler = Sched()
|
||||
self.step_fn = lambda: ({}, True)
|
||||
|
||||
core = Core()
|
||||
ss.instrument_engine_core(core)
|
||||
ss.instrument_engine_core(core) # idempotent
|
||||
assert core.scheduler.schedule(True) == "out" and core.scheduler.update_from_output(1, 2) == {}
|
||||
assert core.step_fn() == ({}, True)
|
||||
counters = ss.StepStats.get().to_dict()["counters"]
|
||||
assert {"sched.schedule_ms", "sched.update_from_output_ms", "core.step_ms"} <= set(counters)
|
||||
assert counters["core.step_ms"]["count"] == 1
|
||||
|
||||
|
||||
def test_shm_lockfile_age_missing_is_none():
|
||||
assert ss.shm_lockfile_age_ms("definitely_missing_key_xyz") is None
|
||||
Reference in new issue
Block a user