# SPDX-License-Identifier: Apache-2.0 """Measure the existing Omni local text path; do not infer model quality from speed.""" import argparse import asyncio import json import os import platform import sys import time import traceback from pathlib import Path def configure_host_environment(): """Reuse the native Windows settings established by the acceptance runs.""" if os.name == "nt": cudart = Path(sys.prefix) / "Lib/site-packages/torch/lib/cudart64_13.dll" if cudart.is_file(): os.environ.setdefault("VLLM_CUDART_SO_PATH", str(cudart)) os.environ.setdefault("VLLM_CACHE_ROOT", str(Path.home() / "c29")) return { key: os.environ[key] for key in ("VLLM_CUDART_SO_PATH", "VLLM_CACHE_ROOT", "CUDA_HOME", "PYTHONUTF8", "PYTHONIOENCODING") if key in os.environ } HOST_ENVIRONMENT = configure_host_environment() def save(path, value): temp = path.with_suffix(".tmp") temp.write_text(json.dumps(value, indent=2, default=str) + "\n", encoding="utf-8") try: temp.replace(path) except PermissionError as error: # Windows readers may deny replacement while holding the previous # snapshot open. Keep request JSONL authoritative and defer a running # snapshot rather than blocking concurrent inference on file I/O. if value.get("status") == "running" and path.is_file(): value["report_write_permission_errors"] = value.get("report_write_permission_errors", 0) + 1 value["last_report_write_error"] = repr(error) print(f"Deferred report snapshot after PermissionError: {path}", file=sys.stderr, flush=True) return # Initial/final snapshots must persist. This wait is outside measured # request execution; a persistent permission failure remains an error. for _ in range(20): time.sleep(0.05) try: temp.replace(path) return except PermissionError: pass raise async def run(args): from vllm_omni.edge.local.engine import LocalTextEngine, apply_runtime_env from vllm_omni.edge.local.plan import plan_text_session args.out.mkdir(parents=True, exist_ok=False) report = { "status": "running", "scope": "complete local text generation; quality gate separate", "argv": sys.argv, "platform": platform.platform(), "python": sys.executable, "start_unix": time.time(), "runtime_env": apply_runtime_env(), "host_environment": HOST_ENVIRONMENT, "settings": vars(args), "cache_condition": "existing disk/JIT cache; not cold disk", "trace": "untraced; API event timings", "requests": 0, } save(args.out / "report.json", report) engine = None try: plan = plan_text_session( args.model, max_model_len=4096, max_num_seqs=4, max_num_batched_tokens=512, enforce_eager=True, digest_weights=True, ) report["plan"] = plan.to_dict() if not plan.admitted: report["status"] = "rejected" return engine = LocalTextEngine(plan) start = time.perf_counter() await engine.start() report["startup_s"] = time.perf_counter() - start report["placement"] = engine.report_placement() save(args.out / "report.json", report) # Repeated natural text gives reproducible length bands; record actual token counts. prompts = { name: ("The garden has trees, flowers, and a small pond. " * n) + "\nDescribe the garden in detail." for name, n in [("short", 4), ("medium", 40), ("long", 160)] } samples = (args.out / "requests.jsonl").open("a", encoding="utf-8") counter = 0 async def request(name, concurrency, phase, slow=False): nonlocal counter counter += 1 session = engine.open_session() start = time.perf_counter() rid, stream = await engine.submit( session, prompts[name], max_tokens=128, temperature=0.0, ignore_eos=True, max_chunks=4 if slow else 64 ) arrivals = [] try: async for event in stream: arrivals.append( { "elapsed_s": time.perf_counter() - start, "kind": event.kind, "sequence": event.seq, "epoch": event.epoch, } ) if slow: await asyncio.sleep(0.03) rec = engine.records[rid].to_dict() rec.update( phase=phase, length_band=name, concurrency=concurrency, wall_s=time.perf_counter() - start, arrivals=arrivals, stream=stream.stats(), ) samples.write(json.dumps(rec) + "\n") samples.flush() if rec["error"] or rec["output_tokens"] != 128 or not rec["finished"]: raise RuntimeError(f"Incomplete request: {rid}: {rec['error']}") report["requests"] = counter save(args.out / "report.json", report) finally: engine.close_session(session.session_id) for concurrency in (1, 2, 4): for name in prompts: await asyncio.gather(*(request(name, concurrency, "warmup") for _ in range(concurrency))) for i in range(0, args.repeats, concurrency): await asyncio.gather( *(request(name, concurrency, "measured") for _ in range(min(concurrency, args.repeats - i))) ) print(f"profiled {name} concurrency={concurrency}", flush=True) thermal_start = time.perf_counter() report["sustained_start_unix"] = time.time() while time.perf_counter() - thermal_start < args.sustained_seconds: await request("medium", 1, "sustained") report["sustained_wall_s"] = time.perf_counter() - thermal_start await request("short", 1, "slow_consumer", slow=True) session = engine.open_session() rid, stream = await engine.submit(session, prompts["short"], max_tokens=512, ignore_eos=True) seen = 0 async for event in stream: if event.kind == "token": seen += 1 if seen >= 8: break start = time.perf_counter() report["cancel"] = await engine.cancel(rid) report["cancel"]["latency_s"] = time.perf_counter() - start report["cancel"]["remaining_events"] = [e.to_dict() async for e in stream] engine.close_session(session.session_id) await request("short", 1, "after_cancel") samples.close() report["usage"] = engine.report_usage() report["status"] = "completed" report["quality_gate"] = "not rerun against high precision reference; see separate acceptance evidence" except Exception as error: report.update(status="failed", error=repr(error), traceback=traceback.format_exc()) traceback.print_exc() finally: if engine is not None: await engine.close() report["usage_after_close"] = engine.report_usage() report["end_unix"] = time.time() save(args.out / "report.json", report) return report["status"] == "completed" if __name__ == "__main__": p = argparse.ArgumentParser(description=__doc__) p.add_argument("--model", required=True) p.add_argument("--out", type=Path, required=True) p.add_argument("--repeats", type=int, default=20) p.add_argument("--sustained-seconds", type=float, default=1800) args = p.parse_args() os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") from vllm_omni.windows.aio import install_selector_policy install_selector_policy() raise SystemExit(0 if asyncio.run(run(args)) else 1)