From 32a81111bd1eced9bb48f7043379c3d2ea8044f4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 24 Sep 2026 17:01:16 +0800 Subject: [PATCH] Qualify scoped Windows RTX Qwen3.8 FP8 text and image path --- .../experiments/audit_qwen38_fp8_copy.py | 69 ++ .../experiments/audit_qwen38_windows_fp8.py | 116 +++ .../experiments/probe_fp8_block_kernel.py | 39 + benchmarks/edge_harness/probe_qwen35_image.py | 43 +- .../qwen38_omni_windows_rtx/README.md | 12 + .../fp8_audit_report.json | 101 +++ .../fp8_copy_integrity.json | 556 +++++++++++++ .../fp8_triton_probe.driver.log | 2 + .../fp8_triton_probe.json | 14 + ...wen38_fp8_triton_ntfs_profile20.driver.log | 407 ++++++++++ .../qwen38_fp8_triton_ntfs_profile20.json | 269 +++++++ .../qwen38_image_fp8_attempt1.driver.log | 746 ++++++++++++++++++ .../qwen38_image_fp8_attempt1.json | 21 + ...wen38_image_fp8_triton_attempt1.driver.log | 30 + .../qwen38_image_fp8_triton_attempt1.json | 22 + ...wen38_image_fp8_triton_attempt2.driver.log | 612 ++++++++++++++ .../qwen38_image_fp8_triton_attempt2.json | 22 + ..._image_fp8_triton_ntfs_attempt1.driver.log | 260 ++++++ ...qwen38_image_fp8_triton_ntfs_attempt1.json | 23 + .../evidence/summary/README.md | 4 +- docs/design/omni_edge_e2e_roadmap.md | 2 + tests/config/test_omni_config.py | 4 +- vllm_omni/config/omni_config.py | 2 + 23 files changed, 3368 insertions(+), 8 deletions(-) create mode 100644 benchmarks/edge_harness/experiments/audit_qwen38_fp8_copy.py create mode 100644 benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py create mode 100644 benchmarks/edge_harness/experiments/probe_fp8_block_kernel.py create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_copy_integrity.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_triton_ntfs_profile20.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_triton_ntfs_profile20.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_attempt1.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_attempt1.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_ntfs_attempt1.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_ntfs_attempt1.json diff --git a/benchmarks/edge_harness/experiments/audit_qwen38_fp8_copy.py b/benchmarks/edge_harness/experiments/audit_qwen38_fp8_copy.py new file mode 100644 index 000000000..e80d6e5eb --- /dev/null +++ b/benchmarks/edge_harness/experiments/audit_qwen38_fp8_copy.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Record file-size and bundled CRC32 checks for a native Qwen3.8 FP8 copy.""" + +from __future__ import annotations + +import argparse +import json +import time +import zlib +from pathlib import Path + + +def crc32(path: Path) -> str: + value = 0 + with path.open("rb") as stream: + while chunk := stream.read(8 * 1024 * 1024): + value = zlib.crc32(chunk, value) + return f"{value:08x}" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--copy", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + started = time.monotonic() + original = {path.name: path.stat().st_size for path in args.source.iterdir() if path.is_file()} + copied = {path.name: path.stat().st_size for path in args.copy.iterdir() if path.is_file()} + if original != copied: + raise ValueError("source and copy differ in file names or sizes") + manifest = {} + for line in (args.source / "crc32.txt").read_text(encoding="utf-8").splitlines(): + expected, name = line.split(maxsplit=1) + manifest[name] = expected.lower() + rows = [] + for name, expected in manifest.items(): + actual = crc32(args.copy / name) + source_actual = crc32(args.source / name) if actual != expected else None + rows.append({"file": name, "bytes": copied[name], "manifest_crc32": expected, + "copy_crc32": actual, "source_crc32_if_manifest_differs": source_actual}) + weight_rows = [row for row in rows if row["file"].endswith(".safetensors")] + mismatches = [row for row in rows if row["manifest_crc32"] != row["copy_crc32"]] + if len(weight_rows) != 66 or any(row["manifest_crc32"] != row["copy_crc32"] for row in weight_rows): + raise ValueError("at least one safetensors file failed its bundled CRC32") + if any(row["source_crc32_if_manifest_differs"] != row["copy_crc32"] for row in mismatches): + raise ValueError("copy differs from source on a stale-manifest file") + result = { + "status": "checked", + "source": str(args.source), + "copy": str(args.copy), + "file_count": len(copied), + "total_bytes": sum(copied.values()), + "manifest_entries": len(rows), + "weight_files_matching_manifest": len(weight_rows), + "stale_manifest_nonweight_files": [row["file"] for row in mismatches], + "elapsed_s": time.monotonic() - started, + "files": rows, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps({key: result[key] for key in ( + "status", "file_count", "manifest_entries", "weight_files_matching_manifest", + "stale_manifest_nonweight_files", "elapsed_s" + )})) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py b/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py new file mode 100644 index 000000000..e37ca0cb4 --- /dev/null +++ b/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Audit the scoped native-Windows Qwen3.8-27B FP8 recovery and serial profile.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +from pathlib import Path + + +REVISION = "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a" + + +def digest(path: Path) -> dict[str, int | str]: + with path.open("rb") as stream: + return {"bytes": path.stat().st_size, "sha256": hashlib.file_digest(stream, "sha256").hexdigest()} + + +def percentile(values: list[float], fraction: float) -> float: + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-dir", type=Path, required=True) + args = parser.parse_args() + root = args.evidence_dir + names = [ + "fp8_copy_integrity.json", + "fp8_triton_probe.json", + "fp8_triton_probe.driver.log", + "qwen38_image_fp8_attempt1.json", + "qwen38_image_fp8_attempt1.driver.log", + "qwen38_image_fp8_triton_attempt1.json", + "qwen38_image_fp8_triton_attempt1.driver.log", + "qwen38_image_fp8_triton_attempt2.json", + "qwen38_image_fp8_triton_attempt2.driver.log", + "qwen38_image_fp8_triton_ntfs_attempt1.json", + "qwen38_image_fp8_triton_ntfs_attempt1.driver.log", + "qwen38_fp8_triton_ntfs_profile20.json", + "qwen38_fp8_triton_ntfs_profile20.driver.log", + ] + paths = {name: root / name for name in names} + reports = {name: json.loads(paths[name].read_text(encoding="utf-8")) for name in names if name.endswith(".json")} + logs = {name: paths[name].read_text(encoding="utf-8", errors="replace") for name in names if name.endswith(".log")} + probe = reports["fp8_triton_probe.json"] + copy = reports["fp8_copy_integrity.json"] + if (copy["status"] != "checked" or copy["file_count"] != 81 + or copy["weight_files_matching_manifest"] != 66 + or set(copy["stale_manifest_nonweight_files"]) != { + "chat_template.jinja", "generation_config.json", "tokenizer_config.json" + }): + raise ValueError("native checkpoint copy integrity record is incomplete") + if probe["status"] != "pass" or probe["capability"] != [12, 0] or probe["value"] != 128: + raise ValueError("FP8 Triton operation did not pass on SM120") + if reports["qwen38_image_fp8_attempt1.json"]["status"] != "failed" or "No compiled cutlass_scaled_mm" not in logs["qwen38_image_fp8_attempt1.driver.log"]: + raise ValueError("default CUTLASS failure is missing") + if reports["qwen38_image_fp8_triton_attempt1.json"]["status"] != "failed" or "no structured config owner: linear_backend" not in logs["qwen38_image_fp8_triton_attempt1.driver.log"]: + raise ValueError("Omni config ownership failure is missing") + if reports["qwen38_image_fp8_triton_attempt2.json"]["status"] != "failed" or "0.33 GiB KV cache is needed" not in logs["qwen38_image_fp8_triton_attempt2.driver.log"]: + raise ValueError("undersized KV-cache refusal is missing") + single = reports["qwen38_image_fp8_triton_ntfs_attempt1.json"] + profile = reports["qwen38_fp8_triton_ntfs_profile20.json"] + for report in (single, profile): + cfg = report["config"] + if (report["status"] != "completed" or report["outputs"] != ["Red"] + or cfg["linear_backend"] != "triton" or cfg["cpu_offload_gb"] != 12 + or cfg["gpu_memory_utilization"] != 0.8 or cfg["max_model_len"] != 512 + or "Qwen3.8-27B-FP8" not in report["model"]): + raise ValueError("successful report does not match the bounded FP8 image case") + for name in ("qwen38_image_fp8_triton_ntfs_attempt1.driver.log", "qwen38_fp8_triton_ntfs_profile20.driver.log"): + if not all(token in logs[name] for token in ( + "Selected TritonFp8BlockScaledMMKernel", + "Loading safetensors checkpoint shards: 100% Completed | 66/66", + )): + raise ValueError(f"successful log lacks kernel/load/cache evidence: {name}") + cache_match = re.search(r"GPU KV cache size: ([\d,]+) tokens", logs[name]) + if cache_match is None or int(cache_match.group(1).replace(",", "")) < 512: + raise ValueError(f"successful log lacks enough KV cache for one request: {name}") + if profile["text_output"] != ["ready"]: + raise ValueError("text warmup answer changed") + timing = {} + for kind, expected in (("image", "Red"), ("text", "ready")): + rows = [row for row in profile["profile"] if row["kind"] == kind] + if len(rows) != 20 or {row["index"] for row in rows} != set(range(20)) or any(row["output"] != expected for row in rows): + raise ValueError(f"{kind} serial profile is incomplete or changed output") + values = [row["wall_s"] for row in rows] + timing[kind] = {"n": len(values), "nearest_rank_p50_s": percentile(values, 0.5), + "nearest_rank_p95_s": percentile(values, 0.95), "min_s": min(values), "max_s": max(values)} + result = { + "status": "audited_scoped_e2e", + "evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX; no long context or broad quality claim", + "artifact": "Qwen/Qwen3.8-27B-FP8", + "checkpoint_revision": REVISION, + "host": "Windows 11 build 26200; Ryzen AI 9 HX 370; RTX 5090 Laptop; driver 610.71", + "runtime": "native vLLM 0.29.0+cu134; PyTorch 2.13.0+cu130; Triton FP8 block linear backend", + "configuration": profile["config"], + "warmups": {"image": 1, "text": 1}, + "timing": timing, + "files": {name: digest(path) for name, path in paths.items()}, + "limits": [ + "The FP8 checkpoint and Triton backend are distinct from the NVFP4 Marlin failure.", + "The image is a synthetic 96x96 red square and the text prompt is a single-word response; no broad multimodal or long-context quality claim follows.", + "The 12 GiB host offload and 0.80 GPU-memory cap are declared settings, not measured loading-peak admission or a sampled power profile.", + "Cancellation, public AsyncOmni, concurrency, video, sustained power/thermal behavior and task quality remain unverified for this native-Windows route.", + ], + } + (root / "fp8_audit_report.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": result["status"], "timing": timing})) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/experiments/probe_fp8_block_kernel.py b/benchmarks/edge_harness/experiments/probe_fp8_block_kernel.py new file mode 100644 index 000000000..539e11bfd --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_fp8_block_kernel.py @@ -0,0 +1,39 @@ +"""Check the installed Windows vLLM block-FP8 kernel before loading 27B weights.""" + +import json +import sys +import traceback +from pathlib import Path + +import torch + + +def main() -> None: + out = Path(sys.argv[1]) + result = { + "torch": torch.__version__, + "device": torch.cuda.get_device_name(0), + "capability": torch.cuda.get_device_capability(0), + } + try: + import vllm.model_executor.kernels.linear.scaled_mm.triton # noqa: F401 + + a = torch.ones((4, 128), device="cuda", dtype=torch.float8_e4m3fn) + b = torch.ones((128, 128), device="cuda", dtype=torch.float8_e4m3fn) + sa = torch.ones((4, 1), device="cuda", dtype=torch.float32) + sb = torch.ones((1, 1), device="cuda", dtype=torch.float32) + y = torch.ops.vllm.w8a8_triton_block_scaled_mm_func( + a, b, sa, sb, [128, 128], torch.bfloat16 + ) + torch.cuda.synchronize() + result.update(status="pass", shape=list(y.shape), value=float(y[0, 0])) + except Exception as exc: + result.update(status="failed", error=repr(exc), traceback=traceback.format_exc()) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + if result["status"] != "pass": + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/probe_qwen35_image.py b/benchmarks/edge_harness/probe_qwen35_image.py index 00a6f36e0..81e259047 100644 --- a/benchmarks/edge_harness/probe_qwen35_image.py +++ b/benchmarks/edge_harness/probe_qwen35_image.py @@ -12,6 +12,10 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", required=True) parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--cpu-offload-gb", type=float, default=2) + parser.add_argument("--gpu-memory-utilization", type=float, default=0.85) + parser.add_argument("--linear-backend", default="auto") + parser.add_argument("--profile-requests", type=int, default=0) args = parser.parse_args() os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") os.environ.setdefault("VLLM_WSL2_ENABLE_PIN_MEMORY", "1") @@ -42,20 +46,47 @@ def main(): "max_num_seqs": 1, "max_num_batched_tokens": 512, "enforce_eager": True, - "gpu_memory_utilization": 0.85, - "cpu_offload_gb": 2, + "gpu_memory_utilization": args.gpu_memory_utilization, + "cpu_offload_gb": args.cpu_offload_gb, "dtype": "bfloat16", "stage_init_timeout": 180, } + if args.linear_backend != "auto": + config["linear_backend"] = args.linear_backend record["config"] = config engine = Omni(**config) - outputs = engine.generate( - {"prompt": prompt, "multi_modal_data": {"image": image}}, - SamplingParams(temperature=0, max_tokens=16), - ) + image_input = {"prompt": prompt, "multi_modal_data": {"image": image}} + sampling = SamplingParams(temperature=0, max_tokens=16) + request_start = time.perf_counter() + outputs = engine.generate(image_input, sampling) + record["image_wall_s"] = time.perf_counter() - request_start record["outputs"] = [out.outputs[0].text for out in outputs if out.outputs] if not record["outputs"]: raise RuntimeError("No image-conditioned output") + if args.profile_requests: + text_prompt = processor.apply_chat_template( + [{"role": "user", "content": "Reply with the single word ready."}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + record["text_output"] = [ + out.outputs[0].text + for out in engine.generate({"prompt": text_prompt}, sampling) + if out.outputs + ] + record["profile"] = [] + for kind, request in (("image", image_input), ("text", {"prompt": text_prompt})): + for index in range(args.profile_requests): + started = time.perf_counter() + measured = engine.generate(request, sampling) + wall_s = time.perf_counter() - started + texts = [out.outputs[0].text for out in measured if out.outputs] + if not texts: + raise RuntimeError(f"No {kind} output on measured request {index}") + record["profile"].append( + {"kind": kind, "index": index, "wall_s": wall_s, "output": texts[0]} + ) record["status"] = "completed" except Exception as exc: record.update(status="failed", error=repr(exc), traceback=traceback.format_exc()) diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md index 6ea268222..7f12a3d65 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md @@ -1,5 +1,17 @@ # Qwen3.8-27B NVFP4 native Windows RTX startup attempt +## FP8/Triton recovery on the same Windows RTX + +The separate official Qwen/Qwen3.8-27B-FP8 checkpoint at revision `017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` now passes scoped complete text and image-to-text requests through this checkout's one-stage Omni pipeline on native Windows 11 build 26200 and RTX 5090 Laptop. It uses the installed vLLM 0.29.0+cu134 wheel, PyTorch 2.13.0+cu130, Triton 3.8.0, driver 610.71, explicit `linear_backend=triton`, BF16 compute, 12 GiB CPU parameter offload, 0.80 GPU-memory utilization, one sequence, eager execution and a 512-token context. The image is a synthetic 96×96 red square; the text prompt asks for the single word “ready.” This is a distinct FP8 artifact and backend, not a repair of the NVFP4 Marlin/PTX path below. + +The [single image pass](qwen38_image_fp8_triton_ntfs_attempt1.json) answered `Red`; the [serial profile](qwen38_fp8_triton_ntfs_profile20.json) answered `Red` on 20/20 measured image requests and `ready` on 20/20 measured text requests. There was one unmeasured request of each type. Nearest-rank complete-request wall p50/p95 was **4.184/4.380 s for image** and **3.070/3.214 s for text**, concurrency one. The profile's engine initialization was 87.37 s, including 31.35 s reported weight loading; neither startup nor warmups are included in the request percentiles. The worker log selected `TritonFp8BlockScaledMMKernel`, loaded all 66 safetensors files, reported 16.36 GiB GPU model-load allocation and initialized a 1,974-token GPU KV cache. The 16.36 GiB value is the runner's model-load allocation, not a total GPU or host loading peak; sustained power and thermal behavior were not measured. + +The checkpoint was copied from its WSL model root to native NTFS for the successful runs. Source and copy have the same 81 file names and sizes. A [full CRC32 record](fp8_copy_integrity.json) of the native copy against the bundled manifest matched all 66 safetensors and 8 other listed files. Three non-weight metadata files (`chat_template.jinja`, `generation_config.json`, `tokenizer_config.json`) differ from that manifest but have identical CRC32 values in source and copy; the manifest is stale for those files. The checkpoint revision comes from the Hugging Face download metadata, whose 81 recorded files all name the same commit. These checks do not substitute for a broad numerical or task-quality evaluation. + +The retained failures delimit the fix. The [default FP8/CUTLASS load](qwen38_image_fp8_attempt1.driver.log) reached a model startup operation but the Windows wheel had no compiled `cutlass_scaled_mm` for SM120. A [small Triton FP8 block probe](fp8_triton_probe.json) passed on SM120 with the expected value 128. The [first full Triton attempt](qwen38_image_fp8_triton_attempt1.json) exposed Omni's missing structured owner for `linear_backend`; the current config projection now carries that vLLM option, with a focused test. The [next attempt](qwen38_image_fp8_triton_attempt2.driver.log) loaded all 66 weights but vLLM refused the 0.75 GPU-memory cap: 0.22 GiB available for KV versus 0.33 GiB required for a 512-token request. Raising the cap to 0.80 admitted the successful run. The [FP8 auditor](fp8_audit_report.json) checks all raw reports, failure strings, successful kernel/load/cache evidence, per-request outputs, sample counts and percentile math. + +This advances the native Windows RTX × Qwen3.8 cell to a scoped complete text/image path. It does not establish a qualified long context, video path, reference-quality suite, explicit Omni host/VRAM loading-peak admission, cancellation/recovery, public AsyncOmni request, concurrency scaling or sustained power and thermal behavior. The NVFP4 failure remains a separate unresolved artifact/runtime combination. + The pinned Qwen3.8-27B NVFP4 checkpoint, revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c`, was attempted through the current Omni/vLLM 0.29 one-stage pipeline on native Windows 11 build 26200. The bounded image-to-text case used a synthetic 96×96 red image, 512-token context, one sequence, 16 output tokens, eager execution, 0.85 GPU-memory utilization and an explicit 2 GiB CPU offload. This is the same scoped case that [completed on WSL RTX](../../../e2e_recovery_20260923/README.md); Windows **did not reach inference**. No output or latency is claimed for this cell. The [first attempt report](qwen38_image_unc_attempt1.json) records an engine-start failure after model load but its console output was not retained. The [second attempt report](qwen38_image_unc_attempt2.json) and [full driver/worker log](qwen38_image_unc_attempt2.driver.log) retain the root cause. All **three safetensors shards completed loading**, then vLLM failed while preparing NVFP4 Marlin weights: `process_weights_after_loading` → `prepare_fp4_layer_for_marlin` → `marlin_permute_scales` raised `torch.AcceleratorError: CUDA error: the provided PTX was compiled with an unsupported toolchain`. The [auditor](audit_report.json) checks the two reports, captured stack, hardware versions and source-file hashes. The probe now exits nonzero after writing a failed report, so automation cannot mistake this startup error for a successful image request. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json new file mode 100644 index 000000000..f9c51f802 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json @@ -0,0 +1,101 @@ +{ + "status": "audited_scoped_e2e", + "evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX; no long context or broad quality claim", + "artifact": "Qwen/Qwen3.8-27B-FP8", + "checkpoint_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", + "host": "Windows 11 build 26200; Ryzen AI 9 HX 370; RTX 5090 Laptop; driver 610.71", + "runtime": "native vLLM 0.29.0+cu134; PyTorch 2.13.0+cu130; Triton FP8 block linear backend", + "configuration": { + "model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8", + "language_model_only": false, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": true, + "gpu_memory_utilization": 0.8, + "cpu_offload_gb": 12.0, + "dtype": "bfloat16", + "stage_init_timeout": 180, + "linear_backend": "triton" + }, + "warmups": { + "image": 1, + "text": 1 + }, + "timing": { + "image": { + "n": 20, + "nearest_rank_p50_s": 4.183892700006254, + "nearest_rank_p95_s": 4.3797133000334725, + "min_s": 4.016021100105718, + "max_s": 4.443590100039728 + }, + "text": { + "n": 20, + "nearest_rank_p50_s": 3.069557400071062, + "nearest_rank_p95_s": 3.213541300036013, + "min_s": 3.025600800057873, + "max_s": 3.2830592999234796 + } + }, + "files": { + "fp8_copy_integrity.json": { + "bytes": 15237, + "sha256": "9915945c4b87b2fd360c458ffe1662b0a3ce51d90fa1b4f18f92e62e4566db9d" + }, + "fp8_triton_probe.json": { + "bytes": 188, + "sha256": "1e3f44d0df5ae15d52a385644c057fda3755654f3a63e035d99b944b664b32b7" + }, + "fp8_triton_probe.driver.log": { + "bytes": 405, + "sha256": "0db7bcab7a0fe0a04a829e6f086f432b8a620eea96bbdab9bad466c79139ebc2" + }, + "qwen38_image_fp8_attempt1.json": { + "bytes": 6212, + "sha256": "e3886220a365b2a50831e7b109fb8b82c74121573dc9538c126fe1feb94ba202" + }, + "qwen38_image_fp8_attempt1.driver.log": { + "bytes": 68051, + "sha256": "0846a9b5e587b0d57dea2319d05d90030894f4175e72b83068b633131924cb63" + }, + "qwen38_image_fp8_triton_attempt1.json": { + "bytes": 3096, + "sha256": "e0ee3b8ceed607e1af5f2670aed3740729c96abd18e7e8210b8f7d30588e41bf" + }, + "qwen38_image_fp8_triton_attempt1.driver.log": { + "bytes": 2622, + "sha256": "43ec6bdad30e58a09f68475a9343e55ed8c910876acdab68bd4ee1780a32f28b" + }, + "qwen38_image_fp8_triton_attempt2.json": { + "bytes": 6247, + "sha256": "db8214d083f2880bd86e88df283cd7a345967da7ee988042ae28e5680da26e69" + }, + "qwen38_image_fp8_triton_attempt2.driver.log": { + "bytes": 53487, + "sha256": "8986870dec87fb5c0fee1970a861a3a4114be3e8235177700d10f19ca3c06ceb" + }, + "qwen38_image_fp8_triton_ntfs_attempt1.json": { + "bytes": 598, + "sha256": "29b730bd6701c9d58a4caf173ee32d7da625c0b8e1671a61ef7d0bc52038985b" + }, + "qwen38_image_fp8_triton_ntfs_attempt1.driver.log": { + "bytes": 30428, + "sha256": "f274a896cb21066b2b5323be74603ed17c26a6f29ef477892d66b2943aa39aee" + }, + "qwen38_fp8_triton_ntfs_profile20.json": { + "bytes": 5175, + "sha256": "c20c0b9c12b297361487522ac04977a2235c54d16b58ca276f0d09b66376a057" + }, + "qwen38_fp8_triton_ntfs_profile20.driver.log": { + "bytes": 38663, + "sha256": "1a324158efe8d6a233eca69cbd4f5f8828e1cf786a976fc9300cd1a2dc56368a" + } + }, + "limits": [ + "The FP8 checkpoint and Triton backend are distinct from the NVFP4 Marlin failure.", + "The image is a synthetic 96x96 red square and the text prompt is a single-word response; no broad multimodal or long-context quality claim follows.", + "The 12 GiB host offload and 0.80 GPU-memory cap are declared settings, not measured loading-peak admission or a sampled power profile.", + "Cancellation, public AsyncOmni, concurrency, video, sustained power/thermal behavior and task quality remain unverified for this native-Windows route." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_copy_integrity.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_copy_integrity.json new file mode 100644 index 000000000..dc6317ec5 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_copy_integrity.json @@ -0,0 +1,556 @@ +{ + "status": "checked", + "source": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-FP8", + "copy": "/mnt/c/Users/zhout/w2/Qwen3.8-27B-FP8", + "file_count": 81, + "total_bytes": 30890049597, + "manifest_entries": 77, + "weight_files_matching_manifest": 66, + "stale_manifest_nonweight_files": [ + "chat_template.jinja", + "generation_config.json", + "tokenizer_config.json" + ], + "elapsed_s": 165.86427295103204, + "files": [ + { + "file": "chat_template.jinja", + "bytes": 8952, + "manifest_crc32": "cf59f3ab", + "copy_crc32": "367cbad5", + "source_crc32_if_manifest_differs": "367cbad5" + }, + { + "file": "config.json", + "bytes": 51350, + "manifest_crc32": "f010e321", + "copy_crc32": "f010e321", + "source_crc32_if_manifest_differs": null + }, + { + "file": "generation_config.json", + "bytes": 202, + "manifest_crc32": "ac8d98f1", + "copy_crc32": "58cd7d0f", + "source_crc32_if_manifest_differs": "58cd7d0f" + }, + { + "file": "layers-0.safetensors", + "bytes": 383865448, + "manifest_crc32": "e3a49ca5", + "copy_crc32": "e3a49ca5", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-1.safetensors", + "bytes": 383865448, + "manifest_crc32": "1310181e", + "copy_crc32": "1310181e", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-10.safetensors", + "bytes": 383865472, + "manifest_crc32": "328dcf83", + "copy_crc32": "328dcf83", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-11.safetensors", + "bytes": 372313760, + "manifest_crc32": "db568b01", + "copy_crc32": "db568b01", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-12.safetensors", + "bytes": 383865472, + "manifest_crc32": "3314b0cc", + "copy_crc32": "3314b0cc", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-13.safetensors", + "bytes": 383865472, + "manifest_crc32": "cc30925a", + "copy_crc32": "cc30925a", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-14.safetensors", + "bytes": 383865472, + "manifest_crc32": "5ac81c1e", + "copy_crc32": "5ac81c1e", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-15.safetensors", + "bytes": 372313760, + "manifest_crc32": "124fbfce", + "copy_crc32": "124fbfce", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-16.safetensors", + "bytes": 383865472, + "manifest_crc32": "01165416", + "copy_crc32": "01165416", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-17.safetensors", + "bytes": 383865472, + "manifest_crc32": "4c312b70", + "copy_crc32": "4c312b70", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-18.safetensors", + "bytes": 383865472, + "manifest_crc32": "a9ce7605", + "copy_crc32": "a9ce7605", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-19.safetensors", + "bytes": 372313760, + "manifest_crc32": "603c75e2", + "copy_crc32": "603c75e2", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-2.safetensors", + "bytes": 383865448, + "manifest_crc32": "f3f4d0f0", + "copy_crc32": "f3f4d0f0", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-20.safetensors", + "bytes": 383865472, + "manifest_crc32": "351ba111", + "copy_crc32": "351ba111", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-21.safetensors", + "bytes": 383865472, + "manifest_crc32": "79171231", + "copy_crc32": "79171231", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-22.safetensors", + "bytes": 383865472, + "manifest_crc32": "3b5eb0af", + "copy_crc32": "3b5eb0af", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-23.safetensors", + "bytes": 372313760, + "manifest_crc32": "72de212c", + "copy_crc32": "72de212c", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-24.safetensors", + "bytes": 383865472, + "manifest_crc32": "94842288", + "copy_crc32": "94842288", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-25.safetensors", + "bytes": 383865472, + "manifest_crc32": "3730541d", + "copy_crc32": "3730541d", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-26.safetensors", + "bytes": 383865472, + "manifest_crc32": "f3b6eca2", + "copy_crc32": "f3b6eca2", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-27.safetensors", + "bytes": 372313760, + "manifest_crc32": "81996bcd", + "copy_crc32": "81996bcd", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-28.safetensors", + "bytes": 383865472, + "manifest_crc32": "4f7edbc0", + "copy_crc32": "4f7edbc0", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-29.safetensors", + "bytes": 383865472, + "manifest_crc32": "0a75e9a5", + "copy_crc32": "0a75e9a5", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-3.safetensors", + "bytes": 372313744, + "manifest_crc32": "5d630060", + "copy_crc32": "5d630060", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-30.safetensors", + "bytes": 383865472, + "manifest_crc32": "1f866a23", + "copy_crc32": "1f866a23", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-31.safetensors", + "bytes": 372313760, + "manifest_crc32": "46d75474", + "copy_crc32": "46d75474", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-32.safetensors", + "bytes": 383865472, + "manifest_crc32": "fa3149dd", + "copy_crc32": "fa3149dd", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-33.safetensors", + "bytes": 383865472, + "manifest_crc32": "15a44a27", + "copy_crc32": "15a44a27", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-34.safetensors", + "bytes": 383865472, + "manifest_crc32": "a3d889c1", + "copy_crc32": "a3d889c1", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-35.safetensors", + "bytes": 372313760, + "manifest_crc32": "931ef07b", + "copy_crc32": "931ef07b", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-36.safetensors", + "bytes": 383865472, + "manifest_crc32": "4de6867b", + "copy_crc32": "4de6867b", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-37.safetensors", + "bytes": 383865472, + "manifest_crc32": "d3506251", + "copy_crc32": "d3506251", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-38.safetensors", + "bytes": 383865472, + "manifest_crc32": "e7aa2993", + "copy_crc32": "e7aa2993", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-39.safetensors", + "bytes": 372313760, + "manifest_crc32": "4c135b24", + "copy_crc32": "4c135b24", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-4.safetensors", + "bytes": 383865448, + "manifest_crc32": "ec6847a1", + "copy_crc32": "ec6847a1", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-40.safetensors", + "bytes": 383865472, + "manifest_crc32": "63109fcb", + "copy_crc32": "63109fcb", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-41.safetensors", + "bytes": 383865472, + "manifest_crc32": "eeb26a22", + "copy_crc32": "eeb26a22", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-42.safetensors", + "bytes": 383865472, + "manifest_crc32": "18fcd233", + "copy_crc32": "18fcd233", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-43.safetensors", + "bytes": 372313760, + "manifest_crc32": "67d843b6", + "copy_crc32": "67d843b6", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-44.safetensors", + "bytes": 383865472, + "manifest_crc32": "5b2dcfe4", + "copy_crc32": "5b2dcfe4", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-45.safetensors", + "bytes": 383865472, + "manifest_crc32": "5965a8d1", + "copy_crc32": "5965a8d1", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-46.safetensors", + "bytes": 383865472, + "manifest_crc32": "7a72933f", + "copy_crc32": "7a72933f", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-47.safetensors", + "bytes": 372313760, + "manifest_crc32": "8f3ff58f", + "copy_crc32": "8f3ff58f", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-48.safetensors", + "bytes": 383865472, + "manifest_crc32": "07d9274e", + "copy_crc32": "07d9274e", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-49.safetensors", + "bytes": 383865472, + "manifest_crc32": "69de6f44", + "copy_crc32": "69de6f44", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-5.safetensors", + "bytes": 383865448, + "manifest_crc32": "b52e883f", + "copy_crc32": "b52e883f", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-50.safetensors", + "bytes": 383865472, + "manifest_crc32": "a0cfc77b", + "copy_crc32": "a0cfc77b", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-51.safetensors", + "bytes": 372313760, + "manifest_crc32": "d516cf08", + "copy_crc32": "d516cf08", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-52.safetensors", + "bytes": 383865472, + "manifest_crc32": "cb1e950d", + "copy_crc32": "cb1e950d", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-53.safetensors", + "bytes": 383865472, + "manifest_crc32": "655009f5", + "copy_crc32": "655009f5", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-54.safetensors", + "bytes": 383865472, + "manifest_crc32": "cb879428", + "copy_crc32": "cb879428", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-55.safetensors", + "bytes": 372313760, + "manifest_crc32": "47a1c95d", + "copy_crc32": "47a1c95d", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-56.safetensors", + "bytes": 383865472, + "manifest_crc32": "f949eaae", + "copy_crc32": "f949eaae", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-57.safetensors", + "bytes": 383865472, + "manifest_crc32": "e7404f68", + "copy_crc32": "e7404f68", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-58.safetensors", + "bytes": 383865472, + "manifest_crc32": "85f11421", + "copy_crc32": "85f11421", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-59.safetensors", + "bytes": 372313760, + "manifest_crc32": "d6fcd84b", + "copy_crc32": "d6fcd84b", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-6.safetensors", + "bytes": 383865448, + "manifest_crc32": "4703449c", + "copy_crc32": "4703449c", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-60.safetensors", + "bytes": 383865472, + "manifest_crc32": "ccead253", + "copy_crc32": "ccead253", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-61.safetensors", + "bytes": 383865472, + "manifest_crc32": "c59d4367", + "copy_crc32": "c59d4367", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-62.safetensors", + "bytes": 383865472, + "manifest_crc32": "5e872803", + "copy_crc32": "5e872803", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-63.safetensors", + "bytes": 372313760, + "manifest_crc32": "6f41ca01", + "copy_crc32": "6f41ca01", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-7.safetensors", + "bytes": 372313744, + "manifest_crc32": "75b6f226", + "copy_crc32": "75b6f226", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-8.safetensors", + "bytes": 383865448, + "manifest_crc32": "f8de3efc", + "copy_crc32": "f8de3efc", + "source_crc32_if_manifest_differs": null + }, + { + "file": "layers-9.safetensors", + "bytes": 383865448, + "manifest_crc32": "be62114d", + "copy_crc32": "be62114d", + "source_crc32_if_manifest_differs": null + }, + { + "file": "merges.txt", + "bytes": 3353259, + "manifest_crc32": "1824fcb5", + "copy_crc32": "1824fcb5", + "source_crc32_if_manifest_differs": null + }, + { + "file": "model.safetensors.index.json", + "bytes": 137335, + "manifest_crc32": "1109266c", + "copy_crc32": "1109266c", + "source_crc32_if_manifest_differs": null + }, + { + "file": "mtp.safetensors", + "bytes": 477202224, + "manifest_crc32": "837f3352", + "copy_crc32": "837f3352", + "source_crc32_if_manifest_differs": null + }, + { + "file": "outside.safetensors", + "bytes": 6007102112, + "manifest_crc32": "8dc22e46", + "copy_crc32": "8dc22e46", + "source_crc32_if_manifest_differs": null + }, + { + "file": "preprocessor_config.json", + "bytes": 390, + "manifest_crc32": "030e383a", + "copy_crc32": "030e383a", + "source_crc32_if_manifest_differs": null + }, + { + "file": "safetensors-md5sum.txt", + "bytes": 0, + "manifest_crc32": "00000000", + "copy_crc32": "00000000", + "source_crc32_if_manifest_differs": null + }, + { + "file": "tokenizer.json", + "bytes": 12809320, + "manifest_crc32": "6b61f895", + "copy_crc32": "6b61f895", + "source_crc32_if_manifest_differs": null + }, + { + "file": "tokenizer_config.json", + "bytes": 17928, + "manifest_crc32": "7adf2118", + "copy_crc32": "f3d608c1", + "source_crc32_if_manifest_differs": "f3d608c1" + }, + { + "file": "video_preprocessor_config.json", + "bytes": 385, + "manifest_crc32": "44c041dc", + "copy_crc32": "44c041dc", + "source_crc32_if_manifest_differs": null + }, + { + "file": "vocab.json", + "bytes": 6722759, + "manifest_crc32": "da8db314", + "copy_crc32": "da8db314", + "source_crc32_if_manifest_differs": null + } + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.driver.log new file mode 100644 index 000000000..a8c1bbb51 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.driver.log @@ -0,0 +1,2 @@ +WARNING 09-24 16:24:29 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=128,K=128,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +remark: fp8_utils.py:737:0: 772 instructions in function diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.json new file mode 100644 index 000000000..347f6e136 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_triton_probe.json @@ -0,0 +1,14 @@ +{ + "torch": "2.13.0+cu130", + "device": "NVIDIA GeForce RTX 5090 Laptop GPU", + "capability": [ + 12, + 0 + ], + "status": "pass", + "shape": [ + 4, + 128 + ], + "value": 128.0 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_triton_ntfs_profile20.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_triton_ntfs_profile20.driver.log new file mode 100644 index 000000000..84e1b4799 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_triton_ntfs_profile20.driver.log @@ -0,0 +1,407 @@ +INFO 09-24 16:42:33 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:42:33 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 16:42:37 [omni_base.py:216] [Omni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8 +INFO 09-24 16:42:37 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8 +INFO 09-24 16:42:37 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-24 16:42:38 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 16:42:38 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-24 16:42:38 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-24 16:42:38 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration +INFO 09-24 16:42:38 [model.py:2021] Using max model len 512 +WARNING 09-24 16:42:38 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-24 16:42:43 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-24 16:42:43 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled +WARNING 09-24 16:42:43 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0. +WARNING 09-24 16:42:43 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-24 16:42:43 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-24 16:42:43 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-24 16:42:43 [vllm.py:1585] Cudagraph is disabled under eager mode +INFO 09-24 16:42:43 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-24 16:42:43 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group +INFO 09-24 16:42:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:42:56 [patch.py:510] inductor factorable-divisibility patch: installed. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:00 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:01 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_d5a7dd0f3a6543fba9ec6e36eadfb2ed backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:01 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:01 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:02 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:02 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=63532) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:07 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:07 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:07 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:07 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:07 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:07 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:08 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:08 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:08 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:08 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:09 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:09 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:13 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:13 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 16.12 GiB. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:13 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (16.12 GiB). +(StageEngineCoreProc_stage0_replica0 pid=63532) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:40 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:40 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:40 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:41 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:41 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:41 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:41 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:42 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:51 [base.py:148] Available KV cache memory: 1.31 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:51 [kv_cache_utils.py:2032] GPU KV cache size: 1,974 tokens, Maximum concurrency for 512 tokens per request: 3.86x +(StageEngineCoreProc_stage0_replica0 pid=63532) WARNING 09-24 16:43:51 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=63532) INFO 09-24 16:43:52 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=63532) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:13 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_88968d4ecf3e4723a96e88303bc28481 backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=64968) WARNING 09-24 16:18:13 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:13 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=64968) WARNING 09-24 16:18:13 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:13 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=64968) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:19 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=64968) WARNING 09-24 16:18:19 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=64968) WARNING 09-24 16:18:19 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:19 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:19 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:19 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:20 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:20 [__init__.py:695] Selected CutlassFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:20 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:20 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:20 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:20 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:26 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:27 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 8.70 GiB. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:18:27 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (8.70 GiB). +(StageEngineCoreProc_stage0_replica0 pid=64968) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:22:15 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:22:15 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=64968) INFO 09-24 16:22:15 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start. + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] Traceback (most recent call last): + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_core_proc.py", line 189, in run_stage_core + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 1093, in __init__ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] super().__init__( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 145, in __init__ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] kv_cache_config = self._initialize_kv_caches(vllm_config) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 308, in _initialize_kv_caches + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] available_gpu_memory = self.model_executor.determine_available_memory() + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\abstract.py", line 149, in determine_available_memory + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self.collective_rpc("determine_available_memory") + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\uniproc_executor.py", line 103, in collective_rpc + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] result = run_method(self.driver_worker, method, args, kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\serial_utils.py", line 510, in run_method + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\utils\_contextlib.py", line 124, in decorate_context + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\base.py", line 124, in determine_available_memory + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] self.model_runner.profile_run() + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\worker\gpu_model_runner.py", line 6574, in profile_run + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] hidden_states, last_hidden_states = self._dummy_run( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\utils\_contextlib.py", line 124, in decorate_context + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\gpu_model_runner.py", line 1174, in _dummy_run + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] outputs = self.model( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self._call_impl(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return forward_call(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\models\qwen3_5.py", line 591, in forward + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] hidden_states = self.language_model.model( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\compilation\decorators.py", line 509, in __call__ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self.forward(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\models\qwen3_next.py", line 714, in forward + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] hidden_states, residual = layer( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self._call_impl(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return forward_call(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\models\qwen3_next.py", line 566, in forward + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] hidden_states = self.linear_attn(hidden_states=hidden_states) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self._call_impl(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return forward_call(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\mamba\gdn\qwen_gdn_linear_attn.py", line 832, in forward + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self._forward_method(hidden_states) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\mamba\gdn\qwen_gdn_linear_attn.py", line 903, in forward_cuda + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] mixed_qkvz, _ = self.in_proj_qkvz(hidden_states) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self._call_impl(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return forward_call(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\linear.py", line 600, in forward + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] output_parallel = self.quant_method.apply(self, input_, bias) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\fp8.py", line 459, in apply + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self.fp8_linear.apply_weights(layer, x, bias) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\kernels\linear\scaled_mm\BlockScaledMMLinearKernel.py", line 132, in apply_weights + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] output = self.apply_block_scaled_mm( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\kernels\linear\scaled_mm\cutlass.py", line 320, in apply_block_scaled_mm + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return ops.cutlass_scaled_mm( + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\_custom_ops.py", line 870, in cutlass_scaled_mm + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b, bias) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\_ops.py", line 1279, in __call__ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] return self._op(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=64968) ERROR 09-24 16:22:15 [stage_engine_core_proc.py:242] RuntimeError: cutlass_scaled_mm, C:\Users\zhout\w2\vllm-src-0.29.0\csrc\libtorch_stable\quantization\w8a8\cutlass\scaled_mm_entry.cu:269, NotImplementedError: No compiled cutlass_scaled_mm for a compute capability less than CUDA device capability: 120 +(StageEngineCoreProc_stage0_replica0 pid=64968) Process StageEngineCoreProc_stage0_replica0: +(StageEngineCoreProc_stage0_replica0 pid=64968) Traceback (most recent call last): +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\multiprocessing\process.py", line 314, in _bootstrap +(StageEngineCoreProc_stage0_replica0 pid=64968) self.run() +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\multiprocessing\process.py", line 108, in run +(StageEngineCoreProc_stage0_replica0 pid=64968) self._target(*self._args, **self._kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage0_replica0 pid=64968) engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage0_replica0 pid=64968) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 1093, in __init__ +(StageEngineCoreProc_stage0_replica0 pid=64968) super().__init__( +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 145, in __init__ +(StageEngineCoreProc_stage0_replica0 pid=64968) kv_cache_config = self._initialize_kv_caches(vllm_config) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage0_replica0 pid=64968) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 308, in _initialize_kv_caches +(StageEngineCoreProc_stage0_replica0 pid=64968) available_gpu_memory = self.model_executor.determine_available_memory() +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\abstract.py", line 149, in determine_available_memory +(StageEngineCoreProc_stage0_replica0 pid=64968) return self.collective_rpc("determine_available_memory") +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\uniproc_executor.py", line 103, in collective_rpc +(StageEngineCoreProc_stage0_replica0 pid=64968) result = run_method(self.driver_worker, method, args, kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\serial_utils.py", line 510, in run_method +(StageEngineCoreProc_stage0_replica0 pid=64968) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\utils\_contextlib.py", line 124, in decorate_context +(StageEngineCoreProc_stage0_replica0 pid=64968) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\base.py", line 124, in determine_available_memory +(StageEngineCoreProc_stage0_replica0 pid=64968) self.model_runner.profile_run() +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\worker\gpu_model_runner.py", line 6574, in profile_run +(StageEngineCoreProc_stage0_replica0 pid=64968) hidden_states, last_hidden_states = self._dummy_run( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\utils\_contextlib.py", line 124, in decorate_context +(StageEngineCoreProc_stage0_replica0 pid=64968) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\gpu_model_runner.py", line 1174, in _dummy_run +(StageEngineCoreProc_stage0_replica0 pid=64968) outputs = self.model( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return self._call_impl(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return forward_call(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\models\qwen3_5.py", line 591, in forward +(StageEngineCoreProc_stage0_replica0 pid=64968) hidden_states = self.language_model.model( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\compilation\decorators.py", line 509, in __call__ +(StageEngineCoreProc_stage0_replica0 pid=64968) return self.forward(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\models\qwen3_next.py", line 714, in forward +(StageEngineCoreProc_stage0_replica0 pid=64968) hidden_states, residual = layer( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return self._call_impl(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return forward_call(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\models\qwen3_next.py", line 566, in forward +(StageEngineCoreProc_stage0_replica0 pid=64968) hidden_states = self.linear_attn(hidden_states=hidden_states) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return self._call_impl(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return forward_call(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\mamba\gdn\qwen_gdn_linear_attn.py", line 832, in forward +(StageEngineCoreProc_stage0_replica0 pid=64968) return self._forward_method(hidden_states) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\mamba\gdn\qwen_gdn_linear_attn.py", line 903, in forward_cuda +(StageEngineCoreProc_stage0_replica0 pid=64968) mixed_qkvz, _ = self.in_proj_qkvz(hidden_states) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return self._call_impl(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl +(StageEngineCoreProc_stage0_replica0 pid=64968) return forward_call(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\linear.py", line 600, in forward +(StageEngineCoreProc_stage0_replica0 pid=64968) output_parallel = self.quant_method.apply(self, input_, bias) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\fp8.py", line 459, in apply +(StageEngineCoreProc_stage0_replica0 pid=64968) return self.fp8_linear.apply_weights(layer, x, bias) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\kernels\linear\scaled_mm\BlockScaledMMLinearKernel.py", line 132, in apply_weights +(StageEngineCoreProc_stage0_replica0 pid=64968) output = self.apply_block_scaled_mm( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\kernels\linear\scaled_mm\cutlass.py", line 320, in apply_block_scaled_mm +(StageEngineCoreProc_stage0_replica0 pid=64968) return ops.cutlass_scaled_mm( +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\_custom_ops.py", line 870, in cutlass_scaled_mm +(StageEngineCoreProc_stage0_replica0 pid=64968) torch.ops._C.cutlass_scaled_mm(out, a, b, scale_a, scale_b, bias) +(StageEngineCoreProc_stage0_replica0 pid=64968) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\torch\_ops.py", line 1279, in __call__ +(StageEngineCoreProc_stage0_replica0 pid=64968) return self._op(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=64968) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=64968) RuntimeError: cutlass_scaled_mm, C:\Users\zhout\w2\vllm-src-0.29.0\csrc\libtorch_stable\quantization\w8a8\cutlass\scaled_mm_entry.cu:269, NotImplementedError: No compiled cutlass_scaled_mm for a compute capability less than CUDA device capability: 120 +ERROR 09-24 16:22:21 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s) + +ERROR 09-24 16:22:21 [stage_runtime.py:265] Traceback (most recent call last): + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + +ERROR 09-24 16:22:21 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + +ERROR 09-24 16:22:21 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + +ERROR 09-24 16:22:21 [stage_runtime.py:265] raise primary_exc + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + +ERROR 09-24 16:22:21 [stage_runtime.py:265] client = self._initialize_replica( + +ERROR 09-24 16:22:21 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + +ERROR 09-24 16:22:21 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout) + +ERROR 09-24 16:22:21 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + +ERROR 09-24 16:22:21 [stage_runtime.py:265] launch_stack.close() + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + +ERROR 09-24 16:22:21 [stage_runtime.py:265] self.__exit__(None, None, None) + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] raise exc_details[1] + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] if cb(*exc_details): + +ERROR 09-24 16:22:21 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + +ERROR 09-24 16:22:21 [stage_runtime.py:265] next(self.gen) + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + +ERROR 09-24 16:22:21 [stage_runtime.py:265] wait_for_engine_startup( + +ERROR 09-24 16:22:21 [stage_runtime.py:265] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + +ERROR 09-24 16:22:21 [stage_runtime.py:265] raise RuntimeError( + +ERROR 09-24 16:22:21 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} +ERROR 09-24 16:22:21 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] Traceback (most recent call last): + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator()) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] return future.result() + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] self._runtime.initialize() + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] raise exc + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] raise primary_exc + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] client = self._initialize_replica( + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] launch_stack.close() + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] self.__exit__(None, None, None) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] raise exc_details[1] + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] if cb(*exc_details): + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] next(self.gen) + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] wait_for_engine_startup( + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] raise RuntimeError( + +ERROR 09-24 16:22:21 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} +INFO 09-24 16:22:21 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +Exception in thread orchestrator: +Traceback (most recent call last): + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1075, in _bootstrap_inner + self.run() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + raise exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + next(self.gen) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} +Traceback (most recent call last): + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1075, in _bootstrap_inner + self.run() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + raise exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + next(self.gen) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\probe_qwen35_image.py", line 53, in main + engine = Omni(**config) + ^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\entrypoints\omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 305, in __init__ + self._wait_for_orchestrator_init(startup_future, startup_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 539, in _wait_for_orchestrator_init + startup_future.result( + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\concurrent\futures\_base.py", line 456, in result + return self.__get_result() + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\concurrent\futures\_base.py", line 401, in __get_result + raise self._exception +RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_attempt1.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_attempt1.json new file mode 100644 index 000000000..4c000fa8d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_attempt1.json @@ -0,0 +1,21 @@ +{ + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8", + "scope": "one synthetic-image Omni image-to-text request", + "start": 1790237850.5871127, + "config": { + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8", + "language_model_only": false, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": true, + "gpu_memory_utilization": 0.75, + "cpu_offload_gb": 12.0, + "dtype": "bfloat16", + "stage_init_timeout": 180 + }, + "status": "failed", + "error": "RuntimeError(\"Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\")", + "traceback": "Traceback (most recent call last):\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1075, in _bootstrap_inner\n self.run()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1012, in run\n self._target(*self._args, **self._kwargs)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 468, in _bootstrap_orchestrator\n loop.run_until_complete(_run_orchestrator())\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\asyncio\\base_events.py\", line 691, in run_until_complete\n return future.result()\n ^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 424, in _run_orchestrator\n self._initialize_stages(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 350, in _initialize_stages\n self._initialize_stages_impl(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 371, in _initialize_stages_impl\n self._runtime.initialize()\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 278, in initialize\n raise exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 252, in initialize\n initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 589, in _initialize_stage_replicas\n raise primary_exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 550, in _init_group\n client = self._initialize_replica(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 828, in _initialize_replica\n return self._initialize_local_llm_replica(plan, stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 934, in _initialize_local_llm_replica\n launch_stack.close()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 618, in close\n self.__exit__(None, None, None)\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 610, in __exit__\n raise exc_details[1]\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 595, in __exit__\n if cb(*exc_details):\n ^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 144, in __exit__\n next(self.gen)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_engine_startup.py\", line 1211, in launch_stage_replica\n wait_for_engine_startup(\n File \"C:\\Users\\zhout\\w2\\omni029venv\\Lib\\site-packages\\vllm\\v1\\engine\\utils.py\", line 1307, in wait_for_engine_startup\n raise RuntimeError(\nRuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\probe_qwen35_image.py\", line 53, in main\n engine = Omni(**config)\n ^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\entrypoints\\omni_base.py\", line 223, in __init__\n self.engine = AsyncOmniEngine(\n ^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 305, in __init__\n self._wait_for_orchestrator_init(startup_future, startup_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 539, in _wait_for_orchestrator_init\n startup_future.result(\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 456, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 401, in __get_result\n raise self._exception\nRuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n", + "end": 1790238141.9250593 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.driver.log new file mode 100644 index 000000000..8ff6a40c7 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.driver.log @@ -0,0 +1,30 @@ +INFO 09-24 16:25:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:25:08 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 16:25:13 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8 +INFO 09-24 16:25:13 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8 +Traceback (most recent call last): + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\probe_qwen35_image.py", line 55, in main + engine = Omni(**config) + ^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\entrypoints\omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 228, in __init__ + self.config_path, self.stage_configs = self._resolve_stage_configs( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 1101, in _resolve_stage_configs + load_and_resolve_stage_configs( + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 106, in load_and_resolve_stage_configs + return resolve_omni_config( + ^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\config\resolver.py", line 238, in resolve_omni_config + structured_config = StageConfigFactory.create_from_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\config\config_factory.py", line 401, in create_from_model + return VllmOmniConfig.from_pipeline_config( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\config\omni_config.py", line 1952, in from_pipeline_config + _validate_global_stage_cli_ownership(pipeline_cfg, cli_overrides) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\config\omni_config.py", line 368, in _validate_global_stage_cli_ownership + raise ValueError( +ValueError: Pipeline 'qwen3_5' has explicit engine argument(s) with no structured config owner: linear_backend diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.json new file mode 100644 index 000000000..b1dd27c06 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt1.json @@ -0,0 +1,22 @@ +{ + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8", + "scope": "one synthetic-image Omni image-to-text request", + "start": 1790238290.0872524, + "config": { + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8", + "language_model_only": false, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": true, + "gpu_memory_utilization": 0.75, + "cpu_offload_gb": 12.0, + "linear_backend": "triton", + "dtype": "bfloat16", + "stage_init_timeout": 180 + }, + "status": "failed", + "error": "ValueError(\"Pipeline 'qwen3_5' has explicit engine argument(s) with no structured config owner: linear_backend\")", + "traceback": "Traceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\probe_qwen35_image.py\", line 55, in main\n engine = Omni(**config)\n ^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\entrypoints\\omni_base.py\", line 223, in __init__\n self.engine = AsyncOmniEngine(\n ^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 228, in __init__\n self.config_path, self.stage_configs = self._resolve_stage_configs(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 1101, in _resolve_stage_configs\n load_and_resolve_stage_configs(\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 106, in load_and_resolve_stage_configs\n return resolve_omni_config(\n ^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\config\\resolver.py\", line 238, in resolve_omni_config\n structured_config = StageConfigFactory.create_from_model(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\config\\config_factory.py\", line 401, in create_from_model\n return VllmOmniConfig.from_pipeline_config(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\config\\omni_config.py\", line 1952, in from_pipeline_config\n _validate_global_stage_cli_ownership(pipeline_cfg, cli_overrides)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\config\\omni_config.py\", line 368, in _validate_global_stage_cli_ownership\n raise ValueError(\nValueError: Pipeline 'qwen3_5' has explicit engine argument(s) with no structured config owner: linear_backend\n", + "end": 1790238314.2134175 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.driver.log new file mode 100644 index 000000000..a140cf056 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.driver.log @@ -0,0 +1,612 @@ +INFO 09-24 16:26:33 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:26:33 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 16:26:37 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8 +INFO 09-24 16:26:37 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8 +INFO 09-24 16:26:37 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-24 16:26:38 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 16:26:38 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-24 16:26:38 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-24 16:26:38 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration +INFO 09-24 16:26:38 [model.py:2021] Using max model len 512 +WARNING 09-24 16:26:38 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-24 16:26:44 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-24 16:26:44 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled +WARNING 09-24 16:26:44 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0. +WARNING 09-24 16:26:44 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-24 16:26:44 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-24 16:26:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-24 16:26:44 [vllm.py:1585] Cudagraph is disabled under eager mode +INFO 09-24 16:26:44 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-24 16:26:44 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group +INFO 09-24 16:26:57 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:26:57 [patch.py:510] inductor factorable-divisibility patch: installed. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:00 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:03 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_074ca444a0cc45079a7b9f12455c8f66 backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:27:03 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:03 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:27:03 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:03 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=52424) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:09 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:27:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:27:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:09 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:09 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:09 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:10 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:10 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:10 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:10 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:11 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:11 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:19 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:20 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 5.39 GiB. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:27:20 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (5.39 GiB). +(StageEngineCoreProc_stage0_replica0 pid=52424) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:34:40 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:34:40 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:34:40 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:34:40 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +remark: fp8_utils.py:737:0: 775 instructions in function +remark: fused_gdn_prefill_post_conv.py:20:0: 380 instructions in function +(StageEngineCoreProc_stage0_replica0 pid=52424) C:\Users\zhout\w2\omni029venv\Lib\site-packages\triton\language\core.py:2625: UserWarning: tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead. +(StageEngineCoreProc_stage0_replica0 pid=52424) warn("tl.make_block_ptr is deprecated. Use TensorDescriptor or tl.make_tensor_descriptor instead.") +remark: cumsum.py:27:0: 87 instructions in function +remark: cumsum.py:27:0: 71 instructions in function +remark: cumsum.py:27:0: 75 instructions in function +remark: cumsum.py:27:0: 75 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 975 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 993 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1001 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 677 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 691 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 697 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 469 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 482 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 486 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1256 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1274 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1293 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 845 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 859 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 870 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 598 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 610 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 617 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1623 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1623 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1623 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1097 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1097 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 1097 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 803 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 803 instructions in function +remark: chunk_scaled_dot_kkt.py:47:0: 803 instructions in function +remark: solve_tril.py:238:0: 1942 instructions in function +remark: solve_tril.py:238:0: 1942 instructions in function +remark: solve_tril.py:238:0: 1942 instructions in function +remark: solve_tril.py:238:0: 1942 instructions in function +remark: solve_tril.py:238:0: 1104 instructions in function +remark: solve_tril.py:238:0: 1104 instructions in function +remark: solve_tril.py:238:0: 1104 instructions in function +remark: solve_tril.py:238:0: 1104 instructions in function +remark: solve_tril.py:238:0: 698 instructions in function +remark: solve_tril.py:238:0: 698 instructions in function +remark: solve_tril.py:238:0: 698 instructions in function +remark: solve_tril.py:238:0: 698 instructions in function +remark: wy_fast.py:29:0: 1227 instructions in function +remark: wy_fast.py:29:0: 1373 instructions in function +remark: wy_fast.py:29:0: 1419 instructions in function +remark: wy_fast.py:29:0: 1093 instructions in function +remark: wy_fast.py:29:0: 1187 instructions in function +remark: wy_fast.py:29:0: 1213 instructions in function +remark: wy_fast.py:29:0: 611 instructions in function +remark: wy_fast.py:29:0: 675 instructions in function +remark: wy_fast.py:29:0: 691 instructions in function +remark: chunk_delta_h.py:45:0: 1529 instructions in function +remark: chunk_delta_h.py:45:0: 2447 instructions in function +remark: chunk_delta_h.py:45:0: 1768 instructions in function +remark: chunk_delta_h.py:45:0: 2739 instructions in function +remark: chunk_delta_h.py:45:0: 1952 instructions in function +remark: chunk_delta_h.py:45:0: 2951 instructions in function +remark: chunk_delta_h.py:45:0: 980 instructions in function +remark: chunk_delta_h.py:45:0: 1441 instructions in function +remark: chunk_delta_h.py:45:0: 1109 instructions in function +remark: chunk_delta_h.py:45:0: 1597 instructions in function +remark: chunk_delta_h.py:45:0: 1211 instructions in function +remark: chunk_delta_h.py:45:0: 1713 instructions in function +remark: chunk_o.py:42:0: 1020 instructions in function +remark: chunk_o.py:42:0: 1058 instructions in function +remark: chunk_o.py:42:0: 1080 instructions in function +remark: chunk_o.py:42:0: 847 instructions in function +remark: chunk_o.py:42:0: 871 instructions in function +remark: chunk_o.py:42:0: 883 instructions in function +remark: chunk_o.py:42:0: 1386 instructions in function +remark: chunk_o.py:42:0: 1422 instructions in function +remark: chunk_o.py:42:0: 1448 instructions in function +remark: chunk_o.py:42:0: 1021 instructions in function +remark: chunk_o.py:42:0: 1041 instructions in function +remark: chunk_o.py:42:0: 1055 instructions in function +remark: chunk_o.py:42:0: 1170 instructions in function +remark: chunk_o.py:42:0: 1234 instructions in function +remark: chunk_o.py:42:0: 1279 instructions in function +remark: chunk_o.py:42:0: 935 instructions in function +remark: chunk_o.py:42:0: 967 instructions in function +remark: chunk_o.py:42:0: 992 instructions in function +remark: chunk_o.py:42:0: 1482 instructions in function +remark: chunk_o.py:42:0: 1548 instructions in function +remark: chunk_o.py:42:0: 1601 instructions in function +remark: chunk_o.py:42:0: 1075 instructions in function +remark: chunk_o.py:42:0: 1101 instructions in function +remark: chunk_o.py:42:0: 1130 instructions in function +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:35:40 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +remark: fp8_utils.py:737:0: 775 instructions in function +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:35:40 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:35:40 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=52424) WARNING 09-24 16:35:41 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +remark: fused_qk_norm_rope.py:17:0: 160 instructions in function +(StageEngineCoreProc_stage0_replica0 pid=52424) INFO 09-24 16:35:49 [base.py:148] Available KV cache memory: 0.26 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=52424) Process StageEngineCoreProc_stage0_replica0: +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start. + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] Traceback (most recent call last): + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_core_proc.py", line 189, in run_stage_core + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc( + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 1093, in __init__ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] super().__init__( + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 145, in __init__ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] kv_cache_config = self._initialize_kv_caches(vllm_config) + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] return func(*args, **kwargs) + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 319, in _initialize_kv_caches + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] kv_cache_configs = get_kv_cache_configs( + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\core\kv_cache_utils.py", line 2331, in get_kv_cache_configs + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] _check_enough_kv_cache_memory( + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\core\kv_cache_utils.py", line 879, in _check_enough_kv_cache_memory + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] raise ValueError( + +(StageEngineCoreProc_stage0_replica0 pid=52424) ERROR 09-24 16:35:49 [stage_engine_core_proc.py:242] ValueError: To serve at least one request with the model's max seq len (512), (0.33 GiB KV cache is needed, which is larger than the available KV cache memory (0.22 GiB). Try increasing `gpu_memory_utilization` (which also controls CPU memory on the CPU backend) or decreasing `max_model_len` when initializing the engine. See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ for more details. +(StageEngineCoreProc_stage0_replica0 pid=52424) Traceback (most recent call last): +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\multiprocessing\process.py", line 314, in _bootstrap +(StageEngineCoreProc_stage0_replica0 pid=52424) self.run() +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\multiprocessing\process.py", line 108, in run +(StageEngineCoreProc_stage0_replica0 pid=52424) self._target(*self._args, **self._kwargs) +(StageEngineCoreProc_stage0_replica0 pid=52424) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage0_replica0 pid=52424) engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage0_replica0 pid=52424) ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage0_replica0 pid=52424) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=52424) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 1093, in __init__ +(StageEngineCoreProc_stage0_replica0 pid=52424) super().__init__( +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 145, in __init__ +(StageEngineCoreProc_stage0_replica0 pid=52424) kv_cache_config = self._initialize_kv_caches(vllm_config) +(StageEngineCoreProc_stage0_replica0 pid=52424) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage0_replica0 pid=52424) return func(*args, **kwargs) +(StageEngineCoreProc_stage0_replica0 pid=52424) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 319, in _initialize_kv_caches +(StageEngineCoreProc_stage0_replica0 pid=52424) kv_cache_configs = get_kv_cache_configs( +(StageEngineCoreProc_stage0_replica0 pid=52424) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\core\kv_cache_utils.py", line 2331, in get_kv_cache_configs +(StageEngineCoreProc_stage0_replica0 pid=52424) _check_enough_kv_cache_memory( +(StageEngineCoreProc_stage0_replica0 pid=52424) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\core\kv_cache_utils.py", line 879, in _check_enough_kv_cache_memory +(StageEngineCoreProc_stage0_replica0 pid=52424) raise ValueError( +(StageEngineCoreProc_stage0_replica0 pid=52424) ValueError: To serve at least one request with the model's max seq len (512), (0.33 GiB KV cache is needed, which is larger than the available KV cache memory (0.22 GiB). Try increasing `gpu_memory_utilization` (which also controls CPU memory on the CPU backend) or decreasing `max_model_len` when initializing the engine. See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ for more details. +ERROR 09-24 16:35:55 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s) + +ERROR 09-24 16:35:55 [stage_runtime.py:265] Traceback (most recent call last): + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + +ERROR 09-24 16:35:55 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + +ERROR 09-24 16:35:55 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + +ERROR 09-24 16:35:55 [stage_runtime.py:265] raise primary_exc + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + +ERROR 09-24 16:35:55 [stage_runtime.py:265] client = self._initialize_replica( + +ERROR 09-24 16:35:55 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + +ERROR 09-24 16:35:55 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout) + +ERROR 09-24 16:35:55 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + +ERROR 09-24 16:35:55 [stage_runtime.py:265] launch_stack.close() + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + +ERROR 09-24 16:35:55 [stage_runtime.py:265] self.__exit__(None, None, None) + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] raise exc_details[1] + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] if cb(*exc_details): + +ERROR 09-24 16:35:55 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + +ERROR 09-24 16:35:55 [stage_runtime.py:265] next(self.gen) + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + +ERROR 09-24 16:35:55 [stage_runtime.py:265] wait_for_engine_startup( + +ERROR 09-24 16:35:55 [stage_runtime.py:265] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + +ERROR 09-24 16:35:55 [stage_runtime.py:265] raise RuntimeError( + +ERROR 09-24 16:35:55 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} +ERROR 09-24 16:35:55 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] Traceback (most recent call last): + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator()) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] return future.result() + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] self._runtime.initialize() + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] raise exc + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] raise primary_exc + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] client = self._initialize_replica( + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] launch_stack.close() + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] self.__exit__(None, None, None) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] raise exc_details[1] + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] if cb(*exc_details): + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] next(self.gen) + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] wait_for_engine_startup( + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] raise RuntimeError( + +ERROR 09-24 16:35:55 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} +INFO 09-24 16:35:55 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +Exception in thread orchestrator: +Traceback (most recent call last): + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1075, in _bootstrap_inner + self.run() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + raise exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + next(self.gen) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} +Traceback (most recent call last): + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1075, in _bootstrap_inner + self.run() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + raise exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__ + next(self.gen) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\probe_qwen35_image.py", line 55, in main + record["config"] = config + ^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\entrypoints\omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 305, in __init__ + self._wait_for_orchestrator_init(startup_future, startup_timeout) + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 539, in _wait_for_orchestrator_init + startup_future.result( + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\concurrent\futures\_base.py", line 456, in result + return self.__get_result() + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\concurrent\futures\_base.py", line 401, in __get_result + raise self._exception +RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.json new file mode 100644 index 000000000..a54819659 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_attempt2.json @@ -0,0 +1,22 @@ +{ + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8", + "scope": "one synthetic-image Omni image-to-text request", + "start": 1790238380.8128512, + "config": { + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-FP8", + "language_model_only": false, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": true, + "gpu_memory_utilization": 0.75, + "cpu_offload_gb": 12.0, + "linear_backend": "triton", + "dtype": "bfloat16", + "stage_init_timeout": 180 + }, + "status": "failed", + "error": "RuntimeError(\"Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\")", + "traceback": "Traceback (most recent call last):\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1075, in _bootstrap_inner\n self.run()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1012, in run\n self._target(*self._args, **self._kwargs)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 468, in _bootstrap_orchestrator\n loop.run_until_complete(_run_orchestrator())\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\asyncio\\base_events.py\", line 691, in run_until_complete\n return future.result()\n ^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 424, in _run_orchestrator\n self._initialize_stages(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 350, in _initialize_stages\n self._initialize_stages_impl(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 371, in _initialize_stages_impl\n self._runtime.initialize()\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 278, in initialize\n raise exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 252, in initialize\n initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 589, in _initialize_stage_replicas\n raise primary_exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 550, in _init_group\n client = self._initialize_replica(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 828, in _initialize_replica\n return self._initialize_local_llm_replica(plan, stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 934, in _initialize_local_llm_replica\n launch_stack.close()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 618, in close\n self.__exit__(None, None, None)\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 610, in __exit__\n raise exc_details[1]\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 595, in __exit__\n if cb(*exc_details):\n ^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 144, in __exit__\n next(self.gen)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_engine_startup.py\", line 1211, in launch_stage_replica\n wait_for_engine_startup(\n File \"C:\\Users\\zhout\\w2\\omni029venv\\Lib\\site-packages\\vllm\\v1\\engine\\utils.py\", line 1307, in wait_for_engine_startup\n raise RuntimeError(\nRuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\probe_qwen35_image.py\", line 55, in main\n record[\"config\"] = config\n ^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\entrypoints\\omni_base.py\", line 223, in __init__\n self.engine = AsyncOmniEngine(\n ^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 305, in __init__\n self._wait_for_orchestrator_init(startup_future, startup_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 539, in _wait_for_orchestrator_init\n startup_future.result(\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 456, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 401, in __get_result\n raise self._exception\nRuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n", + "end": 1790238956.060437 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_ntfs_attempt1.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_ntfs_attempt1.driver.log new file mode 100644 index 000000000..27ab6b6a5 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_fp8_triton_ntfs_attempt1.driver.log @@ -0,0 +1,260 @@ +INFO 09-24 16:39:07 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:39:07 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 16:39:12 [omni_base.py:216] [Omni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8 +INFO 09-24 16:39:12 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8 +INFO 09-24 16:39:12 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-24 16:39:12 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 16:39:12 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-24 16:39:12 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-24 16:39:12 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration +INFO 09-24 16:39:12 [model.py:2021] Using max model len 512 +WARNING 09-24 16:39:12 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-24 16:39:18 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-24 16:39:18 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled +WARNING 09-24 16:39:18 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0. +WARNING 09-24 16:39:18 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-24 16:39:18 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-24 16:39:18 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-24 16:39:18 [vllm.py:1585] Cudagraph is disabled under eager mode +INFO 09-24 16:39:18 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-24 16:39:18 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group +INFO 09-24 16:39:32 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 16:39:32 [patch.py:510] inductor factorable-divisibility patch: installed. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:35 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:37 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_184a36a1caf0425a8339674b858f95e7 backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:39:37 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:37 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:39:37 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:37 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=66956) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:42 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:39:42 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:39:42 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:42 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:43 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:43 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:43 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:43 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:43 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:43 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:44 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:44 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:49 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:49 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 16.81 GiB. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:39:49 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (16.81 GiB). +(StageEngineCoreProc_stage0_replica0 pid=66956) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:40:12 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:40:12 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:40:13 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:40:13 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:40:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:40:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:40:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:40:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:40:23 [base.py:148] Available KV cache memory: 1.57 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:40:23 [kv_cache_utils.py:2032] GPU KV cache size: 2,340 tokens, Maximum concurrency for 512 tokens per request: 4.57x +(StageEngineCoreProc_stage0_replica0 pid=66956) WARNING 09-24 16:40:23 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +remark: utils.py:58:0: 58 instructions in function +(StageEngineCoreProc_stage0_replica0 pid=66956) INFO 09-24 16:40:23 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=66956) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00