From 74d284acba7ab818cea24e64551d0dc79fb79efd Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 24 Sep 2026 15:52:59 +0800 Subject: [PATCH] Record native Windows Qwen3.8 NVFP4 PTX startup blocker --- .../audit_qwen38_windows_attempt.py | 73 +++ benchmarks/edge_harness/probe_qwen35_image.py | 3 + .../results/e2e_expansion_20260924/README.md | 6 + .../qwen38_omni_windows_rtx/README.md | 9 + .../qwen38_omni_windows_rtx/audit_report.json | 34 ++ .../hardware_runtime.json | 33 ++ .../qwen38_image_unc_attempt1.json | 21 + .../qwen38_image_unc_attempt2.driver.log | 508 ++++++++++++++++++ .../qwen38_image_unc_attempt2.json | 21 + .../evidence/summary/README.md | 6 +- docs/design/omni_edge_e2e_roadmap.md | 2 + 11 files changed, 714 insertions(+), 2 deletions(-) create mode 100644 benchmarks/edge_harness/experiments/audit_qwen38_windows_attempt.py create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/audit_report.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/hardware_runtime.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt1.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt2.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt2.json diff --git a/benchmarks/edge_harness/experiments/audit_qwen38_windows_attempt.py b/benchmarks/edge_harness/experiments/audit_qwen38_windows_attempt.py new file mode 100644 index 000000000..375a56937 --- /dev/null +++ b/benchmarks/edge_harness/experiments/audit_qwen38_windows_attempt.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Audit the retained native-Windows Qwen3.8 NVFP4 startup failure.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +ROOT_CAUSE = "CUDA error: the provided PTX was compiled with an unsupported toolchain" +MODEL = "Qwen3.8-27B-NVFP4" + + +def sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-dir", type=Path, required=True) + args = parser.parse_args() + root = args.evidence_dir + attempts = [root / f"qwen38_image_unc_attempt{i}.json" for i in (1, 2)] + log_path = root / "qwen38_image_unc_attempt2.driver.log" + hardware_path = root / "hardware_runtime.json" + reports = [json.loads(path.read_text(encoding="utf-8")) for path in attempts] + log = log_path.read_text(encoding="utf-8", errors="replace") + hardware = json.loads(hardware_path.read_text(encoding="utf-8")) + if any(report["status"] != "failed" or MODEL not in report["model"] + or report["config"]["max_model_len"] != 512 + or report["config"]["gpu_memory_utilization"] != 0.85 + or report["config"]["cpu_offload_gb"] != 2 for report in reports): + raise ValueError("retained reports do not describe the bounded Windows NVFP4 attempt") + if not all(token in log for token in ( + "Loading safetensors checkpoint shards: 100% Completed | 3/3", + "process_weights_after_loading", + "prepare_fp4_layer_for_marlin", + "marlin_permute_scales", + ROOT_CAUSE, + )): + raise ValueError("captured worker log lacks the post-load Marlin/PTX root cause") + if ("610.71" not in hardware["nvidia_smi_header"] + or "CUDA UMD Version: 13.3" not in hardware["nvidia_smi_header"] + or "+cu134" not in hardware["installed_vllm_wheel"] + or hardware["nvcc"] != "13.4.59"): + raise ValueError("recorded driver/toolkit versions differ from the observed combination") + files = [*attempts, log_path, hardware_path] + result = { + "status": "audited_failure", + "evidence_depth": "backend load failure; no model inference or E2E output", + "artifact": MODEL, + "checkpoint_revision": "dbb8f445b3145f8a4c18ddc769f032d57d32867c", + "root_cause": ROOT_CAUSE, + "failure_location": "vLLM NVFP4 Marlin scale permutation during post-load weight processing", + "driver": "NVIDIA 610.71 / CUDA UMD 13.3", + "vllm_wheel": hardware["installed_vllm_wheel"], + "inference_completed": False, + "files": {path.name: {"bytes": path.stat().st_size, "sha256": sha256(path)} for path in files}, + "limits": [ + "The error is specific to the tested native-Windows wheel, driver and NVFP4 artifact.", + "The previously verified WSL text/image route is a different execution environment.", + "The driver/toolchain relationship is inferred from versions and NVIDIA's PTX compatibility guidance; no upgraded-driver control has run.", + ], + } + (root / "audit_report.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": result["status"], "root_cause": result["root_cause"]})) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/probe_qwen35_image.py b/benchmarks/edge_harness/probe_qwen35_image.py index 9a210fab3..00a6f36e0 100644 --- a/benchmarks/edge_harness/probe_qwen35_image.py +++ b/benchmarks/edge_harness/probe_qwen35_image.py @@ -64,7 +64,10 @@ def main(): if engine is not None: engine.close() record["end"] = time.time() + args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + if record["status"] != "completed": + raise SystemExit(1) if __name__ == "__main__": diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md index 976ff72f9..7718c774f 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md @@ -34,6 +34,12 @@ nearest-rank wall p50/p95 **0.294/0.302 s**, along with public AsyncOmni text, context refusal and fresh-stage recovery after server-started cancellation. This is a different artifact/runtime from the earlier 4B BF16 CUDA profile. +The [native Windows Qwen3.8-27B NVFP4 startup attempt](evidence/qwen38_omni_windows_rtx/README.md) +loaded all three shards under vLLM 0.29 but failed in Marlin post-load scale +permutation with an unsupported PTX toolchain error. Driver 610.71 reports +CUDA 13.3 while the installed Windows vLLM wheel was built with CUDA 13.4. +No native image/text inference completed; the WSL pass remains separate. + A [native Windows Qwen3-TTS hybrid restart probe](evidence/qwen_tts_omni_windows_restart/README.md) also confirmed backend prefill before cancellation. The worker and host-RAM reservation drained; a fresh Omni stage with the same pinned Radeon Vulkan0 diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md new file mode 100644 index 000000000..6ea268222 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md @@ -0,0 +1,9 @@ +# Qwen3.8-27B NVFP4 native Windows RTX startup attempt + +The pinned Qwen3.8-27B NVFP4 checkpoint, revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c`, was attempted through the current Omni/vLLM 0.29 one-stage pipeline on native Windows 11 build 26200. The bounded image-to-text case used a synthetic 96×96 red image, 512-token context, one sequence, 16 output tokens, eager execution, 0.85 GPU-memory utilization and an explicit 2 GiB CPU offload. This is the same scoped case that [completed on WSL RTX](../../../e2e_recovery_20260923/README.md); Windows **did not reach inference**. No output or latency is claimed for this cell. + +The [first attempt report](qwen38_image_unc_attempt1.json) records an engine-start failure after model load but its console output was not retained. The [second attempt report](qwen38_image_unc_attempt2.json) and [full driver/worker log](qwen38_image_unc_attempt2.driver.log) retain the root cause. All **three safetensors shards completed loading**, then vLLM failed while preparing NVFP4 Marlin weights: `process_weights_after_loading` → `prepare_fp4_layer_for_marlin` → `marlin_permute_scales` raised `torch.AcceleratorError: CUDA error: the provided PTX was compiled with an unsupported toolchain`. The [auditor](audit_report.json) checks the two reports, captured stack, hardware versions and source-file hashes. The probe now exits nonzero after writing a failed report, so automation cannot mistake this startup error for a successful image request. + +The [hardware/runtime record](hardware_runtime.json) identifies RTX 5090 Laptop driver **610.71**, CUDA UMD **13.3**, local CUDA toolkit **13.4.59**, PyTorch **2.13.0+cu130**, and the installed native vLLM **0.29.0+cu134** wheel. NVIDIA maps CUDA 13.4 to driver branch R615, while CUDA 13.3 maps to R610 in its [CUDA release notes](https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/). Its [minor-version compatibility guidance](https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html) says applications requiring PTX JIT can fail on an older driver. The version combination and exact PTX error make this a **likely driver/toolchain mismatch**, but no R615 control has run, so the cause is not proven by an A/B driver test. This does not establish that Qwen3.8 or native Windows is intrinsically unsupported. + +Reproduce with [the image probe](../../../../probe_qwen35_image.py) in the native Windows Python/vLLM 0.29 environment, `PYTHONPATH` pointing to this checkout, `PYTHONUTF8=1`, `VLLM_ENABLE_V1_MULTIPROCESSING=0`, `VLLM_USE_FLASHINFER_SAMPLER=0`, the local NVFP4 checkpoint as `--model`, and a fresh `--out` path. Capture stdout/stderr: the parent report alone omits the nested worker root cause. Run [the auditor](../../../../experiments/audit_qwen38_windows_attempt.py) on this evidence directory. Next, use an R615-or-later compatible display driver or a vLLM wheel/toolchain whose PTX is supported by R610, then rerun the exact text and image cases with placement, admission, cancellation and memory evidence. A Windows driver change is system-wide and may require a reboot; neither route has been applied in this record. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/audit_report.json new file mode 100644 index 000000000..542b281cf --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/audit_report.json @@ -0,0 +1,34 @@ +{ + "status": "audited_failure", + "evidence_depth": "backend load failure; no model inference or E2E output", + "artifact": "Qwen3.8-27B-NVFP4", + "checkpoint_revision": "dbb8f445b3145f8a4c18ddc769f032d57d32867c", + "root_cause": "CUDA error: the provided PTX was compiled with an unsupported toolchain", + "failure_location": "vLLM NVFP4 Marlin scale permutation during post-load weight processing", + "driver": "NVIDIA 610.71 / CUDA UMD 13.3", + "vllm_wheel": "vllm-0.29.0+cu134-cp312-cp312-win_amd64.whl", + "inference_completed": false, + "files": { + "qwen38_image_unc_attempt1.json": { + "bytes": 6210, + "sha256": "cd0c33532e3a7b921e769e6ca923c614600fec97fe786cc55cb9628c1e6c1391" + }, + "qwen38_image_unc_attempt2.json": { + "bytes": 6213, + "sha256": "b3fcae9c2cc626cf7cad16082e406fae2bbb12a44d57ad45d554c5096b2bc482" + }, + "qwen38_image_unc_attempt2.driver.log": { + "bytes": 48742, + "sha256": "7cffbb734382ef26fa7f335a47696071e2eb5c79ee4ab820b508b2185fbf8624" + }, + "hardware_runtime.json": { + "bytes": 1273, + "sha256": "200c8eb75770f5f5deab6d90a6aac4bbf9e4f2c6ef7d51d2fe628e47178ede88" + } + }, + "limits": [ + "The error is specific to the tested native-Windows wheel, driver and NVFP4 artifact.", + "The previously verified WSL text/image route is a different execution environment.", + "The driver/toolchain relationship is inferred from versions and NVIDIA's PTX compatibility guidance; no upgraded-driver control has run." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/hardware_runtime.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/hardware_runtime.json new file mode 100644 index 000000000..21a64704e --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/hardware_runtime.json @@ -0,0 +1,33 @@ +{ + "installed_vllm": "0.29.0", + "os": { + "Caption": "Microsoft Windows 11 家庭版 中文版", + "BuildNumber": "26200", + "TotalVisibleMemorySize": 66185068 + }, + "installed_vllm_wheel": "vllm-0.29.0+cu134-cp312-cp312-win_amd64.whl", + "triton": "3.8.0", + "gpus": [ + { + "Name": "NVIDIA GeForce RTX 5090 Laptop GPU", + "DriverVersion": "32.0.16.1071" + }, + { + "Name": "AMD Radeon(TM) 890M Graphics", + "DriverVersion": "32.0.22018.6001" + } + ], + "nvcc": "13.4.59", + "nvidia_smi_header": "Thu Sep 24 15:50:03 2026 \r\n+-----------------------------------------------------------------------------------------+\r\n| NVIDIA-SMI 610.71 KMD Version: 610.71 CUDA UMD Version: 13.3 |", + "cpu": { + "Name": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ", + "NumberOfCores": 12, + "NumberOfLogicalProcessors": 24 + }, + "power_scheme": "电源方案 GUID: 27fa6203-3987-4dcc-918d-748559d549ec (Performance)", + "torch": "2.13.0+cu130", + "captured_at": "2026-09-24T15:50:03.3415588+08:00", + "torch_cuda": "13.0", + "installed_vllm_wheel_sha256": "17b3dc7753f85561d9488e6ffe42b48e459d2d551d8aa788ab652e0a1c6cf77a", + "cuda_path": "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v13.4" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt1.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt1.json new file mode 100644 index 000000000..c65a839b1 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt1.json @@ -0,0 +1,21 @@ +{ + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4", + "scope": "one synthetic-image Omni image-to-text request", + "start": 1790235471.3124, + "config": { + "model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4", + "language_model_only": false, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": true, + "gpu_memory_utilization": 0.85, + "cpu_offload_gb": 2, + "dtype": "bfloat16", + "stage_init_timeout": 180 + }, + "status": "failed", + "error": "RuntimeError(\"Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\")", + "traceback": "Traceback (most recent call last):\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1075, in _bootstrap_inner\n self.run()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1012, in run\n self._target(*self._args, **self._kwargs)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 468, in _bootstrap_orchestrator\n loop.run_until_complete(_run_orchestrator())\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\asyncio\\base_events.py\", line 691, in run_until_complete\n return future.result()\n ^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 424, in _run_orchestrator\n self._initialize_stages(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 350, in _initialize_stages\n self._initialize_stages_impl(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 371, in _initialize_stages_impl\n self._runtime.initialize()\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 278, in initialize\n raise exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 252, in initialize\n initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 589, in _initialize_stage_replicas\n raise primary_exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 550, in _init_group\n client = self._initialize_replica(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 828, in _initialize_replica\n return self._initialize_local_llm_replica(plan, stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 934, in _initialize_local_llm_replica\n launch_stack.close()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 618, in close\n self.__exit__(None, None, None)\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 610, in __exit__\n raise exc_details[1]\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 595, in __exit__\n if cb(*exc_details):\n ^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 144, in __exit__\n next(self.gen)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_engine_startup.py\", line 1211, in launch_stage_replica\n wait_for_engine_startup(\n File \"C:\\Users\\zhout\\w2\\omni029venv\\Lib\\site-packages\\vllm\\v1\\engine\\utils.py\", line 1307, in wait_for_engine_startup\n raise RuntimeError(\nRuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\probe_qwen35_image.py\", line 51, in main\n engine = Omni(**config)\n ^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\entrypoints\\omni_base.py\", line 223, in __init__\n self.engine = AsyncOmniEngine(\n ^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 305, in __init__\n self._wait_for_orchestrator_init(startup_future, startup_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 539, in _wait_for_orchestrator_init\n startup_future.result(\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 456, in result\n return self.__get_result()\n ^^^^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 401, in __get_result\n raise self._exception\nRuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n", + "end": 1790235698.6267703 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt2.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt2.driver.log new file mode 100644 index 000000000..d580ce96b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_image_unc_attempt2.driver.log @@ -0,0 +1,508 @@ +INFO 09-24 15:42:33 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 15:42:33 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 15:42:37 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4 +INFO 09-24 15:42:37 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4 +INFO 09-24 15:42:37 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-24 15:42:38 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 15:42:38 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-24 15:42:38 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-24 15:42:38 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration +INFO 09-24 15:42:38 [model.py:2021] Using max model len 512 +WARNING 09-24 15:42:38 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-24 15:42:44 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-24 15:42:44 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled +WARNING 09-24 15:42:44 [modelopt.py:379] Detected ModelOpt fp8 checkpoint (quant_algo=FP8). Please note that the format is experimental and could change. +WARNING 09-24 15:42:44 [modelopt.py:1029] Detected ModelOpt NVFP4 checkpoint (quant_algo=NVFP4). Please note that the format is experimental and could change in future. +WARNING 09-24 15:42:44 [modelopt.py:1029] Detected ModelOpt NVFP4 checkpoint (quant_algo=W4A16_NVFP4). Please note that the format is experimental and could change in future. +WARNING 09-24 15:42:44 [modelopt.py:1711] Detected ModelOpt MXFP8 checkpoint. Please note that the format is experimental and could change in future. +WARNING 09-24 15:42:44 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-24 15:42:44 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-24 15:42:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-24 15:42:44 [vllm.py:1585] Cudagraph is disabled under eager mode +INFO 09-24 15:42:44 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-24 15:42:44 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group +INFO 09-24 15:42:57 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 15:42:57 [patch.py:510] inductor factorable-divisibility patch: installed. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:00 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=modelopt_mixed, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:03 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_47ff13a08296498c9719343b5e759f57 backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:03 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:03 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:03 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:03 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=65340) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4... +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [__init__.py:726] Selected CutlassFP8ScaledMMLinearKernel for ModelOptFp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [__init__.py:1106] Using MarlinNvFp4LinearKernel for NVFP4 GEMM +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:11 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:11 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:12 [uva.py:65] Total CPU offloaded parameters: 2.03 +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:13 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 20.42 GiB. Available RAM: 10.39 GiB. +(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:13 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (20.42 GiB) exceeds 90% of available RAM (10.39 GiB). +(StageEngineCoreProc_stage0_replica0 pid=65340) +Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00