Record native Windows Qwen3.8 NVFP4 PTX startup blocker
This commit is contained in:
tzhouam committed
1 parent
605b68cc5f
commit
74d284acba
11 files changed
+714
-2
No files matched your search
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit the retained native-Windows Qwen3.8 NVFP4 startup failure."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT_CAUSE = "CUDA error: the provided PTX was compiled with an unsupported toolchain"
|
||||
MODEL = "Qwen3.8-27B-NVFP4"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--evidence-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
root = args.evidence_dir
|
||||
attempts = [root / f"qwen38_image_unc_attempt{i}.json" for i in (1, 2)]
|
||||
log_path = root / "qwen38_image_unc_attempt2.driver.log"
|
||||
hardware_path = root / "hardware_runtime.json"
|
||||
reports = [json.loads(path.read_text(encoding="utf-8")) for path in attempts]
|
||||
log = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
hardware = json.loads(hardware_path.read_text(encoding="utf-8"))
|
||||
if any(report["status"] != "failed" or MODEL not in report["model"]
|
||||
or report["config"]["max_model_len"] != 512
|
||||
or report["config"]["gpu_memory_utilization"] != 0.85
|
||||
or report["config"]["cpu_offload_gb"] != 2 for report in reports):
|
||||
raise ValueError("retained reports do not describe the bounded Windows NVFP4 attempt")
|
||||
if not all(token in log for token in (
|
||||
"Loading safetensors checkpoint shards: 100% Completed | 3/3",
|
||||
"process_weights_after_loading",
|
||||
"prepare_fp4_layer_for_marlin",
|
||||
"marlin_permute_scales",
|
||||
ROOT_CAUSE,
|
||||
)):
|
||||
raise ValueError("captured worker log lacks the post-load Marlin/PTX root cause")
|
||||
if ("610.71" not in hardware["nvidia_smi_header"]
|
||||
or "CUDA UMD Version: 13.3" not in hardware["nvidia_smi_header"]
|
||||
or "+cu134" not in hardware["installed_vllm_wheel"]
|
||||
or hardware["nvcc"] != "13.4.59"):
|
||||
raise ValueError("recorded driver/toolkit versions differ from the observed combination")
|
||||
files = [*attempts, log_path, hardware_path]
|
||||
result = {
|
||||
"status": "audited_failure",
|
||||
"evidence_depth": "backend load failure; no model inference or E2E output",
|
||||
"artifact": MODEL,
|
||||
"checkpoint_revision": "dbb8f445b3145f8a4c18ddc769f032d57d32867c",
|
||||
"root_cause": ROOT_CAUSE,
|
||||
"failure_location": "vLLM NVFP4 Marlin scale permutation during post-load weight processing",
|
||||
"driver": "NVIDIA 610.71 / CUDA UMD 13.3",
|
||||
"vllm_wheel": hardware["installed_vllm_wheel"],
|
||||
"inference_completed": False,
|
||||
"files": {path.name: {"bytes": path.stat().st_size, "sha256": sha256(path)} for path in files},
|
||||
"limits": [
|
||||
"The error is specific to the tested native-Windows wheel, driver and NVFP4 artifact.",
|
||||
"The previously verified WSL text/image route is a different execution environment.",
|
||||
"The driver/toolchain relationship is inferred from versions and NVIDIA's PTX compatibility guidance; no upgraded-driver control has run.",
|
||||
],
|
||||
}
|
||||
(root / "audit_report.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"status": result["status"], "root_cause": result["root_cause"]}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -64,7 +64,10 @@ def main():
|
||||
if engine is not None:
|
||||
engine.close()
|
||||
record["end"] = time.time()
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
|
||||
if record["status"] != "completed":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -34,6 +34,12 @@ nearest-rank wall p50/p95 **0.294/0.302 s**, along with public AsyncOmni text,
|
||||
context refusal and fresh-stage recovery after server-started cancellation.
|
||||
This is a different artifact/runtime from the earlier 4B BF16 CUDA profile.
|
||||
|
||||
The [native Windows Qwen3.8-27B NVFP4 startup attempt](evidence/qwen38_omni_windows_rtx/README.md)
|
||||
loaded all three shards under vLLM 0.29 but failed in Marlin post-load scale
|
||||
permutation with an unsupported PTX toolchain error. Driver 610.71 reports
|
||||
CUDA 13.3 while the installed Windows vLLM wheel was built with CUDA 13.4.
|
||||
No native image/text inference completed; the WSL pass remains separate.
|
||||
|
||||
A [native Windows Qwen3-TTS hybrid restart probe](evidence/qwen_tts_omni_windows_restart/README.md)
|
||||
also confirmed backend prefill before cancellation. The worker and host-RAM
|
||||
reservation drained; a fresh Omni stage with the same pinned Radeon Vulkan0
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
# Qwen3.8-27B NVFP4 native Windows RTX startup attempt
|
||||
|
||||
The pinned Qwen3.8-27B NVFP4 checkpoint, revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c`, was attempted through the current Omni/vLLM 0.29 one-stage pipeline on native Windows 11 build 26200. The bounded image-to-text case used a synthetic 96×96 red image, 512-token context, one sequence, 16 output tokens, eager execution, 0.85 GPU-memory utilization and an explicit 2 GiB CPU offload. This is the same scoped case that [completed on WSL RTX](../../../e2e_recovery_20260923/README.md); Windows **did not reach inference**. No output or latency is claimed for this cell.
|
||||
|
||||
The [first attempt report](qwen38_image_unc_attempt1.json) records an engine-start failure after model load but its console output was not retained. The [second attempt report](qwen38_image_unc_attempt2.json) and [full driver/worker log](qwen38_image_unc_attempt2.driver.log) retain the root cause. All **three safetensors shards completed loading**, then vLLM failed while preparing NVFP4 Marlin weights: `process_weights_after_loading` → `prepare_fp4_layer_for_marlin` → `marlin_permute_scales` raised `torch.AcceleratorError: CUDA error: the provided PTX was compiled with an unsupported toolchain`. The [auditor](audit_report.json) checks the two reports, captured stack, hardware versions and source-file hashes. The probe now exits nonzero after writing a failed report, so automation cannot mistake this startup error for a successful image request.
|
||||
|
||||
The [hardware/runtime record](hardware_runtime.json) identifies RTX 5090 Laptop driver **610.71**, CUDA UMD **13.3**, local CUDA toolkit **13.4.59**, PyTorch **2.13.0+cu130**, and the installed native vLLM **0.29.0+cu134** wheel. NVIDIA maps CUDA 13.4 to driver branch R615, while CUDA 13.3 maps to R610 in its [CUDA release notes](https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/). Its [minor-version compatibility guidance](https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html) says applications requiring PTX JIT can fail on an older driver. The version combination and exact PTX error make this a **likely driver/toolchain mismatch**, but no R615 control has run, so the cause is not proven by an A/B driver test. This does not establish that Qwen3.8 or native Windows is intrinsically unsupported.
|
||||
|
||||
Reproduce with [the image probe](../../../../probe_qwen35_image.py) in the native Windows Python/vLLM 0.29 environment, `PYTHONPATH` pointing to this checkout, `PYTHONUTF8=1`, `VLLM_ENABLE_V1_MULTIPROCESSING=0`, `VLLM_USE_FLASHINFER_SAMPLER=0`, the local NVFP4 checkpoint as `--model`, and a fresh `--out` path. Capture stdout/stderr: the parent report alone omits the nested worker root cause. Run [the auditor](../../../../experiments/audit_qwen38_windows_attempt.py) on this evidence directory. Next, use an R615-or-later compatible display driver or a vLLM wheel/toolchain whose PTX is supported by R610, then rerun the exact text and image cases with placement, admission, cancellation and memory evidence. A Windows driver change is system-wide and may require a reboot; neither route has been applied in this record.
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"status": "audited_failure",
|
||||
"evidence_depth": "backend load failure; no model inference or E2E output",
|
||||
"artifact": "Qwen3.8-27B-NVFP4",
|
||||
"checkpoint_revision": "dbb8f445b3145f8a4c18ddc769f032d57d32867c",
|
||||
"root_cause": "CUDA error: the provided PTX was compiled with an unsupported toolchain",
|
||||
"failure_location": "vLLM NVFP4 Marlin scale permutation during post-load weight processing",
|
||||
"driver": "NVIDIA 610.71 / CUDA UMD 13.3",
|
||||
"vllm_wheel": "vllm-0.29.0+cu134-cp312-cp312-win_amd64.whl",
|
||||
"inference_completed": false,
|
||||
"files": {
|
||||
"qwen38_image_unc_attempt1.json": {
|
||||
"bytes": 6210,
|
||||
"sha256": "cd0c33532e3a7b921e769e6ca923c614600fec97fe786cc55cb9628c1e6c1391"
|
||||
},
|
||||
"qwen38_image_unc_attempt2.json": {
|
||||
"bytes": 6213,
|
||||
"sha256": "b3fcae9c2cc626cf7cad16082e406fae2bbb12a44d57ad45d554c5096b2bc482"
|
||||
},
|
||||
"qwen38_image_unc_attempt2.driver.log": {
|
||||
"bytes": 48742,
|
||||
"sha256": "7cffbb734382ef26fa7f335a47696071e2eb5c79ee4ab820b508b2185fbf8624"
|
||||
},
|
||||
"hardware_runtime.json": {
|
||||
"bytes": 1273,
|
||||
"sha256": "200c8eb75770f5f5deab6d90a6aac4bbf9e4f2c6ef7d51d2fe628e47178ede88"
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"The error is specific to the tested native-Windows wheel, driver and NVFP4 artifact.",
|
||||
"The previously verified WSL text/image route is a different execution environment.",
|
||||
"The driver/toolchain relationship is inferred from versions and NVIDIA's PTX compatibility guidance; no upgraded-driver control has run."
|
||||
]
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"installed_vllm": "0.29.0",
|
||||
"os": {
|
||||
"Caption": "Microsoft Windows 11 家庭版 中文版",
|
||||
"BuildNumber": "26200",
|
||||
"TotalVisibleMemorySize": 66185068
|
||||
},
|
||||
"installed_vllm_wheel": "vllm-0.29.0+cu134-cp312-cp312-win_amd64.whl",
|
||||
"triton": "3.8.0",
|
||||
"gpus": [
|
||||
{
|
||||
"Name": "NVIDIA GeForce RTX 5090 Laptop GPU",
|
||||
"DriverVersion": "32.0.16.1071"
|
||||
},
|
||||
{
|
||||
"Name": "AMD Radeon(TM) 890M Graphics",
|
||||
"DriverVersion": "32.0.22018.6001"
|
||||
}
|
||||
],
|
||||
"nvcc": "13.4.59",
|
||||
"nvidia_smi_header": "Thu Sep 24 15:50:03 2026 \r\n+-----------------------------------------------------------------------------------------+\r\n| NVIDIA-SMI 610.71 KMD Version: 610.71 CUDA UMD Version: 13.3 |",
|
||||
"cpu": {
|
||||
"Name": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M ",
|
||||
"NumberOfCores": 12,
|
||||
"NumberOfLogicalProcessors": 24
|
||||
},
|
||||
"power_scheme": "电源方案 GUID: 27fa6203-3987-4dcc-918d-748559d549ec (Performance)",
|
||||
"torch": "2.13.0+cu130",
|
||||
"captured_at": "2026-09-24T15:50:03.3415588+08:00",
|
||||
"torch_cuda": "13.0",
|
||||
"installed_vllm_wheel_sha256": "17b3dc7753f85561d9488e6ffe42b48e459d2d551d8aa788ab652e0a1c6cf77a",
|
||||
"cuda_path": "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v13.4"
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4",
|
||||
"scope": "one synthetic-image Omni image-to-text request",
|
||||
"start": 1790235471.3124,
|
||||
"config": {
|
||||
"model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.85,
|
||||
"cpu_offload_gb": 2,
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "RuntimeError(\"Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\")",
|
||||
"traceback": "Traceback (most recent call last):\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1075, in _bootstrap_inner\n self.run()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1012, in run\n self._target(*self._args, **self._kwargs)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 468, in _bootstrap_orchestrator\n loop.run_until_complete(_run_orchestrator())\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\asyncio\\base_events.py\", line 691, in run_until_complete\n return future.result()\n ^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 424, in _run_orchestrator\n self._initialize_stages(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 350, in _initialize_stages\n self._initialize_stages_impl(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 371, in _initialize_stages_impl\n self._runtime.initialize()\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 278, in initialize\n raise exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 252, in initialize\n initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 589, in _initialize_stage_replicas\n raise primary_exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 550, in _init_group\n client = self._initialize_replica(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 828, in _initialize_replica\n return self._initialize_local_llm_replica(plan, stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 934, in _initialize_local_llm_replica\n launch_stack.close()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 618, in close\n self.__exit__(None, None, None)\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 610, in __exit__\n raise exc_details[1]\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 595, in __exit__\n if cb(*exc_details):\n ^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 144, in __exit__\n next(self.gen)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_engine_startup.py\", line 1211, in launch_stage_replica\n wait_for_engine_startup(\n File \"C:\\Users\\zhout\\w2\\omni029venv\\Lib\\site-packages\\vllm\\v1\\engine\\utils.py\", line 1307, in wait_for_engine_startup\n raise RuntimeError(\nRuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\probe_qwen35_image.py\", line 51, in main\n engine = Omni(**config)\n ^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\entrypoints\\omni_base.py\", line 223, in __init__\n self.engine = AsyncOmniEngine(\n ^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 305, in __init__\n self._wait_for_orchestrator_init(startup_future, startup_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 539, in _wait_for_orchestrator_init\n startup_future.result(\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 456, in resultLine truncated
|
||||
"end": 1790235698.6267703
|
||||
}
|
||||
+508
@@ -0,0 +1,508 @@
|
||||
INFO 09-24 15:42:33 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 15:42:33 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 15:42:37 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4
|
||||
INFO 09-24 15:42:37 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4
|
||||
INFO 09-24 15:42:37 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-24 15:42:38 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 15:42:38 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-24 15:42:38 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-24 15:42:38 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration
|
||||
INFO 09-24 15:42:38 [model.py:2021] Using max model len 512
|
||||
WARNING 09-24 15:42:38 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-24 15:42:44 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-24 15:42:44 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled
|
||||
WARNING 09-24 15:42:44 [modelopt.py:379] Detected ModelOpt fp8 checkpoint (quant_algo=FP8). Please note that the format is experimental and could change.
|
||||
WARNING 09-24 15:42:44 [modelopt.py:1029] Detected ModelOpt NVFP4 checkpoint (quant_algo=NVFP4). Please note that the format is experimental and could change in future.
|
||||
WARNING 09-24 15:42:44 [modelopt.py:1029] Detected ModelOpt NVFP4 checkpoint (quant_algo=W4A16_NVFP4). Please note that the format is experimental and could change in future.
|
||||
WARNING 09-24 15:42:44 [modelopt.py:1711] Detected ModelOpt MXFP8 checkpoint. Please note that the format is experimental and could change in future.
|
||||
WARNING 09-24 15:42:44 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-24 15:42:44 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-24 15:42:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-24 15:42:44 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 15:42:44 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-24 15:42:44 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-24 15:42:57 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 15:42:57 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:00 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=modelopt_mixed, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:03 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_47ff13a08296498c9719343b5e759f57 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:03 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:03 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:03 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:03 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:43:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Qwen3.8-27B-NVFP4...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:09 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [uva.py:65] Total CPU offloaded parameters: 0.86
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [__init__.py:726] Selected CutlassFP8ScaledMMLinearKernel for ModelOptFp8LinearMethod
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:10 [__init__.py:1106] Using MarlinNvFp4LinearKernel for NVFP4 GEMM
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:11 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:11 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:12 [uva.py:65] Total CPU offloaded parameters: 2.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:13 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 20.42 GiB. Available RAM: 10.39 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:43:13 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (20.42 GiB) exceeds 90% of available RAM (10.39 GiB).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
Loading safetensors checkpoint shards: 33% Completed | 1/3 [01:24<02:48, 84.41s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
Loading safetensors checkpoint shards: 67% Completed | 2/3 [02:36<01:17, 77.42s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 3/3 [02:44<00:00, 45.60s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 3/3 [02:44<00:00, 54.89s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) INFO 09-24 15:45:57 [default_loader.py:430] Loading weights took 164.84 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) WARNING 09-24 15:45:58 [marlin.py:34] Your GPU does not have native support for FP4 computation but FP4 quantization is being used. Weight-only FP4 compression will be used leveraging the Marlin kernel. This may degrade performance for compute-heavy workloads.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 1093, in __init__
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] super().__init__(
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 134, in __init__
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\abstract.py", line 110, in __init__
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] self._init_executor()
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\uniproc_executor.py", line 74, in _init_executor
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] self.driver_worker.load_model()
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\base.py", line 41, in load_model
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] res = super().load_model(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\worker\gpu_worker.py", line 495, in load_model
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\gpu_ar_model_runner.py", line 400, in load_model
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] super().load_model(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\init_timeline.py", line 343, in wrapper
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] return fn(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\gpu_model_runner.py", line 188, in load_model
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] super().load_model(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\worker\gpu_model_runner.py", line 5395, in load_model
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] self.model = model_loader.load_model(
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\model_loader\base_loader.py", line 80, in load_model
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] process_weights_after_loading(model, model_config, target_device)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\model_loader\utils.py", line 113, in process_weights_after_loading
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] quant_method.process_weights_after_loading(module)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\patch.py", line 242, in _patched_nvfp4_pwal
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] _original_nvfp4_pwal(self, layer, *args, **kwargs)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\modelopt.py", line 1249, in process_weights_after_loading
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\kernels\linear\nvfp4\marlin.py", line 40, in process_weights_after_loading
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] prepare_fp4_layer_for_marlin(layer)
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\marlin_utils_fp4.py", line 275, in prepare_fp4_layer_for_marlin
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] weight_scale = marlin_permute_scales(
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) Process StageEngineCoreProc_stage0_replica0:
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\marlin_utils.py", line 475, in marlin_permute_scales
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] torch.AcceleratorError: CUDA error: the provided PTX was compiled with an unsupported toolchain.
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] Search for `cudaErrorUnsupportedPtxVersion' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242] For more detailed error information, run with CUDA_LOG_FILE=stderr
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ERROR 09-24 15:45:59 [stage_engine_core_proc.py:242]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\multiprocessing\process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self.run()
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\multiprocessing\process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 1093, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) super().__init__(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\core.py", line 134, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self._init_executor()
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\executor\uniproc_executor.py", line 74, in _init_executor
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self.driver_worker.load_model()
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\base.py", line 41, in load_model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) res = super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\worker\gpu_worker.py", line 495, in load_model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\gpu_ar_model_runner.py", line 400, in load_model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\worker\gpu_model_runner.py", line 188, in load_model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\worker\gpu_model_runner.py", line 5395, in load_model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self.model = model_loader.load_model(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\tracing\otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\model_loader\base_loader.py", line 80, in load_model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) process_weights_after_loading(model, model_config, target_device)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\model_loader\utils.py", line 113, in process_weights_after_loading
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) quant_method.process_weights_after_loading(module)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\patch.py", line 242, in _patched_nvfp4_pwal
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) _original_nvfp4_pwal(self, layer, *args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\modelopt.py", line 1249, in process_weights_after_loading
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) self.kernel.process_weights_after_loading(layer)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\kernels\linear\nvfp4\marlin.py", line 40, in process_weights_after_loading
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) prepare_fp4_layer_for_marlin(layer)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\marlin_utils_fp4.py", line 275, in prepare_fp4_layer_for_marlin
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) weight_scale = marlin_permute_scales(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\marlin_utils.py", line 475, in marlin_permute_scales
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) torch.AcceleratorError: CUDA error: the provided PTX was compiled with an unsupported toolchain.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) Search for `cudaErrorUnsupportedPtxVersion' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340) For more detailed error information, run with CUDA_LOG_FILE=stderr
|
||||
(StageEngineCoreProc_stage0_replica0 pid=65340)
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s)
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] raise primary_exc
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] launch_stack.close()
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] raise exc_details[1]
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] if cb(*exc_details):
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] next(self.gen)
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] raise RuntimeError(
|
||||
|
||||
ERROR 09-24 15:46:02 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] return future.result()
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] raise exc
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] raise primary_exc
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] launch_stack.close()
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] raise exc_details[1]
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] next(self.gen)
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] raise RuntimeError(
|
||||
|
||||
ERROR 09-24 15:46:02 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}
|
||||
INFO 09-24 15:46:02 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}
|
||||
Traceback (most recent call last):
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\v1\engine\utils.py", line 1307, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\probe_qwen35_image.py", line 51, in main
|
||||
engine = Omni(**config)
|
||||
^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\entrypoints\omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\concurrent\futures\_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\concurrent\futures\_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4",
|
||||
"scope": "one synthetic-image Omni image-to-text request",
|
||||
"start": 1790235738.7803814,
|
||||
"config": {
|
||||
"model": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\Qwen3.8-27B-NVFP4",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.85,
|
||||
"cpu_offload_gb": 2,
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "RuntimeError(\"Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\")",
|
||||
"traceback": "Traceback (most recent call last):\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1075, in _bootstrap_inner\n self.run()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\threading.py\", line 1012, in run\n self._target(*self._args, **self._kwargs)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 468, in _bootstrap_orchestrator\n loop.run_until_complete(_run_orchestrator())\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\asyncio\\base_events.py\", line 691, in run_until_complete\n return future.result()\n ^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 424, in _run_orchestrator\n self._initialize_stages(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 350, in _initialize_stages\n self._initialize_stages_impl(stage_init_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 371, in _initialize_stages_impl\n self._runtime.initialize()\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 278, in initialize\n raise exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 252, in initialize\n initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 589, in _initialize_stage_replicas\n raise primary_exc\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 550, in _init_group\n client = self._initialize_replica(\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 828, in _initialize_replica\n return self._initialize_local_llm_replica(plan, stage_init_timeout)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_runtime.py\", line 934, in _initialize_local_llm_replica\n launch_stack.close()\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 618, in close\n self.__exit__(None, None, None)\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 610, in __exit__\n raise exc_details[1]\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 595, in __exit__\n if cb(*exc_details):\n ^^^^^^^^^^^^^^^^\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\contextlib.py\", line 144, in __exit__\n next(self.gen)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\stage_engine_startup.py\", line 1211, in launch_stage_replica\n wait_for_engine_startup(\n File \"C:\\Users\\zhout\\w2\\omni029venv\\Lib\\site-packages\\vllm\\v1\\engine\\utils.py\", line 1307, in wait_for_engine_startup\n raise RuntimeError(\nRuntimeError: Engine core initialization failed. See root cause above. Failed proc(s): {'StageEngineCoreProc_stage0_replica0': 1}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\probe_qwen35_image.py\", line 51, in main\n engine = Omni(**config)\n ^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\entrypoints\\omni_base.py\", line 223, in __init__\n self.engine = AsyncOmniEngine(\n ^^^^^^^^^^^^^^^^\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 305, in __init__\n self._wait_for_orchestrator_init(startup_future, startup_timeout)\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\vllm_omni\\engine\\async_omni_engine.py\", line 539, in _wait_for_orchestrator_init\n startup_future.result(\n File \"C:\\Users\\zhout\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\concurrent\\futures\\_base.py\", line 456, in resultLine truncated
|
||||
"end": 1790235962.5793004
|
||||
}
|
||||
@@ -12,6 +12,8 @@ A [native Windows Radeon+CPU Qwen3-TTS recovery probe](../../../e2e_expansion_20
|
||||
|
||||
A separate [native Windows RTX Spark GGUF Omni profile](../../../e2e_expansion_20260924/evidence/spark_omni_windows_rtx/README.md) passed two named complete text requests, 20/20 serial inventory requests at 0.294/0.302 s p50/p95, public `AsyncOmni.generate`, context refusal and a fresh-stage restart after confirmed server-started cancellation. Its pinned 1.7B Q4_K_M llama.cpp Vulkan0 route offloaded 29/29 layers to the RTX 5090 Laptop. This is distinct from the earlier 4B BF16 vLLM CUDA profile and does not establish token streaming or broad quality.
|
||||
|
||||
The [native Windows RTX Qwen3.8-27B NVFP4 attempt](../../../e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md) loaded all three pinned weight shards but failed before inference during vLLM Marlin post-load scale permutation with an unsupported PTX toolchain error. The current Windows driver reports CUDA 13.3 while the installed vLLM wheel was built with CUDA 13.4. This narrows the Windows blocker to the tested runtime combination; it does not reverse the WSL scoped text/image pass or qualify a Windows output.
|
||||
|
||||
A complete pinned [Spark 1.7B BF16 WSL CPU checkpoint and acceptance run](../../../e2e_expansion_20260924/evidence/spark_bf16_wsl_cpu/README.md) adds a second current-Omni text artifact to the GGUF route. Twelve prompts produced 128 tokens each with clean cancellation; all 12 token-ID sequences exactly matched standalone vLLM on the same checkpoint. One warmup plus 20 serial 28-prompt-token/64-output-token requests measured p50/p95 complete-request wall time 5.411/5.751 s and TTFT 0.129/0.146 s on the fork branch. A fresh-session restart after an eight-token cancellation reproduced the same 128 greedy token IDs, while the retired handle was explicitly rejected. This is WSL CPU full-text scope, not mobile generation or broad quality qualification.
|
||||
|
||||
On a hosted [Galaxy S24 Spark attention-layer experiment](../../../e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md), a real one-layer W8A16 QDQ artifact executed on NPU and matched prior S25 component outputs exactly on one dataset. The 100-sample layer p50/p95 was 1.109/1.205 ms; the service's 1.098 ms estimated-time field was the sample minimum. FP32-source versus S24 output hidden-state relative L2 was 1.351% without a downstream token-quality gate. This adds component C evidence only; device-local generation remains absent.
|
||||
@@ -49,7 +51,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370: CPU / WSL | scoped E2E: Omni GGUF or BF16 Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF combined image+audio → text+speech | synthetic policy pass: Omni CPU |
|
||||
| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text; fresh restart after in-flight cancel | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | scoped E2E: Omni Spark 1.7B GGUF text on RTX Vulkan; fresh restart after in-flight cancel (separate 4B BF16 CUDA profile) | profile completed; 30-min repeat no collapse, brief playback jitter | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | scoped E2E: Omni Spark 1.7B GGUF text on RTX Vulkan; fresh restart after in-flight cancel (separate 4B BF16 CUDA profile) | profile completed; 30-min repeat no collapse, brief playback jitter | NOT E2E (NVFP4 post-load PTX startup failed on current driver/wheel) | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
|
||||
| PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text; fresh restart after in-flight cancel | scoped E2E: Omni Radeon talker/codec + CPU predictor; fresh restart after in-flight cancel | NOT E2E | scoped E2E: Omni Radeon+CPU image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU+Radeon Cosmos (WSL/native Windows) |
|
||||
| PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
@@ -80,7 +82,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI_CUDA; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted Omni whole-policy graph stage with BF16 policy and Cosmos encoder on the RTX 5090 Laptop GPU. One warmup plus 20/20 serial patterned synthetic observation/state/noise requests returned identical finite float32 [1,50,32] actions with terminal events; nearest-rank complete-request wall p50/p95 was 0.382/0.399 s, excluding 43.82 s startup. The 16 GiB host-RAM and 16 GiB CUDA demands were within explicit 28/21 GiB ceilings checked against available memory. Actual CUDA placement was verified, PyTorch peak reserved VRAM reached 7.86 GB, and an 8 GiB VRAM plan was refused before worker launch. An in-flight abort and shutdown released both ledgers with no stale output; a separate public AsyncOmni.generate action request passed. Relative to a separate patterned WSL CPU output, action relative L2 difference was 2.006% with no task tolerance. The earlier one-forward zero-input direct CUDA run was a different experiment. This is synthetic action-policy integration, not robot-task E2E; units/order/step time remain unverified and control_ready=false. | Validate real observations/reference actions and physical action semantics, post-cancel restart, concurrent admission, loading peak, deadlines and sustained latency/power/thermal behavior; [Omni CUDA evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_cuda/README.md), [earlier direct result](../../../e2e_recovery_20260923/evidence/internvla/internvla_finetune_synthetic.json). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST_GGUF_VULKAN; RESTART_AFTER_INFLIGHT_CANCEL_PASS; SEPARATE_4B_BF16_CUDA_PROFILE | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran through an Omni whole-session StageRuntime/StagePool backend on RTX Vulkan0. The server log identifies NVIDIA GeForce RTX 5090 Laptop GPU, 29/29 layers offloaded and a 1051.02 MiB Vulkan0 model buffer. The short and 120-item inventory prompts answered `Paris` and `1511`; one warmup plus 20/20 serial inventory requests answered `1511` with nearest-rank complete-request wall p50/p95 0.294/0.302 s (2,463 prompt tokens, five completion tokens), excluding 4.319 s startup. A public AsyncOmni request also answered `Paris` on the RTX. An over-context request was rejected without truncation, followed by a correct recovery request. In a separate cancellation check, the owned server task started before abort; the worker and host-RAM reservation drained, and a fresh stage with the same artifact and new generation answered `Paris` without stale output. This GGUF/Vulkan profile is distinct from the earlier Spark 4B BF16 vLLM CUDA profile; the precisions, model sizes and workloads preclude a speedup comparison. There is no incremental token stream, automatic same-stage restart, broad token/quality parity, explicit GPU-VRAM peak admission, concurrency or sustained power/thermal qualification. | Validate streaming, larger quality and parity workloads, VRAM/load peaks, concurrent requests and sustained power; [Omni RTX raw evidence](../../../e2e_expansion_20260924/evidence/spark_omni_windows_rtx/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED; SUSTAINED_REPEAT_NO_COLLAPSE; PLAYBACK_JITTER_REMAINS | The historical complete-model CUDA stream slowed sharply around minute 26–27: all 17 requests in minutes 27–30 had simulated playback underruns and median RTF 1.331. A current-source, instrumented 30-minute repeat completed 1,232 serial medium requests within the analysis windows. Its final 119 requests had nearest-rank p50/p95 RTF 0.185/0.203, complete wall 1.493/1.708 s, and three brief simulated underruns needing at most 26.6 ms extra prebuffer. The late collapse did not recur; device-wide temperature p50/p95 was 68/71 C and SM clock 1,672/1,875 MHz in the final window, without an identifiable thermal transition. The repeat had 69 simulated underrun requests in the 30-minute windows, so underrun-free playback is not qualified. Run conditions were not controlled across repeats and the original cause is unknown. | Isolate playback jitter and original late-collapse conditions; validate speech quality, interruption/recovery and sustained thermal behavior; [both raw runs and telemetry analysis](../../../e2e_expansion_20260924/evidence/qwen_tts_sustained_drift/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The Omni pipeline binding now exists and passed scoped text/image cases in WSL, but no native Windows Omni run has verified the NVFP4 checkpoint, memory admission or modality behavior. | Run the same checkpoint and explicit context on native Windows, then broaden quality and profiling; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | NVFP4_POST_LOAD_PTX_STARTUP_FAILED; NOT_E2E_PROFILEABLE | The pinned NVFP4 checkpoint passed scoped text/image cases in WSL, but two native Windows attempts did not reach inference. In the captured retry, all three safetensors shards loaded; vLLM 0.29.0+cu134 then failed in NVFP4 Marlin scale permutation during post-load processing with `CUDA error: the provided PTX was compiled with an unsupported toolchain`. The installed RTX driver 610.71 reports CUDA UMD 13.3, while the local CUDA toolkit/wheel use 13.4. NVIDIA documents PTX JIT limitations across that driver gap; this is a likely runtime compatibility cause, not an A/B-proven driver fix or a model-wide unsupported conclusion. No native Windows text/image output, admission or latency is qualified. | Validate an R615-or-later compatible driver or a wheel/toolchain supported by R610, then rerun the same 512-token text/image workload and broaden quality, memory and profiling; [Windows failure evidence](../../../e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md), [WSL pass](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed native Windows three-stage BF16 text, one synthetic image, and one synthetic audio input to text+speech in separate constrained plans with 5 GiB thinker host offload and explicit 2,048/512-token limits. Text and all 32 thinker token IDs matched WSL CUDA; a 5.2 s WAV was nonzero. The red-square image produced the same description and 27 thinker token IDs as WSL plus a nonzero 5.2 s WAV in 17.15 s. The 2 s/440 Hz tone produced the same whistling description and nine IDs as WSL plus a nonzero 2.16 s WAV in 8.26 s. The initial text run needed 900/600 s startup limits after a 300 s timeout; a separate 1-warmup/20-measured serial text run had request wall p50 18.38 s/p95 18.73 s, with sampled host available RAM as low as 0.78 GB and pagefile use up to 11.17 GB. These synthetic inputs do not establish general image/audio or speech quality. | Check real image/audio suites, video input, speech intelligibility/alignment, concurrency, interruption, loading-peak admission and sustained thermal/power behavior before broader qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI_CUDA; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted native Windows Omni whole-policy graph stage with BF16 policy and Cosmos encoder on the RTX 5090 Laptop GPU. After one warmup, 20/20 serial patterned synthetic requests returned identical finite float32 [1,50,32] actions with terminal events; nearest-rank complete-request wall p50/p95 was 0.746/0.806 s, excluding 147.77 s startup. The 16 GiB host-RAM and 16 GiB CUDA demands were within explicit 18/21 GiB ceilings checked against available memory. Actual CUDA placement was verified, PyTorch peak reserved VRAM reached 7.86 GB, and an 8 GiB VRAM plan was refused before worker launch. In-flight cancellation and shutdown released both ledgers with no stale output; a separate public AsyncOmni.generate request returned the same action hash. Relative to the separate native CPU patterned output, action relative L2 difference was 2.340% with no task tolerance; native-vs-WSL CUDA relative L2 was 1.555%. The earlier one-forward zero-input direct CUDA run was a different experiment. This is synthetic action-policy integration, not robot-task E2E; units/order/step time remain unverified and control_ready=false. | Validate real observations/reference actions and physical action semantics, post-cancel restart, concurrent admission, loading peak, deadlines and sustained latency/power/thermal behavior; [Omni Windows CUDA evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_cuda/README.md), [earlier direct result](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST; RESTART_AFTER_INFLIGHT_CANCEL_PASS | The same pinned Spark-X2.5-1.7B Q4_K_M GGUF completed text prefill/decode through a bounded Omni whole-session stage with llama.cpp Vulkan1. The log identifies Radeon 890M and 29/29 layers offloaded there, rather than RTX 5090; an intentionally wrong device name was refused. `Paris` and `1511` passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 2.10/2.14 s. A public AsyncOmni request passed; an oversized context was rejected with a later correct request; in-flight cancellation produced no stale output and released the ledger. The prior standalone run was 2.18/2.20 s. A separate confirmed server-start cancellation of the 120-item request terminated the Vulkan1 worker, left no stale output or reservation, and a fresh StageRuntime returned Paris under a new worker generation with Radeon 890M placement again verified. This is fresh-stage recovery, not same-session continuation. No streaming, GPU-memory peak or sustained power/thermal check is claimed. | Qualify public configuration, broader text/token quality, streaming, concurrent recovery, shared-RAM/GPU peak and sustained use; [restart evidence](../../../e2e_expansion_20260924/evidence/spark_omni_windows_restart/README.md), [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
|
||||
@@ -329,6 +329,8 @@ The 2026-09-23 [Spark GGUF Omni follow-up](../../benchmarks/edge_harness/results
|
||||
|
||||
The [native Windows RTX Spark GGUF follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/spark_omni_windows_rtx/README.md) runs that same 1.7B Q4_K_M artifact through an Omni whole-session stage on RTX Vulkan0, with 29/29 layers offloaded. Two named requests, 20/20 serial inventory requests, a public `AsyncOmni` request, context refusal and fresh-stage restart after confirmed in-flight cancellation passed. Its 0.294/0.302 s complete-request p50/p95 is specific to the 2,463-prompt-token/five-output-token serial workload and cannot be compared directly to the earlier Spark 4B BF16 vLLM CUDA results. Token streaming, broad quality, VRAM/load peaks, concurrency and sustained power remain open; this does not establish mobile M1 support.
|
||||
|
||||
A [native Windows Qwen3.8-27B NVFP4 attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md) did not reach inference despite loading all three weight shards. The installed Windows vLLM 0.29+cu134 wheel failed during Marlin post-load scale permutation with an unsupported PTX toolchain error under RTX driver 610.71/CUDA UMD 13.3. NVIDIA's PTX compatibility guidance makes a driver/toolchain mismatch likely; an upgraded-driver or compatible-wheel control is still needed before rerunning the 512-token text/image cases. This failure constrains the tested Windows route, not the previously passed WSL checkpoint path.
|
||||
|
||||
The [Qwen3-TTS Omni hybrid follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_hybrid/README.md) adds one complete-request stage for the native Windows HX370 CPU+Radeon 890M route. A pinned Q8_0 talker and F16 codec produced two named PCM outputs exactly matching the prior standalone backend; one warmup plus 20 serial requests measured 3.298/3.336 s p50/p95 for 2.56 s of audio. Public `AsyncOmni` audio, cancellation drain and explicit 8 GiB host admission passed. A separate [in-flight cancellation audit](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_omni_windows_restart/README.md) confirmed that the request reached server prefill before cancellation; a fresh stage with the same artifacts and hybrid placement emitted the identical PCM under a new worker generation, without stale output or a retained reservation. This is progress toward M2 integration, not M2 acceptance: RTF remains above 1, and playable chunk streaming, same-session continuation, loading/iGPU peaks, broader voice quality and 30-minute thermal behavior remain open. No NPU participated.
|
||||
|
||||
The [WSL CPU Qwen3-TTS Omni follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_omni_wsl_cpu/README.md)
|
||||
|
||||
Reference in new issue
Block a user