From bf42971416bf3311bac567b683aeb8fc297227af Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 24 Sep 2026 17:26:14 +0800 Subject: [PATCH] Verify public Qwen3.8 FP8 requests and abort recovery --- .../experiments/audit_qwen38_windows_fp8.py | 39 ++- .../experiments/probe_qwen38_windows_async.py | 154 +++++++++++ .../qwen38_omni_windows_rtx/README.md | 4 +- .../fp8_audit_report.json | 44 +++- .../qwen38_fp8_public_async.driver.log | 239 +++++++++++++++++ .../qwen38_fp8_public_async.json | 58 +++++ .../qwen38_fp8_public_async_abort.driver.log | 244 ++++++++++++++++++ .../qwen38_fp8_public_async_abort.json | 77 ++++++ ...fp8_public_async_abort_recovery.driver.log | 240 +++++++++++++++++ ...wen38_fp8_public_async_abort_recovery.json | 88 +++++++ ...fp8_public_async_abort_terminal.driver.log | 244 ++++++++++++++++++ ...wen38_fp8_public_async_abort_terminal.json | 92 +++++++ ...38_fp8_public_async_abort_timed.driver.log | 244 ++++++++++++++++++ .../qwen38_fp8_public_async_abort_timed.json | 85 ++++++ .../evidence/summary/README.md | 2 +- docs/design/omni_edge_e2e_roadmap.md | 2 +- 16 files changed, 1849 insertions(+), 7 deletions(-) create mode 100644 benchmarks/edge_harness/experiments/probe_qwen38_windows_async.py create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort_recovery.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort_recovery.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort_terminal.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort_terminal.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort_timed.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async_abort_timed.json diff --git a/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py b/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py index e37ca0cb4..a09d18df3 100644 --- a/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py +++ b/benchmarks/edge_harness/experiments/audit_qwen38_windows_fp8.py @@ -42,6 +42,16 @@ def main() -> None: "qwen38_image_fp8_triton_ntfs_attempt1.driver.log", "qwen38_fp8_triton_ntfs_profile20.json", "qwen38_fp8_triton_ntfs_profile20.driver.log", + "qwen38_fp8_public_async.json", + "qwen38_fp8_public_async.driver.log", + "qwen38_fp8_public_async_abort.json", + "qwen38_fp8_public_async_abort.driver.log", + "qwen38_fp8_public_async_abort_timed.json", + "qwen38_fp8_public_async_abort_timed.driver.log", + "qwen38_fp8_public_async_abort_terminal.json", + "qwen38_fp8_public_async_abort_terminal.driver.log", + "qwen38_fp8_public_async_abort_recovery.json", + "qwen38_fp8_public_async_abort_recovery.driver.log", ] paths = {name: root / name for name in names} reports = {name: json.loads(paths[name].read_text(encoding="utf-8")) for name in names if name.endswith(".json")} @@ -82,6 +92,31 @@ def main() -> None: raise ValueError(f"successful log lacks enough KV cache for one request: {name}") if profile["text_output"] != ["ready"]: raise ValueError("text warmup answer changed") + public = reports["qwen38_fp8_public_async.json"] + recovery = reports["qwen38_fp8_public_async_abort_recovery.json"] + for report in (public, recovery): + if report["status"] != "completed" or report["entrypoint"] != "AsyncOmni.generate": + raise ValueError("public AsyncOmni request did not complete") + cases = {case["kind"]: case for case in report["cases"]} + if set(cases) != {"text", "image"} or any( + cases[kind]["outputs"][-1]["text"] != expected + or ("finished" in cases[kind]["outputs"][-1] and not cases[kind]["outputs"][-1]["finished"]) + for kind, expected in (("text", "ready"), ("image", "Red")) + ): + raise ValueError("public text/image final output changed") + if any(reports[name]["status"] != "failed" for name in ( + "qwen38_fp8_public_async_abort.json", + "qwen38_fp8_public_async_abort_timed.json", + "qwen38_fp8_public_async_abort_terminal.json", + )): + raise ValueError("retained abort-probe assertion failures are missing") + abort = recovery["abort_check"] + if (not abort["task_active_before_abort"] or abort["remaining_request_states"] != 0 + or abort["fresh_outputs"][-1] != "ready"): + raise ValueError("abort did not clear state and recover") + terminal = [row for row in abort["emissions"] if row["at_monotonic"] > abort["abort_completed_monotonic"]] + if len(terminal) != 1 or terminal[0]["finish_reason"] != "abort" or not terminal[0]["finished"]: + raise ValueError("abort did not emit one terminal marker") timing = {} for kind, expected in (("image", "Red"), ("text", "ready")): rows = [row for row in profile["profile"] if row["kind"] == kind] @@ -92,7 +127,7 @@ def main() -> None: "nearest_rank_p95_s": percentile(values, 0.95), "min_s": min(values), "max_s": max(values)} result = { "status": "audited_scoped_e2e", - "evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX; no long context or broad quality claim", + "evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX, plus public AsyncOmni and same-engine abort/recovery; no long context or broad quality claim", "artifact": "Qwen/Qwen3.8-27B-FP8", "checkpoint_revision": REVISION, "host": "Windows 11 build 26200; Ryzen AI 9 HX 370; RTX 5090 Laptop; driver 610.71", @@ -105,7 +140,7 @@ def main() -> None: "The FP8 checkpoint and Triton backend are distinct from the NVFP4 Marlin failure.", "The image is a synthetic 96x96 red square and the text prompt is a single-word response; no broad multimodal or long-context quality claim follows.", "The 12 GiB host offload and 0.80 GPU-memory cap are declared settings, not measured loading-peak admission or a sampled power profile.", - "Cancellation, public AsyncOmni, concurrency, video, sustained power/thermal behavior and task quality remain unverified for this native-Windows route.", + "Cancellation used a terminal abort marker and a fresh request in the same engine; server-started cancellation, separate stage restart, concurrency, video, sustained power/thermal behavior and task quality remain unverified.", ], } (root / "fp8_audit_report.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") diff --git a/benchmarks/edge_harness/experiments/probe_qwen38_windows_async.py b/benchmarks/edge_harness/experiments/probe_qwen38_windows_async.py new file mode 100644 index 000000000..8294c19e4 --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_qwen38_windows_async.py @@ -0,0 +1,154 @@ +"""Exercise public AsyncOmni text and image requests with the pinned FP8 model.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import time +import traceback +from pathlib import Path + + +async def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--abort-check", action="store_true") + args = parser.parse_args() + os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") + report = {"model": args.model, "entrypoint": "AsyncOmni.generate", "started": time.time()} + engine = None + try: + from PIL import Image + from transformers import AutoProcessor + from vllm import SamplingParams + + from vllm_omni.entrypoints.async_omni import AsyncOmni + from vllm_omni.windows.aio import install_selector_policy + + install_selector_policy() + processor = AutoProcessor.from_pretrained(args.model, local_files_only=True) + image = Image.new("RGB", (96, 96), (255, 0, 0)) + sampling = SamplingParams(temperature=0, max_tokens=16) + config = { + "model": args.model, + "language_model_only": False, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": True, + "gpu_memory_utilization": 0.80, + "cpu_offload_gb": 12, + "linear_backend": "triton", + "dtype": "bfloat16", + "stage_init_timeout": 180, + "init_timeout": 240, + } + report["config"] = config + start = time.perf_counter() + engine = AsyncOmni(**config) + report["startup_s"] = time.perf_counter() - start + cases = ( + ("text", "Reply with the single word ready.", None, "ready"), + ("image", "What is the dominant color in the image? Answer in one word.", image, "Red"), + ) + report["cases"] = [] + for kind, question, visual, expected in cases: + content = question if visual is None else [ + {"type": "image", "image": visual}, {"type": "text", "text": question} + ] + prompt = processor.apply_chat_template( + [{"role": "user", "content": content}], + tokenize=False, add_generation_prompt=True, enable_thinking=False, + ) + request = {"prompt": prompt} + if visual is not None: + request["multi_modal_data"] = {"image": visual} + outputs = [] + start = time.perf_counter() + async for output in engine.generate(request, sampling, request_id=f"public-qwen38-{kind}"): + outputs.append({"request_id": output.request_id, "error": output.error, + "text": output.outputs[0].text if output.outputs else None, + "finished": output.finished, "stage_id": output.stage_id}) + row = {"kind": kind, "wall_s": time.perf_counter() - start, + "outputs": outputs, "expected": expected} + report["cases"].append(row) + if not outputs or any(item["error"] for item in outputs) or outputs[-1]["text"].strip() != expected: + raise RuntimeError(f"public {kind} request did not complete with {expected!r}") + if args.abort_check: + cancel_id = "public-qwen38-cancel" + long_prompt = processor.apply_chat_template( + [{"role": "user", "content": "Write the integers from 1 through 200, one per line, with no explanation."}], + tokenize=False, add_generation_prompt=True, enable_thinking=False, + ) + emitted = [] + + async def collect_long() -> None: + async for output in engine.generate( + {"prompt": long_prompt}, SamplingParams(temperature=0, max_tokens=256), request_id=cancel_id + ): + emitted.append({"at_monotonic": time.perf_counter(), + "text": output.outputs[0].text if output.outputs else None, + "finished": output.finished, + "finish_reason": output.outputs[0].finish_reason if output.outputs else None}) + + task = asyncio.create_task(collect_long()) + deadline = time.monotonic() + 20 + while not any(state.external_request_id == cancel_id for state in engine.request_states.values()): + if task.done(): + raise RuntimeError("long request completed before cancellation check") + if time.monotonic() >= deadline: + raise TimeoutError("long request was not admitted") + await asyncio.sleep(0.02) + await asyncio.sleep(2) + if task.done(): + raise RuntimeError("long request completed during cancellation wait") + before_abort = len(emitted) + abort_started = time.perf_counter() + await engine.abort(cancel_id) + abort_completed = time.perf_counter() + error = None + try: + await asyncio.wait_for(task, timeout=30) + except BaseException as exc: + error = f"{type(exc).__name__}: {exc}" + await asyncio.sleep(0.3) + report["abort_check"] = { + "request_id": cancel_id, "submitted": True, "task_active_before_abort": True, + "emitted_before_abort": before_abort, "emissions": emitted, + "abort_started_monotonic": abort_started, + "abort_completed_monotonic": abort_completed, + "task_error": error, "remaining_request_states": len(engine.request_states), + } + after_abort = [row for row in emitted if row["at_monotonic"] > abort_completed] + if len(after_abort) != 1 or after_abort[0]["finish_reason"] != "abort" or not after_abort[0]["finished"]: + raise RuntimeError("abort did not produce exactly one terminal abort marker") + fresh_outputs = [] + async for output in engine.generate( + {"prompt": processor.apply_chat_template( + [{"role": "user", "content": "Reply with the single word ready."}], + tokenize=False, add_generation_prompt=True, enable_thinking=False, + )}, sampling, request_id="public-qwen38-after-abort" + ): + fresh_outputs.append(output.outputs[0].text if output.outputs else None) + report["abort_check"]["fresh_outputs"] = fresh_outputs + if not fresh_outputs or fresh_outputs[-1] != "ready": + raise RuntimeError("fresh request after abort failed") + report["status"] = "completed" + except Exception as exc: + report.update(status="failed", error=repr(exc), traceback=traceback.format_exc()) + traceback.print_exc() + finally: + if engine is not None: + engine.shutdown() + report["ended"] = time.time() + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + if report["status"] != "completed": + raise SystemExit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md index 7f12a3d65..7d0d38deb 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md @@ -10,7 +10,9 @@ The checkpoint was copied from its WSL model root to native NTFS for the success The retained failures delimit the fix. The [default FP8/CUTLASS load](qwen38_image_fp8_attempt1.driver.log) reached a model startup operation but the Windows wheel had no compiled `cutlass_scaled_mm` for SM120. A [small Triton FP8 block probe](fp8_triton_probe.json) passed on SM120 with the expected value 128. The [first full Triton attempt](qwen38_image_fp8_triton_attempt1.json) exposed Omni's missing structured owner for `linear_backend`; the current config projection now carries that vLLM option, with a focused test. The [next attempt](qwen38_image_fp8_triton_attempt2.driver.log) loaded all 66 weights but vLLM refused the 0.75 GPU-memory cap: 0.22 GiB available for KV versus 0.33 GiB required for a 512-token request. Raising the cap to 0.80 admitted the successful run. The [FP8 auditor](fp8_audit_report.json) checks all raw reports, failure strings, successful kernel/load/cache evidence, per-request outputs, sample counts and percentile math. -This advances the native Windows RTX × Qwen3.8 cell to a scoped complete text/image path. It does not establish a qualified long context, video path, reference-quality suite, explicit Omni host/VRAM loading-peak admission, cancellation/recovery, public AsyncOmni request, concurrency scaling or sustained power and thermal behavior. The NVFP4 failure remains a separate unresolved artifact/runtime combination. +A separate [public AsyncOmni request](qwen38_fp8_public_async.json) returned `ready` for text and `Red` for the same synthetic image. Both public calls emitted cumulative text once with `finished=False` and again with `finished=True` in the later [cancellation/recovery run](qwen38_fp8_public_async_abort_recovery.json); consumers must use the completion flag rather than treating both as independent answers. The recovery run submitted a long 256-token text request, observed the task active before abort, then received exactly one finished output with `finish_reason=abort` after abort completed. The request state cleared and a fresh text request on the **same engine** returned `ready`. The earlier [count-only](qwen38_fp8_public_async_abort.json), [timestamped](qwen38_fp8_public_async_abort_timed.json) and [empty-only terminal](qwen38_fp8_public_async_abort_terminal.json) probes failed their own assertions because they treated Omni's documented terminal abort marker, which may retain partial text, as a late model answer. Their raw records are retained. This validates public request and same-engine state recovery but does not prove a server-started cancellation timing boundary or separate-stage restart. + +This advances the native Windows RTX × Qwen3.8 cell to a scoped complete text/image path with public AsyncOmni and same-engine abort/recovery. It does not establish a qualified long context, video path, reference-quality suite, explicit Omni host/VRAM loading-peak admission, server-started cancellation boundary, concurrency scaling or sustained power and thermal behavior. The NVFP4 failure remains a separate unresolved artifact/runtime combination. The pinned Qwen3.8-27B NVFP4 checkpoint, revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c`, was attempted through the current Omni/vLLM 0.29 one-stage pipeline on native Windows 11 build 26200. The bounded image-to-text case used a synthetic 96×96 red image, 512-token context, one sequence, 16 output tokens, eager execution, 0.85 GPU-memory utilization and an explicit 2 GiB CPU offload. This is the same scoped case that [completed on WSL RTX](../../../e2e_recovery_20260923/README.md); Windows **did not reach inference**. No output or latency is claimed for this cell. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json index f9c51f802..3b3b5c41e 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/fp8_audit_report.json @@ -1,6 +1,6 @@ { "status": "audited_scoped_e2e", - "evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX; no long context or broad quality claim", + "evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX, plus public AsyncOmni and same-engine abort/recovery; no long context or broad quality claim", "artifact": "Qwen/Qwen3.8-27B-FP8", "checkpoint_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a", "host": "Windows 11 build 26200; Ryzen AI 9 HX 370; RTX 5090 Laptop; driver 610.71", @@ -90,12 +90,52 @@ "qwen38_fp8_triton_ntfs_profile20.driver.log": { "bytes": 38663, "sha256": "1a324158efe8d6a233eca69cbd4f5f8828e1cf786a976fc9300cd1a2dc56368a" + }, + "qwen38_fp8_public_async.json": { + "bytes": 1340, + "sha256": "8b235e6014dfcc5f97421c9abf661082e86e0320201adcbe07727ffb511550c4" + }, + "qwen38_fp8_public_async.driver.log": { + "bytes": 29105, + "sha256": "fbc87851748d7513b2e42f0648bf90f3c6d3a01b1dc4da0ad445c2512e7eeec1" + }, + "qwen38_fp8_public_async_abort.json": { + "bytes": 2207, + "sha256": "6de9d799d45d42ed7b21f64a2fc93a0ea19bd60f2bfce89b751c21ccd663187f" + }, + "qwen38_fp8_public_async_abort.driver.log": { + "bytes": 29525, + "sha256": "1f3477638b3cafdd5013eb922916db2e63e42ff04c2c0a692540e58c1458ed7f" + }, + "qwen38_fp8_public_async_abort_timed.json": { + "bytes": 2433, + "sha256": "406725b4097829e485bcc42026e72480cdf23e51cbb9080d0ce0ff71ab7c30fe" + }, + "qwen38_fp8_public_async_abort_timed.driver.log": { + "bytes": 29541, + "sha256": "e57f76b74e7f70140c3dc99512e415860b3485fc4403821eb01d3b6a98a551d1" + }, + "qwen38_fp8_public_async_abort_terminal.json": { + "bytes": 2619, + "sha256": "cd7be80314b122e4ab03fd9bf53f1bedf6dc849eee46a44851ae2ef7bcb1c282" + }, + "qwen38_fp8_public_async_abort_terminal.driver.log": { + "bytes": 29555, + "sha256": "b62ac62b85e347045e457a8626f132cd9ff137b459bae98ce5afa269ef91d2ce" + }, + "qwen38_fp8_public_async_abort_recovery.json": { + "bytes": 2086, + "sha256": "12aecbd9eed84c114976ab32c9a796d17500a47947f37a92756737dc0310f606" + }, + "qwen38_fp8_public_async_abort_recovery.driver.log": { + "bytes": 29216, + "sha256": "487c16ef1d3b2bdedbcca6531fed24c318f16873469bb790dfbef3456e52c515" } }, "limits": [ "The FP8 checkpoint and Triton backend are distinct from the NVFP4 Marlin failure.", "The image is a synthetic 96x96 red square and the text prompt is a single-word response; no broad multimodal or long-context quality claim follows.", "The 12 GiB host offload and 0.80 GPU-memory cap are declared settings, not measured loading-peak admission or a sampled power profile.", - "Cancellation, public AsyncOmni, concurrency, video, sustained power/thermal behavior and task quality remain unverified for this native-Windows route." + "Cancellation used a terminal abort marker and a fresh request in the same engine; server-started cancellation, separate stage restart, concurrency, video, sustained power/thermal behavior and task quality remain unverified." ] } diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async.driver.log new file mode 100644 index 000000000..6601cf686 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/qwen38_fp8_public_async.driver.log @@ -0,0 +1,239 @@ +INFO 09-24 17:03:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 17:03:56 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 17:04:00 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8 +INFO 09-24 17:04:00 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8 +INFO 09-24 17:04:00 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-24 17:04:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 17:04:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-24 17:04:01 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-24 17:04:01 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration +INFO 09-24 17:04:01 [model.py:2021] Using max model len 512 +WARNING 09-24 17:04:01 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-24 17:04:07 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-24 17:04:07 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled +WARNING 09-24 17:04:07 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0. +WARNING 09-24 17:04:07 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-24 17:04:07 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-24 17:04:07 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']) +INFO 09-24 17:04:07 [vllm.py:1585] Cudagraph is disabled under eager mode +INFO 09-24 17:04:07 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-24 17:04:07 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group +INFO 09-24 17:04:21 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 17:04:21 [patch.py:510] inductor factorable-divisibility patch: installed. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:25 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:27 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_f0d0c4be40c747db8c77b30c0df5f07c backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:27 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:27 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:27 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:27 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=50132) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:34 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:34 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:34 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:34 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:35 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:35 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:35 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:44 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:45 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 5.14 GiB. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:45 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (5.14 GiB). +(StageEngineCoreProc_stage0_replica0 pid=50132) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:13 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:15 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [base.py:148] Available KV cache memory: 1.07 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [kv_cache_utils.py:2032] GPU KV cache size: 1,609 tokens, Maximum concurrency for 512 tokens per request: 3.14x +(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:23 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=50132) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:49 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_3a2cbf0703ea4b12bd6133332d341d3d backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:49 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:49 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:50 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:50 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=43940) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:55 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:55 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:55 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:55 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:57 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:57 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:04 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:04 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 7.52 GiB. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:04 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (7.52 GiB). +(StageEngineCoreProc_stage0_replica0 pid=43940) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:30 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:30 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:31 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:31 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:31 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:31 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:32 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:32 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [base.py:148] Available KV cache memory: 1.56 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [kv_cache_utils.py:2032] GPU KV cache size: 2,340 tokens, Maximum concurrency for 512 tokens per request: 4.57x +(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:41 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=43940) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:23 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_6823e98dd89849b5863ea6e9a102cfa0 backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:23 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:24 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:24 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:24 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=57020) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:31 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:31 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:36 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:37 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 11.61 GiB. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:37 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (11.61 GiB). +(StageEngineCoreProc_stage0_replica0 pid=57020) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:03 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [base.py:148] Available KV cache memory: 1.5 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [kv_cache_utils.py:2032] GPU KV cache size: 2,267 tokens, Maximum concurrency for 512 tokens per request: 4.43x +(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:11 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=57020) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:07 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_2342da557dc44b13b84988c9a0316f0b backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:07 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:07 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:07 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:07 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=46532) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:12 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:12 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:12 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:12 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:14 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:14 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:21 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:22 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 5.53 GiB. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:22 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (5.53 GiB). +(StageEngineCoreProc_stage0_replica0 pid=46532) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:48 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:48 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:49 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:49 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [base.py:148] Available KV cache memory: 1.48 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [kv_cache_utils.py:2032] GPU KV cache size: 2,194 tokens, Maximum concurrency for 512 tokens per request: 4.29x +(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:59 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=46532) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton') +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:55 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_77a2e6f7769b4220a64e54db825edfb6 backend=nccl +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:11:55 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:55 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:11:56 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:56 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0. +(StageEngineCoreProc_stage0_replica0 pid=14700) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [base.py:138] Offloader set to UVAOffloader +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8... +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [uva.py:65] Total CPU offloaded parameters: 0.86 +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128). +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION']. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [flash_attn.py:897] Using FlashAttention version 2 +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:10 [uva.py:65] Total CPU offloaded parameters: 12.13 +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:10 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 7.56 GiB. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:10 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (7.56 GiB). +(StageEngineCoreProc_stage0_replica0 pid=14700) +Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00= mamba page size. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:34 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:34 [utils.py:306] Using LBNHC KV cache layout. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:35 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache. +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:35 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [base.py:148] Available KV cache memory: 1.6 GiB (device-level profiling) +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [kv_cache_utils.py:2032] GPU KV cache size: 2,413 tokens, Maximum concurrency for 512 tokens per request: 4.71x +(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:45 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED. +(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [kernel_warmup.py:124] JIT kernel warmup starting. +(StageEngineCoreProc_stage0_replica0 pid=14700) +JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00