Verify public Qwen3.8 FP8 requests and abort recovery
This commit is contained in:
tzhouam committed
1 parent
32a81111bd
commit
bf42971416
16 files changed
+1849
-7
No files matched your search
@@ -42,6 +42,16 @@ def main() -> None:
|
||||
"qwen38_image_fp8_triton_ntfs_attempt1.driver.log",
|
||||
"qwen38_fp8_triton_ntfs_profile20.json",
|
||||
"qwen38_fp8_triton_ntfs_profile20.driver.log",
|
||||
"qwen38_fp8_public_async.json",
|
||||
"qwen38_fp8_public_async.driver.log",
|
||||
"qwen38_fp8_public_async_abort.json",
|
||||
"qwen38_fp8_public_async_abort.driver.log",
|
||||
"qwen38_fp8_public_async_abort_timed.json",
|
||||
"qwen38_fp8_public_async_abort_timed.driver.log",
|
||||
"qwen38_fp8_public_async_abort_terminal.json",
|
||||
"qwen38_fp8_public_async_abort_terminal.driver.log",
|
||||
"qwen38_fp8_public_async_abort_recovery.json",
|
||||
"qwen38_fp8_public_async_abort_recovery.driver.log",
|
||||
]
|
||||
paths = {name: root / name for name in names}
|
||||
reports = {name: json.loads(paths[name].read_text(encoding="utf-8")) for name in names if name.endswith(".json")}
|
||||
@@ -82,6 +92,31 @@ def main() -> None:
|
||||
raise ValueError(f"successful log lacks enough KV cache for one request: {name}")
|
||||
if profile["text_output"] != ["ready"]:
|
||||
raise ValueError("text warmup answer changed")
|
||||
public = reports["qwen38_fp8_public_async.json"]
|
||||
recovery = reports["qwen38_fp8_public_async_abort_recovery.json"]
|
||||
for report in (public, recovery):
|
||||
if report["status"] != "completed" or report["entrypoint"] != "AsyncOmni.generate":
|
||||
raise ValueError("public AsyncOmni request did not complete")
|
||||
cases = {case["kind"]: case for case in report["cases"]}
|
||||
if set(cases) != {"text", "image"} or any(
|
||||
cases[kind]["outputs"][-1]["text"] != expected
|
||||
or ("finished" in cases[kind]["outputs"][-1] and not cases[kind]["outputs"][-1]["finished"])
|
||||
for kind, expected in (("text", "ready"), ("image", "Red"))
|
||||
):
|
||||
raise ValueError("public text/image final output changed")
|
||||
if any(reports[name]["status"] != "failed" for name in (
|
||||
"qwen38_fp8_public_async_abort.json",
|
||||
"qwen38_fp8_public_async_abort_timed.json",
|
||||
"qwen38_fp8_public_async_abort_terminal.json",
|
||||
)):
|
||||
raise ValueError("retained abort-probe assertion failures are missing")
|
||||
abort = recovery["abort_check"]
|
||||
if (not abort["task_active_before_abort"] or abort["remaining_request_states"] != 0
|
||||
or abort["fresh_outputs"][-1] != "ready"):
|
||||
raise ValueError("abort did not clear state and recover")
|
||||
terminal = [row for row in abort["emissions"] if row["at_monotonic"] > abort["abort_completed_monotonic"]]
|
||||
if len(terminal) != 1 or terminal[0]["finish_reason"] != "abort" or not terminal[0]["finished"]:
|
||||
raise ValueError("abort did not emit one terminal marker")
|
||||
timing = {}
|
||||
for kind, expected in (("image", "Red"), ("text", "ready")):
|
||||
rows = [row for row in profile["profile"] if row["kind"] == kind]
|
||||
@@ -92,7 +127,7 @@ def main() -> None:
|
||||
"nearest_rank_p95_s": percentile(values, 0.95), "min_s": min(values), "max_s": max(values)}
|
||||
result = {
|
||||
"status": "audited_scoped_e2e",
|
||||
"evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX; no long context or broad quality claim",
|
||||
"evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX, plus public AsyncOmni and same-engine abort/recovery; no long context or broad quality claim",
|
||||
"artifact": "Qwen/Qwen3.8-27B-FP8",
|
||||
"checkpoint_revision": REVISION,
|
||||
"host": "Windows 11 build 26200; Ryzen AI 9 HX 370; RTX 5090 Laptop; driver 610.71",
|
||||
@@ -105,7 +140,7 @@ def main() -> None:
|
||||
"The FP8 checkpoint and Triton backend are distinct from the NVFP4 Marlin failure.",
|
||||
"The image is a synthetic 96x96 red square and the text prompt is a single-word response; no broad multimodal or long-context quality claim follows.",
|
||||
"The 12 GiB host offload and 0.80 GPU-memory cap are declared settings, not measured loading-peak admission or a sampled power profile.",
|
||||
"Cancellation, public AsyncOmni, concurrency, video, sustained power/thermal behavior and task quality remain unverified for this native-Windows route.",
|
||||
"Cancellation used a terminal abort marker and a fresh request in the same engine; server-started cancellation, separate stage restart, concurrency, video, sustained power/thermal behavior and task quality remain unverified.",
|
||||
],
|
||||
}
|
||||
(root / "fp8_audit_report.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Exercise public AsyncOmni text and image requests with the pinned FP8 model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--abort-check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
report = {"model": args.model, "entrypoint": "AsyncOmni.generate", "started": time.time()}
|
||||
engine = None
|
||||
try:
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor
|
||||
from vllm import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.windows.aio import install_selector_policy
|
||||
|
||||
install_selector_policy()
|
||||
processor = AutoProcessor.from_pretrained(args.model, local_files_only=True)
|
||||
image = Image.new("RGB", (96, 96), (255, 0, 0))
|
||||
sampling = SamplingParams(temperature=0, max_tokens=16)
|
||||
config = {
|
||||
"model": args.model,
|
||||
"language_model_only": False,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": True,
|
||||
"gpu_memory_utilization": 0.80,
|
||||
"cpu_offload_gb": 12,
|
||||
"linear_backend": "triton",
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180,
|
||||
"init_timeout": 240,
|
||||
}
|
||||
report["config"] = config
|
||||
start = time.perf_counter()
|
||||
engine = AsyncOmni(**config)
|
||||
report["startup_s"] = time.perf_counter() - start
|
||||
cases = (
|
||||
("text", "Reply with the single word ready.", None, "ready"),
|
||||
("image", "What is the dominant color in the image? Answer in one word.", image, "Red"),
|
||||
)
|
||||
report["cases"] = []
|
||||
for kind, question, visual, expected in cases:
|
||||
content = question if visual is None else [
|
||||
{"type": "image", "image": visual}, {"type": "text", "text": question}
|
||||
]
|
||||
prompt = processor.apply_chat_template(
|
||||
[{"role": "user", "content": content}],
|
||||
tokenize=False, add_generation_prompt=True, enable_thinking=False,
|
||||
)
|
||||
request = {"prompt": prompt}
|
||||
if visual is not None:
|
||||
request["multi_modal_data"] = {"image": visual}
|
||||
outputs = []
|
||||
start = time.perf_counter()
|
||||
async for output in engine.generate(request, sampling, request_id=f"public-qwen38-{kind}"):
|
||||
outputs.append({"request_id": output.request_id, "error": output.error,
|
||||
"text": output.outputs[0].text if output.outputs else None,
|
||||
"finished": output.finished, "stage_id": output.stage_id})
|
||||
row = {"kind": kind, "wall_s": time.perf_counter() - start,
|
||||
"outputs": outputs, "expected": expected}
|
||||
report["cases"].append(row)
|
||||
if not outputs or any(item["error"] for item in outputs) or outputs[-1]["text"].strip() != expected:
|
||||
raise RuntimeError(f"public {kind} request did not complete with {expected!r}")
|
||||
if args.abort_check:
|
||||
cancel_id = "public-qwen38-cancel"
|
||||
long_prompt = processor.apply_chat_template(
|
||||
[{"role": "user", "content": "Write the integers from 1 through 200, one per line, with no explanation."}],
|
||||
tokenize=False, add_generation_prompt=True, enable_thinking=False,
|
||||
)
|
||||
emitted = []
|
||||
|
||||
async def collect_long() -> None:
|
||||
async for output in engine.generate(
|
||||
{"prompt": long_prompt}, SamplingParams(temperature=0, max_tokens=256), request_id=cancel_id
|
||||
):
|
||||
emitted.append({"at_monotonic": time.perf_counter(),
|
||||
"text": output.outputs[0].text if output.outputs else None,
|
||||
"finished": output.finished,
|
||||
"finish_reason": output.outputs[0].finish_reason if output.outputs else None})
|
||||
|
||||
task = asyncio.create_task(collect_long())
|
||||
deadline = time.monotonic() + 20
|
||||
while not any(state.external_request_id == cancel_id for state in engine.request_states.values()):
|
||||
if task.done():
|
||||
raise RuntimeError("long request completed before cancellation check")
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("long request was not admitted")
|
||||
await asyncio.sleep(0.02)
|
||||
await asyncio.sleep(2)
|
||||
if task.done():
|
||||
raise RuntimeError("long request completed during cancellation wait")
|
||||
before_abort = len(emitted)
|
||||
abort_started = time.perf_counter()
|
||||
await engine.abort(cancel_id)
|
||||
abort_completed = time.perf_counter()
|
||||
error = None
|
||||
try:
|
||||
await asyncio.wait_for(task, timeout=30)
|
||||
except BaseException as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
await asyncio.sleep(0.3)
|
||||
report["abort_check"] = {
|
||||
"request_id": cancel_id, "submitted": True, "task_active_before_abort": True,
|
||||
"emitted_before_abort": before_abort, "emissions": emitted,
|
||||
"abort_started_monotonic": abort_started,
|
||||
"abort_completed_monotonic": abort_completed,
|
||||
"task_error": error, "remaining_request_states": len(engine.request_states),
|
||||
}
|
||||
after_abort = [row for row in emitted if row["at_monotonic"] > abort_completed]
|
||||
if len(after_abort) != 1 or after_abort[0]["finish_reason"] != "abort" or not after_abort[0]["finished"]:
|
||||
raise RuntimeError("abort did not produce exactly one terminal abort marker")
|
||||
fresh_outputs = []
|
||||
async for output in engine.generate(
|
||||
{"prompt": processor.apply_chat_template(
|
||||
[{"role": "user", "content": "Reply with the single word ready."}],
|
||||
tokenize=False, add_generation_prompt=True, enable_thinking=False,
|
||||
)}, sampling, request_id="public-qwen38-after-abort"
|
||||
):
|
||||
fresh_outputs.append(output.outputs[0].text if output.outputs else None)
|
||||
report["abort_check"]["fresh_outputs"] = fresh_outputs
|
||||
if not fresh_outputs or fresh_outputs[-1] != "ready":
|
||||
raise RuntimeError("fresh request after abort failed")
|
||||
report["status"] = "completed"
|
||||
except Exception as exc:
|
||||
report.update(status="failed", error=repr(exc), traceback=traceback.format_exc())
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if engine is not None:
|
||||
engine.shutdown()
|
||||
report["ended"] = time.time()
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
if report["status"] != "completed":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+3
-1
@@ -10,7 +10,9 @@ The checkpoint was copied from its WSL model root to native NTFS for the success
|
||||
|
||||
The retained failures delimit the fix. The [default FP8/CUTLASS load](qwen38_image_fp8_attempt1.driver.log) reached a model startup operation but the Windows wheel had no compiled `cutlass_scaled_mm` for SM120. A [small Triton FP8 block probe](fp8_triton_probe.json) passed on SM120 with the expected value 128. The [first full Triton attempt](qwen38_image_fp8_triton_attempt1.json) exposed Omni's missing structured owner for `linear_backend`; the current config projection now carries that vLLM option, with a focused test. The [next attempt](qwen38_image_fp8_triton_attempt2.driver.log) loaded all 66 weights but vLLM refused the 0.75 GPU-memory cap: 0.22 GiB available for KV versus 0.33 GiB required for a 512-token request. Raising the cap to 0.80 admitted the successful run. The [FP8 auditor](fp8_audit_report.json) checks all raw reports, failure strings, successful kernel/load/cache evidence, per-request outputs, sample counts and percentile math.
|
||||
|
||||
This advances the native Windows RTX × Qwen3.8 cell to a scoped complete text/image path. It does not establish a qualified long context, video path, reference-quality suite, explicit Omni host/VRAM loading-peak admission, cancellation/recovery, public AsyncOmni request, concurrency scaling or sustained power and thermal behavior. The NVFP4 failure remains a separate unresolved artifact/runtime combination.
|
||||
A separate [public AsyncOmni request](qwen38_fp8_public_async.json) returned `ready` for text and `Red` for the same synthetic image. Both public calls emitted cumulative text once with `finished=False` and again with `finished=True` in the later [cancellation/recovery run](qwen38_fp8_public_async_abort_recovery.json); consumers must use the completion flag rather than treating both as independent answers. The recovery run submitted a long 256-token text request, observed the task active before abort, then received exactly one finished output with `finish_reason=abort` after abort completed. The request state cleared and a fresh text request on the **same engine** returned `ready`. The earlier [count-only](qwen38_fp8_public_async_abort.json), [timestamped](qwen38_fp8_public_async_abort_timed.json) and [empty-only terminal](qwen38_fp8_public_async_abort_terminal.json) probes failed their own assertions because they treated Omni's documented terminal abort marker, which may retain partial text, as a late model answer. Their raw records are retained. This validates public request and same-engine state recovery but does not prove a server-started cancellation timing boundary or separate-stage restart.
|
||||
|
||||
This advances the native Windows RTX × Qwen3.8 cell to a scoped complete text/image path with public AsyncOmni and same-engine abort/recovery. It does not establish a qualified long context, video path, reference-quality suite, explicit Omni host/VRAM loading-peak admission, server-started cancellation boundary, concurrency scaling or sustained power and thermal behavior. The NVFP4 failure remains a separate unresolved artifact/runtime combination.
|
||||
|
||||
The pinned Qwen3.8-27B NVFP4 checkpoint, revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c`, was attempted through the current Omni/vLLM 0.29 one-stage pipeline on native Windows 11 build 26200. The bounded image-to-text case used a synthetic 96×96 red image, 512-token context, one sequence, 16 output tokens, eager execution, 0.85 GPU-memory utilization and an explicit 2 GiB CPU offload. This is the same scoped case that [completed on WSL RTX](../../../e2e_recovery_20260923/README.md); Windows **did not reach inference**. No output or latency is claimed for this cell.
|
||||
|
||||
|
||||
+42
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"status": "audited_scoped_e2e",
|
||||
"evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX; no long context or broad quality claim",
|
||||
"evidence_depth": "P: complete text and synthetic-image requests on native Windows RTX, plus public AsyncOmni and same-engine abort/recovery; no long context or broad quality claim",
|
||||
"artifact": "Qwen/Qwen3.8-27B-FP8",
|
||||
"checkpoint_revision": "017b9c7af6b5689d5dd426a76e0bc077eb5ca20a",
|
||||
"host": "Windows 11 build 26200; Ryzen AI 9 HX 370; RTX 5090 Laptop; driver 610.71",
|
||||
@@ -90,12 +90,52 @@
|
||||
"qwen38_fp8_triton_ntfs_profile20.driver.log": {
|
||||
"bytes": 38663,
|
||||
"sha256": "1a324158efe8d6a233eca69cbd4f5f8828e1cf786a976fc9300cd1a2dc56368a"
|
||||
},
|
||||
"qwen38_fp8_public_async.json": {
|
||||
"bytes": 1340,
|
||||
"sha256": "8b235e6014dfcc5f97421c9abf661082e86e0320201adcbe07727ffb511550c4"
|
||||
},
|
||||
"qwen38_fp8_public_async.driver.log": {
|
||||
"bytes": 29105,
|
||||
"sha256": "fbc87851748d7513b2e42f0648bf90f3c6d3a01b1dc4da0ad445c2512e7eeec1"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort.json": {
|
||||
"bytes": 2207,
|
||||
"sha256": "6de9d799d45d42ed7b21f64a2fc93a0ea19bd60f2bfce89b751c21ccd663187f"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort.driver.log": {
|
||||
"bytes": 29525,
|
||||
"sha256": "1f3477638b3cafdd5013eb922916db2e63e42ff04c2c0a692540e58c1458ed7f"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort_timed.json": {
|
||||
"bytes": 2433,
|
||||
"sha256": "406725b4097829e485bcc42026e72480cdf23e51cbb9080d0ce0ff71ab7c30fe"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort_timed.driver.log": {
|
||||
"bytes": 29541,
|
||||
"sha256": "e57f76b74e7f70140c3dc99512e415860b3485fc4403821eb01d3b6a98a551d1"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort_terminal.json": {
|
||||
"bytes": 2619,
|
||||
"sha256": "cd7be80314b122e4ab03fd9bf53f1bedf6dc849eee46a44851ae2ef7bcb1c282"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort_terminal.driver.log": {
|
||||
"bytes": 29555,
|
||||
"sha256": "b62ac62b85e347045e457a8626f132cd9ff137b459bae98ce5afa269ef91d2ce"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort_recovery.json": {
|
||||
"bytes": 2086,
|
||||
"sha256": "12aecbd9eed84c114976ab32c9a796d17500a47947f37a92756737dc0310f606"
|
||||
},
|
||||
"qwen38_fp8_public_async_abort_recovery.driver.log": {
|
||||
"bytes": 29216,
|
||||
"sha256": "487c16ef1d3b2bdedbcca6531fed24c318f16873469bb790dfbef3456e52c515"
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"The FP8 checkpoint and Triton backend are distinct from the NVFP4 Marlin failure.",
|
||||
"The image is a synthetic 96x96 red square and the text prompt is a single-word response; no broad multimodal or long-context quality claim follows.",
|
||||
"The 12 GiB host offload and 0.80 GPU-memory cap are declared settings, not measured loading-peak admission or a sampled power profile.",
|
||||
"Cancellation, public AsyncOmni, concurrency, video, sustained power/thermal behavior and task quality remain unverified for this native-Windows route."
|
||||
"Cancellation used a terminal abort marker and a fresh request in the same engine; server-started cancellation, separate stage restart, concurrency, video, sustained power/thermal behavior and task quality remain unverified."
|
||||
]
|
||||
}
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
INFO 09-24 17:03:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:03:56 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 17:04:00 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:04:00 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:04:00 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-24 17:04:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 17:04:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-24 17:04:01 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-24 17:04:01 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration
|
||||
INFO 09-24 17:04:01 [model.py:2021] Using max model len 512
|
||||
WARNING 09-24 17:04:01 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-24 17:04:07 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-24 17:04:07 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled
|
||||
WARNING 09-24 17:04:07 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.
|
||||
WARNING 09-24 17:04:07 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-24 17:04:07 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-24 17:04:07 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-24 17:04:07 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:04:07 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-24 17:04:07 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-24 17:04:21 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:04:21 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:25 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:27 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_f0d0c4be40c747db8c77b30c0df5f07c backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:27 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:27 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:27 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:27 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:34 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:34 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:04:34 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:34 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:35 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:35 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:35 [uva.py:65] Total CPU offloaded parameters: 0.86
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:36 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:44 [uva.py:65] Total CPU offloaded parameters: 12.13
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:45 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 5.14 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:04:45 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (5.14 GiB).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 2% Completed | 1/66 [00:00<00:18, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 3% Completed | 2/66 [00:00<00:19, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 5% Completed | 3/66 [00:00<00:18, 3.40it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 4/66 [00:01<00:17, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 8% Completed | 5/66 [00:01<00:17, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 9% Completed | 6/66 [00:01<00:16, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 11% Completed | 7/66 [00:02<00:16, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 8/66 [00:02<00:16, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 14% Completed | 9/66 [00:02<00:15, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 15% Completed | 10/66 [00:02<00:15, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 17% Completed | 11/66 [00:03<00:15, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 18% Completed | 12/66 [00:03<00:14, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 20% Completed | 13/66 [00:03<00:14, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 21% Completed | 14/66 [00:03<00:14, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 23% Completed | 15/66 [00:04<00:14, 3.52it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 24% Completed | 16/66 [00:04<00:14, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 26% Completed | 17/66 [00:04<00:14, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 27% Completed | 18/66 [00:05<00:14, 3.35it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 29% Completed | 19/66 [00:05<00:13, 3.41it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 30% Completed | 20/66 [00:05<00:13, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 32% Completed | 21/66 [00:05<00:12, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 33% Completed | 22/66 [00:06<00:12, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 35% Completed | 23/66 [00:06<00:12, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 36% Completed | 24/66 [00:06<00:11, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 25/66 [00:07<00:11, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 39% Completed | 26/66 [00:07<00:10, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 41% Completed | 27/66 [00:07<00:10, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 42% Completed | 28/66 [00:07<00:10, 3.74it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 29/66 [00:08<00:09, 3.76it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 45% Completed | 30/66 [00:08<00:09, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 47% Completed | 31/66 [00:08<00:09, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 48% Completed | 32/66 [00:09<00:10, 3.34it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 33/66 [00:09<00:10, 3.26it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 52% Completed | 34/66 [00:09<00:09, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 53% Completed | 35/66 [00:09<00:09, 3.34it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 55% Completed | 36/66 [00:10<00:08, 3.41it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 37/66 [00:10<00:08, 3.48it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 58% Completed | 38/66 [00:10<00:07, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 59% Completed | 39/66 [00:11<00:07, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 61% Completed | 40/66 [00:11<00:07, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 41/66 [00:11<00:06, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 64% Completed | 42/66 [00:11<00:06, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 65% Completed | 43/66 [00:12<00:06, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 67% Completed | 44/66 [00:12<00:06, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 68% Completed | 45/66 [00:12<00:05, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 70% Completed | 46/66 [00:13<00:05, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 71% Completed | 47/66 [00:13<00:05, 3.52it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 73% Completed | 48/66 [00:13<00:05, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 74% Completed | 49/66 [00:13<00:04, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 76% Completed | 50/66 [00:14<00:04, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 77% Completed | 51/66 [00:14<00:04, 3.47it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 79% Completed | 52/66 [00:14<00:04, 3.30it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 80% Completed | 53/66 [00:15<00:03, 3.35it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 82% Completed | 54/66 [00:15<00:03, 3.42it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 83% Completed | 55/66 [00:15<00:03, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 85% Completed | 56/66 [00:15<00:02, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 86% Completed | 57/66 [00:16<00:02, 3.52it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 58/66 [00:16<00:02, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 89% Completed | 59/66 [00:16<00:02, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 91% Completed | 60/66 [00:17<00:01, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 92% Completed | 61/66 [00:17<00:01, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 62/66 [00:17<00:01, 3.47it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 95% Completed | 63/66 [00:17<00:00, 3.47it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 97% Completed | 64/66 [00:18<00:00, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:24<00:00, 1.51s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:24<00:00, 2.74it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:10 [default_loader.py:430] Loading weights took 24.91 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:12 [gpu_model_runner.py:5471] Model loading took 16.36 GiB memory and 36.809273 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [interface.py:918] Setting attention block size to 784 tokens to ensure that attention page size is >= mamba page size.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:13 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:13 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:14 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:15 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [base.py:148] Available KV cache memory: 1.07 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [kv_cache_utils.py:2032] GPU KV cache size: 1,609 tokens, Maximum concurrency for 512 tokens per request: 3.14x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:23 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132)
|
||||
JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (2 compile keys): 100%|██████████| 1/1 [00:00<00:00, 62.51kernel/s, ComputeSlotMappingKernel (2 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [kernel_warmup.py:134] JIT kernel warmup finished in 0.02s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:23 [qwen_triton_warmup.py:270] Warming up Qwen Triton kernels for model_type=qwen3_5_text.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:24 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:24 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.8, 19.11 GiB). Actual usage is 17.98 GiB for consumed memory (weights + non-torch), 0.12 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=937723085` (0.87 GiB) to fit into requested memory, or `--kv-cache-memory=4692621312` (4.37 GiB) to fully utilize gpu memory. Current kv cache memory in use is 1.07 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:25 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:25 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:25 [core.py:368] init engine (profile, create kv cache, warmup model) took 12.53 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:25 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:25 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:25 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:25 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:25 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:25 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:05:25 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-24 17:05:25 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:25 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-24 17:05:26 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-24 17:05:26 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
[transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
INFO 09-24 17:05:34 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-24 17:05:34 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-24 17:05:34 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 93.88 seconds
|
||||
INFO 09-24 17:05:34 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
WARNING 09-24 17:05:34 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:34 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _w8a8_triton_block_scaled_mm. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:34 [jit_monitor.py:141] Triton kernel JIT compilation during inference: layer_norm_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) INFO 09-24 17:05:35 [prefix_cache.py:160] Omni prefix cache async-write copy stream initialized.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:35 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _causal_conv1d_update_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:35 [jit_monitor.py:141] Triton kernel JIT compilation during inference: fused_recurrent_gated_delta_rule_packed_decode_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:50 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _bilinear_pos_embed_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=50132) WARNING 09-24 17:05:50 [jit_monitor.py:141] Triton kernel JIT compilation during inference: rotary_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
INFO 09-24 17:05:55 [omni_base.py:781] [AsyncOmni] Shutting down
|
||||
INFO 09-24 17:05:55 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-24 17:05:55 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-24 17:05:55 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-24 17:05:55 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-24 17:05:55 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-24 17:05:55 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-24 17:05:57 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-24 17:05:57 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-24 17:05:57 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-24 17:05:57 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"started": 1790240618.8739433,
|
||||
"config": {
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"cpu_offload_gb": 12,
|
||||
"linear_backend": "triton",
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180,
|
||||
"init_timeout": 240
|
||||
},
|
||||
"startup_s": 93.87675339996349,
|
||||
"cases": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 3.2853383000474423,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready"
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready"
|
||||
}
|
||||
],
|
||||
"expected": "ready"
|
||||
},
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 17.321737900027074,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red"
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red"
|
||||
}
|
||||
],
|
||||
"expected": "Red"
|
||||
}
|
||||
],
|
||||
"status": "completed",
|
||||
"ended": 1790240757.747901
|
||||
}
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
INFO 09-24 17:08:19 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:08:19 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 17:08:23 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:08:23 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:08:23 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-24 17:08:24 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 17:08:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-24 17:08:24 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-24 17:08:24 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration
|
||||
INFO 09-24 17:08:24 [model.py:2021] Using max model len 512
|
||||
WARNING 09-24 17:08:24 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-24 17:08:30 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-24 17:08:30 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled
|
||||
WARNING 09-24 17:08:30 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.
|
||||
WARNING 09-24 17:08:30 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-24 17:08:30 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-24 17:08:30 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-24 17:08:30 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:08:30 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-24 17:08:30 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-24 17:08:44 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:08:44 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:47 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:49 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_3a2cbf0703ea4b12bd6133332d341d3d backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:49 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:49 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:50 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:50 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:55 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:55 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:08:55 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:55 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [uva.py:65] Total CPU offloaded parameters: 0.86
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:56 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:57 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:08:57 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:04 [uva.py:65] Total CPU offloaded parameters: 12.13
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:04 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 7.52 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:04 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (7.52 GiB).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 2% Completed | 1/66 [00:00<00:44, 1.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 3% Completed | 2/66 [00:00<00:29, 2.15it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 5% Completed | 3/66 [00:01<00:24, 2.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 4/66 [00:01<00:20, 3.00it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 8% Completed | 5/66 [00:01<00:19, 3.19it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 9% Completed | 6/66 [00:02<00:18, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 11% Completed | 7/66 [00:02<00:17, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 8/66 [00:02<00:17, 3.40it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 14% Completed | 9/66 [00:02<00:16, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 15% Completed | 10/66 [00:03<00:15, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 17% Completed | 11/66 [00:03<00:15, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 18% Completed | 12/66 [00:03<00:15, 3.44it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 20% Completed | 13/66 [00:04<00:15, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 21% Completed | 14/66 [00:04<00:14, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 23% Completed | 15/66 [00:04<00:14, 3.52it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 24% Completed | 16/66 [00:04<00:14, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 26% Completed | 17/66 [00:05<00:14, 3.31it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 27% Completed | 18/66 [00:05<00:13, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 29% Completed | 19/66 [00:05<00:13, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 30% Completed | 20/66 [00:06<00:13, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 32% Completed | 21/66 [00:06<00:12, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 33% Completed | 22/66 [00:06<00:12, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 35% Completed | 23/66 [00:06<00:12, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 36% Completed | 24/66 [00:07<00:11, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 25/66 [00:07<00:11, 3.62it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 39% Completed | 26/66 [00:07<00:10, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 41% Completed | 27/66 [00:08<00:10, 3.69it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 42% Completed | 28/66 [00:08<00:10, 3.69it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 29/66 [00:08<00:09, 3.73it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 45% Completed | 30/66 [00:08<00:09, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 47% Completed | 31/66 [00:09<00:09, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 48% Completed | 32/66 [00:09<00:09, 3.62it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 33/66 [00:09<00:09, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 52% Completed | 34/66 [00:09<00:09, 3.52it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 53% Completed | 35/66 [00:10<00:08, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 55% Completed | 36/66 [00:10<00:08, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 37/66 [00:10<00:08, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 58% Completed | 38/66 [00:11<00:07, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 59% Completed | 39/66 [00:11<00:07, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 61% Completed | 40/66 [00:11<00:07, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 41/66 [00:11<00:07, 3.42it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 64% Completed | 42/66 [00:12<00:06, 3.47it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 65% Completed | 43/66 [00:12<00:06, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 67% Completed | 44/66 [00:12<00:06, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 68% Completed | 45/66 [00:13<00:05, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 70% Completed | 46/66 [00:13<00:05, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 71% Completed | 47/66 [00:13<00:05, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 73% Completed | 48/66 [00:13<00:04, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 74% Completed | 49/66 [00:14<00:04, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 76% Completed | 50/66 [00:14<00:04, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 77% Completed | 51/66 [00:14<00:04, 3.26it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 79% Completed | 52/66 [00:15<00:04, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 80% Completed | 53/66 [00:15<00:03, 3.36it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 82% Completed | 54/66 [00:15<00:03, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 83% Completed | 55/66 [00:15<00:03, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 85% Completed | 56/66 [00:16<00:02, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 86% Completed | 57/66 [00:16<00:02, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 58/66 [00:16<00:02, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 89% Completed | 59/66 [00:17<00:01, 3.55it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 91% Completed | 60/66 [00:17<00:01, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 92% Completed | 61/66 [00:17<00:01, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 62/66 [00:17<00:01, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 95% Completed | 63/66 [00:18<00:00, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 97% Completed | 64/66 [00:18<00:00, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:23<00:00, 1.29s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:23<00:00, 2.82it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:28 [default_loader.py:430] Loading weights took 24.16 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:30 [gpu_model_runner.py:5471] Model loading took 16.36 GiB memory and 33.646984 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:30 [interface.py:918] Setting attention block size to 784 tokens to ensure that attention page size is >= mamba page size.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:30 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:30 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:31 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:31 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:31 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:31 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:32 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:32 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [base.py:148] Available KV cache memory: 1.56 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [kv_cache_utils.py:2032] GPU KV cache size: 2,340 tokens, Maximum concurrency for 512 tokens per request: 4.57x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:41 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940)
|
||||
JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (2 compile keys): 100%|██████████| 1/1 [00:00<00:00, 71.21kernel/s, ComputeSlotMappingKernel (2 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [kernel_warmup.py:134] JIT kernel warmup finished in 0.02s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [qwen_triton_warmup.py:270] Warming up Qwen Triton kernels for model_type=qwen3_5_text.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:41 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.8, 19.11 GiB). Actual usage is 17.49 GiB for consumed memory (weights + non-torch), 0.12 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=1455359181` (1.36 GiB) to fit into requested memory, or `--kv-cache-memory=5210257408` (4.85 GiB) to fully utilize gpu memory. Current kv cache memory in use is 1.56 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:42 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:43 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:43 [core.py:368] init engine (profile, create kv cache, warmup model) took 12.28 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:43 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:43 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:43 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:43 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:43 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:43 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:09:43 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-24 17:09:43 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:43 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-24 17:09:43 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-24 17:09:43 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
[transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
INFO 09-24 17:09:51 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-24 17:09:51 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-24 17:09:51 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 87.80 seconds
|
||||
INFO 09-24 17:09:51 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
WARNING 09-24 17:09:51 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:51 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _w8a8_triton_block_scaled_mm. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:51 [jit_monitor.py:141] Triton kernel JIT compilation during inference: layer_norm_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) INFO 09-24 17:09:52 [prefix_cache.py:160] Omni prefix cache async-write copy stream initialized.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:53 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _causal_conv1d_update_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:09:53 [jit_monitor.py:141] Triton kernel JIT compilation during inference: fused_recurrent_gated_delta_rule_packed_decode_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:10:07 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _bilinear_pos_embed_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=43940) WARNING 09-24 17:10:07 [jit_monitor.py:141] Triton kernel JIT compilation during inference: rotary_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
INFO 09-24 17:10:14 [orchestrator.py:917] [Orchestrator] Aborted request(s) ['public-qwen38-cancel-878f4cb2']
|
||||
Traceback (most recent call last):
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_qwen38_windows_async.py", line 119, in main
|
||||
raise RuntimeError("aborted request emitted late output")
|
||||
RuntimeError: aborted request emitted late output
|
||||
INFO 09-24 17:10:14 [omni_base.py:781] [AsyncOmni] Shutting down
|
||||
INFO 09-24 17:10:14 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-24 17:10:14 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-24 17:10:14 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-24 17:10:14 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-24 17:10:14 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-24 17:10:14 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-24 17:10:16 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-24 17:10:16 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-24 17:10:16 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-24 17:10:16 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"started": 1790240882.72909,
|
||||
"config": {
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"cpu_offload_gb": 12,
|
||||
"linear_backend": "triton",
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180,
|
||||
"init_timeout": 240
|
||||
},
|
||||
"startup_s": 87.79908420005813,
|
||||
"cases": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 3.327248699963093,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "ready"
|
||||
},
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 17.33421260002069,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "Red"
|
||||
}
|
||||
],
|
||||
"abort_check": {
|
||||
"request_id": "public-qwen38-cancel",
|
||||
"submitted": true,
|
||||
"task_active_before_abort": true,
|
||||
"emitted_before_abort": 0,
|
||||
"emitted_after_abort": 1,
|
||||
"task_error": null,
|
||||
"remaining_request_states": 0
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "RuntimeError('aborted request emitted late output')",
|
||||
"traceback": "Traceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\experiments\\probe_qwen38_windows_async.py\", line 119, in main\n raise RuntimeError(\"aborted request emitted late output\")\nRuntimeError: aborted request emitted late output\n",
|
||||
"ended": 1790241016.596035
|
||||
}
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
INFO 09-24 17:17:53 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:17:53 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 17:17:58 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:17:58 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:17:58 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-24 17:17:58 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 17:17:58 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-24 17:17:58 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-24 17:17:58 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration
|
||||
INFO 09-24 17:17:58 [model.py:2021] Using max model len 512
|
||||
WARNING 09-24 17:17:58 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-24 17:18:04 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-24 17:18:04 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled
|
||||
WARNING 09-24 17:18:04 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.
|
||||
WARNING 09-24 17:18:04 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-24 17:18:04 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-24 17:18:04 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-24 17:18:04 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:18:04 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-24 17:18:04 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-24 17:18:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:18:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:22 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:23 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_6823e98dd89849b5863ea6e9a102cfa0 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:23 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:24 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:24 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:24 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:18:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:29 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [uva.py:65] Total CPU offloaded parameters: 0.86
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:30 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:31 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:31 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:36 [uva.py:65] Total CPU offloaded parameters: 12.13
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:37 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 11.61 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:37 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (11.61 GiB).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 2% Completed | 1/66 [00:00<00:22, 2.94it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 3% Completed | 2/66 [00:00<00:18, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 5% Completed | 3/66 [00:00<00:18, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 4/66 [00:01<00:17, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 8% Completed | 5/66 [00:01<00:16, 3.65it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 9% Completed | 6/66 [00:01<00:16, 3.71it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 11% Completed | 7/66 [00:01<00:16, 3.66it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 8/66 [00:02<00:15, 3.72it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 14% Completed | 9/66 [00:02<00:15, 3.79it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 15% Completed | 10/66 [00:02<00:15, 3.72it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 17% Completed | 11/66 [00:03<00:14, 3.75it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 18% Completed | 12/66 [00:03<00:14, 3.73it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 20% Completed | 13/66 [00:03<00:14, 3.72it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 21% Completed | 14/66 [00:03<00:14, 3.71it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 23% Completed | 15/66 [00:04<00:13, 3.66it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 24% Completed | 16/66 [00:04<00:13, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 26% Completed | 17/66 [00:04<00:13, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 27% Completed | 18/66 [00:04<00:12, 3.73it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 29% Completed | 19/66 [00:05<00:12, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 30% Completed | 20/66 [00:05<00:12, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 32% Completed | 21/66 [00:05<00:12, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 33% Completed | 22/66 [00:05<00:11, 3.71it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 35% Completed | 23/66 [00:06<00:11, 3.76it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 36% Completed | 24/66 [00:06<00:11, 3.78it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 25/66 [00:06<00:10, 3.80it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 39% Completed | 26/66 [00:07<00:10, 3.81it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 41% Completed | 27/66 [00:07<00:10, 3.81it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 42% Completed | 28/66 [00:07<00:10, 3.77it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 29/66 [00:07<00:09, 3.75it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 45% Completed | 30/66 [00:08<00:09, 3.77it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 47% Completed | 31/66 [00:08<00:09, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 48% Completed | 32/66 [00:08<00:09, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 33/66 [00:08<00:09, 3.55it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 52% Completed | 34/66 [00:09<00:08, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 53% Completed | 35/66 [00:09<00:08, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 55% Completed | 36/66 [00:09<00:08, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 37/66 [00:10<00:07, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 58% Completed | 38/66 [00:10<00:07, 3.71it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 59% Completed | 39/66 [00:10<00:07, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 61% Completed | 40/66 [00:10<00:07, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 41/66 [00:11<00:07, 3.25it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 64% Completed | 42/66 [00:11<00:08, 3.00it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 65% Completed | 43/66 [00:11<00:07, 3.10it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 67% Completed | 44/66 [00:12<00:07, 3.13it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 68% Completed | 45/66 [00:12<00:06, 3.23it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 70% Completed | 46/66 [00:12<00:06, 3.25it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 71% Completed | 47/66 [00:13<00:05, 3.33it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 73% Completed | 48/66 [00:13<00:05, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 74% Completed | 49/66 [00:13<00:04, 3.48it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 76% Completed | 50/66 [00:13<00:04, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 77% Completed | 51/66 [00:14<00:04, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 79% Completed | 52/66 [00:14<00:03, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 80% Completed | 53/66 [00:14<00:03, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 82% Completed | 54/66 [00:15<00:03, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 83% Completed | 55/66 [00:15<00:03, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 85% Completed | 56/66 [00:15<00:02, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 86% Completed | 57/66 [00:15<00:02, 3.65it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 58/66 [00:16<00:02, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 89% Completed | 59/66 [00:16<00:01, 3.71it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 91% Completed | 60/66 [00:16<00:01, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 92% Completed | 61/66 [00:17<00:01, 3.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 62/66 [00:17<00:01, 3.55it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 95% Completed | 63/66 [00:17<00:00, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 97% Completed | 64/66 [00:17<00:00, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:21<00:00, 1.09s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:21<00:00, 3.01it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:18:59 [default_loader.py:430] Loading weights took 22.63 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [gpu_model_runner.py:5471] Model loading took 16.36 GiB memory and 30.510909 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [interface.py:918] Setting attention block size to 784 tokens to ensure that attention page size is >= mamba page size.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:01 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:02 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:03 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [base.py:148] Available KV cache memory: 1.5 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [kv_cache_utils.py:2032] GPU KV cache size: 2,267 tokens, Maximum concurrency for 512 tokens per request: 4.43x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:11 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020)
|
||||
JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (2 compile keys): 100%|██████████| 1/1 [00:00<00:00, 83.35kernel/s, ComputeSlotMappingKernel (2 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [kernel_warmup.py:134] JIT kernel warmup finished in 0.01s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:11 [qwen_triton_warmup.py:270] Warming up Qwen Triton kernels for model_type=qwen3_5_text.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:12 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:12 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.8, 19.11 GiB). Actual usage is 17.55 GiB for consumed memory (weights + non-torch), 0.12 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=1389860045` (1.29 GiB) to fit into requested memory, or `--kv-cache-memory=5144758272` (4.79 GiB) to fully utilize gpu memory. Current kv cache memory in use is 1.5 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:13 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:13 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:13 [core.py:368] init engine (profile, create kv cache, warmup model) took 12.33 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:13 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:13 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:14 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:14 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:14 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:14 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:19:14 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-24 17:19:14 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:14 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-24 17:19:14 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-24 17:19:14 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
[transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
INFO 09-24 17:19:26 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-24 17:19:26 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-24 17:19:26 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 88.40 seconds
|
||||
INFO 09-24 17:19:26 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
WARNING 09-24 17:19:26 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:27 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _w8a8_triton_block_scaled_mm. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:27 [jit_monitor.py:141] Triton kernel JIT compilation during inference: layer_norm_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) INFO 09-24 17:19:28 [prefix_cache.py:160] Omni prefix cache async-write copy stream initialized.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:28 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _causal_conv1d_update_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:28 [jit_monitor.py:141] Triton kernel JIT compilation during inference: fused_recurrent_gated_delta_rule_packed_decode_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:42 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _bilinear_pos_embed_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=57020) WARNING 09-24 17:19:42 [jit_monitor.py:141] Triton kernel JIT compilation during inference: rotary_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
INFO 09-24 17:19:49 [orchestrator.py:917] [Orchestrator] Aborted request(s) ['public-qwen38-cancel-b8953e63']
|
||||
INFO 09-24 17:19:52 [omni_base.py:781] [AsyncOmni] Shutting down
|
||||
INFO 09-24 17:19:52 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-24 17:19:52 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-24 17:19:52 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-24 17:19:52 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-24 17:19:52 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-24 17:19:52 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-24 17:19:54 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-24 17:19:54 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-24 17:19:54 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-24 17:19:54 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
{
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"started": 1790241458.5255756,
|
||||
"config": {
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"cpu_offload_gb": 12,
|
||||
"linear_backend": "triton",
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180,
|
||||
"init_timeout": 240
|
||||
},
|
||||
"startup_s": 88.39937659993302,
|
||||
"cases": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 3.3884481999557465,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "ready"
|
||||
},
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 17.125542300054803,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "Red"
|
||||
}
|
||||
],
|
||||
"abort_check": {
|
||||
"request_id": "public-qwen38-cancel",
|
||||
"submitted": true,
|
||||
"task_active_before_abort": true,
|
||||
"emitted_before_abort": 0,
|
||||
"emissions": [
|
||||
{
|
||||
"at_monotonic": 696329.9361154,
|
||||
"text": "",
|
||||
"finished": true,
|
||||
"finish_reason": "abort"
|
||||
}
|
||||
],
|
||||
"abort_started_monotonic": 696329.9333832,
|
||||
"abort_completed_monotonic": 696329.936032,
|
||||
"task_error": null,
|
||||
"remaining_request_states": 0,
|
||||
"fresh_outputs": [
|
||||
"ready",
|
||||
"ready"
|
||||
]
|
||||
},
|
||||
"status": "completed",
|
||||
"ended": 1790241594.7300231
|
||||
}
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
INFO 09-24 17:14:37 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:14:37 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 17:14:41 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:14:41 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:14:41 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-24 17:14:42 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 17:14:42 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-24 17:14:42 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-24 17:14:42 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration
|
||||
INFO 09-24 17:14:42 [model.py:2021] Using max model len 512
|
||||
WARNING 09-24 17:14:42 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-24 17:14:47 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-24 17:14:47 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled
|
||||
WARNING 09-24 17:14:47 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.
|
||||
WARNING 09-24 17:14:47 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-24 17:14:47 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-24 17:14:47 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-24 17:14:47 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:14:47 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-24 17:14:47 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-24 17:15:01 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:15:01 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:05 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:07 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_2342da557dc44b13b84988c9a0316f0b backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:07 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:07 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:07 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:07 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:12 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:12 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:12 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:12 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [uva.py:65] Total CPU offloaded parameters: 0.86
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:13 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:14 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:14 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:21 [uva.py:65] Total CPU offloaded parameters: 12.13
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:22 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 5.53 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:22 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (5.53 GiB).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 2% Completed | 1/66 [00:00<00:42, 1.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 3% Completed | 2/66 [00:00<00:27, 2.33it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 5% Completed | 3/66 [00:01<00:22, 2.81it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 4/66 [00:01<00:19, 3.14it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 8% Completed | 5/66 [00:01<00:18, 3.28it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 9% Completed | 6/66 [00:02<00:17, 3.39it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 11% Completed | 7/66 [00:02<00:16, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 8/66 [00:02<00:16, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 14% Completed | 9/66 [00:02<00:15, 3.62it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 15% Completed | 10/66 [00:03<00:15, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 17% Completed | 11/66 [00:03<00:14, 3.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 18% Completed | 12/66 [00:03<00:14, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 20% Completed | 13/66 [00:03<00:14, 3.74it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 21% Completed | 14/66 [00:04<00:14, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 23% Completed | 15/66 [00:04<00:14, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 24% Completed | 16/66 [00:04<00:14, 3.47it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 26% Completed | 17/66 [00:05<00:13, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 27% Completed | 18/66 [00:05<00:13, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 29% Completed | 19/66 [00:05<00:13, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 30% Completed | 20/66 [00:05<00:12, 3.65it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 32% Completed | 21/66 [00:06<00:12, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 33% Completed | 22/66 [00:06<00:12, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 35% Completed | 23/66 [00:06<00:11, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 36% Completed | 24/66 [00:06<00:11, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 25/66 [00:07<00:11, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 39% Completed | 26/66 [00:07<00:10, 3.66it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 41% Completed | 27/66 [00:07<00:10, 3.66it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 42% Completed | 28/66 [00:08<00:10, 3.72it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 29/66 [00:08<00:09, 3.73it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 45% Completed | 30/66 [00:08<00:09, 3.69it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 47% Completed | 31/66 [00:08<00:09, 3.75it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 48% Completed | 32/66 [00:09<00:09, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 33/66 [00:09<00:09, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 52% Completed | 34/66 [00:09<00:08, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 53% Completed | 35/66 [00:09<00:08, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 55% Completed | 36/66 [00:10<00:08, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 37/66 [00:10<00:08, 3.48it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 58% Completed | 38/66 [00:10<00:08, 3.48it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 59% Completed | 39/66 [00:11<00:07, 3.41it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 61% Completed | 40/66 [00:11<00:07, 3.27it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 41/66 [00:11<00:07, 3.29it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 64% Completed | 42/66 [00:12<00:07, 3.24it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 65% Completed | 43/66 [00:12<00:06, 3.29it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 67% Completed | 44/66 [00:12<00:06, 3.30it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 68% Completed | 45/66 [00:13<00:06, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 70% Completed | 46/66 [00:13<00:06, 3.21it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 71% Completed | 47/66 [00:13<00:05, 3.26it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 73% Completed | 48/66 [00:13<00:05, 3.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 74% Completed | 49/66 [00:14<00:05, 3.37it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 76% Completed | 50/66 [00:14<00:04, 3.38it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 77% Completed | 51/66 [00:14<00:04, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 79% Completed | 52/66 [00:15<00:04, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 80% Completed | 53/66 [00:15<00:03, 3.47it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 82% Completed | 54/66 [00:15<00:03, 3.48it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 83% Completed | 55/66 [00:15<00:03, 3.52it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 85% Completed | 56/66 [00:16<00:02, 3.38it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 86% Completed | 57/66 [00:16<00:02, 3.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 58/66 [00:16<00:02, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 89% Completed | 59/66 [00:17<00:02, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 91% Completed | 60/66 [00:17<00:01, 3.31it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 92% Completed | 61/66 [00:17<00:01, 3.37it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 62/66 [00:18<00:01, 3.37it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 95% Completed | 63/66 [00:18<00:00, 3.38it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 97% Completed | 64/66 [00:18<00:00, 3.42it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:23<00:00, 1.31s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:23<00:00, 2.80it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:46 [default_loader.py:430] Loading weights took 24.31 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:48 [gpu_model_runner.py:5471] Model loading took 16.36 GiB memory and 34.653076 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:48 [interface.py:918] Setting attention block size to 784 tokens to ensure that attention page size is >= mamba page size.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:48 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:48 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:49 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:49 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:50 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [base.py:148] Available KV cache memory: 1.48 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [kv_cache_utils.py:2032] GPU KV cache size: 2,194 tokens, Maximum concurrency for 512 tokens per request: 4.29x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:15:59 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532)
|
||||
JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (2 compile keys): 100%|██████████| 1/1 [00:00<00:00, 68.94kernel/s, ComputeSlotMappingKernel (2 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [kernel_warmup.py:134] JIT kernel warmup finished in 0.02s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [qwen_triton_warmup.py:270] Warming up Qwen Triton kernels for model_type=qwen3_5_text.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:15:59 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.8, 19.11 GiB). Actual usage is 17.57 GiB for consumed memory (weights + non-torch), 0.12 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=1369441485` (1.28 GiB) to fit into requested memory, or `--kv-cache-memory=5124339712` (4.77 GiB) to fully utilize gpu memory. Current kv cache memory in use is 1.48 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:01 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:01 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:01 [core.py:368] init engine (profile, create kv cache, warmup model) took 12.59 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:01 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:01 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:01 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:01 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:01 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:01 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:16:01 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-24 17:16:01 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:01 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-24 17:16:02 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-24 17:16:02 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
[transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
INFO 09-24 17:16:14 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-24 17:16:14 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-24 17:16:14 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 93.32 seconds
|
||||
INFO 09-24 17:16:14 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
WARNING 09-24 17:16:15 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:15 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _w8a8_triton_block_scaled_mm. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:15 [jit_monitor.py:141] Triton kernel JIT compilation during inference: layer_norm_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) INFO 09-24 17:16:16 [prefix_cache.py:160] Omni prefix cache async-write copy stream initialized.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:16 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _causal_conv1d_update_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:16 [jit_monitor.py:141] Triton kernel JIT compilation during inference: fused_recurrent_gated_delta_rule_packed_decode_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:40 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _bilinear_pos_embed_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=46532) WARNING 09-24 17:16:40 [jit_monitor.py:141] Triton kernel JIT compilation during inference: rotary_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
INFO 09-24 17:16:47 [orchestrator.py:917] [Orchestrator] Aborted request(s) ['public-qwen38-cancel-84b1a548']
|
||||
Traceback (most recent call last):
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_qwen38_windows_async.py", line 130, in main
|
||||
raise RuntimeError("abort did not produce only a terminal abort marker")
|
||||
RuntimeError: abort did not produce only a terminal abort marker
|
||||
INFO 09-24 17:16:47 [omni_base.py:781] [AsyncOmni] Shutting down
|
||||
INFO 09-24 17:16:47 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-24 17:16:47 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-24 17:16:47 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-24 17:16:47 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-24 17:16:47 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-24 17:16:47 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-24 17:16:49 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-24 17:16:49 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-24 17:16:49 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-24 17:16:49 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"started": 1790241262.2354827,
|
||||
"config": {
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"cpu_offload_gb": 12,
|
||||
"linear_backend": "triton",
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180,
|
||||
"init_timeout": 240
|
||||
},
|
||||
"startup_s": 93.32475709996652,
|
||||
"cases": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 3.298354000085965,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "ready"
|
||||
},
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 26.607640099944547,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "Red"
|
||||
}
|
||||
],
|
||||
"abort_check": {
|
||||
"request_id": "public-qwen38-cancel",
|
||||
"submitted": true,
|
||||
"task_active_before_abort": true,
|
||||
"emitted_before_abort": 0,
|
||||
"emissions": [
|
||||
{
|
||||
"at_monotonic": 696147.5697939,
|
||||
"text": "1",
|
||||
"finished": false,
|
||||
"finish_reason": null
|
||||
},
|
||||
{
|
||||
"at_monotonic": 696147.5715562,
|
||||
"text": "1",
|
||||
"finished": true,
|
||||
"finish_reason": "abort"
|
||||
}
|
||||
],
|
||||
"abort_started_monotonic": 696147.5670737,
|
||||
"abort_completed_monotonic": 696147.5714223,
|
||||
"task_error": null,
|
||||
"remaining_request_states": 0
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "RuntimeError('abort did not produce only a terminal abort marker')",
|
||||
"traceback": "Traceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\experiments\\probe_qwen38_windows_async.py\", line 130, in main\n raise RuntimeError(\"abort did not produce only a terminal abort marker\")\nRuntimeError: abort did not produce only a terminal abort marker\n",
|
||||
"ended": 1790241409.4759328
|
||||
}
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
INFO 09-24 17:11:25 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:11:25 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 17:11:29 [omni_base.py:216] [AsyncOmni] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:11:29 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
INFO 09-24 17:11:29 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-24 17:11:30 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 17:11:30 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-24 17:11:30 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-24 17:11:30 [model.py:684] Resolved architecture: Qwen3_5ForConditionalGeneration
|
||||
INFO 09-24 17:11:30 [model.py:2021] Using max model len 512
|
||||
WARNING 09-24 17:11:30 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-24 17:11:35 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-24 17:11:35 [config.py:625] Mamba cache mode is set to 'align' for Qwen3_5ForConditionalGeneration by default when prefix caching is enabled
|
||||
WARNING 09-24 17:11:35 [vllm.py:1211] Auto-disabled DeepGemm for model_type=qwen3_5_text on Blackwell. DeepGemm E8M0 scale format causes accuracy degradation for this architecture. Falling back to CUTLASS. To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.
|
||||
WARNING 09-24 17:11:35 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-24 17:11:35 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-24 17:11:35 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-24 17:11:35 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:11:36 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-24 17:11:36 [stage_runtime.py:778] [stage_init] Stage-0 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-24 17:11:50 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 17:11:50 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:53 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', speculative_config=None, tokenizer='C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=fp8, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=C:\Users\zhout\w2\Qwen3.8-27B-FP8, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['+quant_fp8', 'all', '+quant_fp8'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='triton')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:55 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_77a2e6f7769b4220a64e54db825edfb6 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:11:55 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:55 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:11:56 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:11:56 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) [transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [gpu_model_runner.py:5379] Starting to load model C:\Users\zhout\w2\Qwen3.8-27B-FP8...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [cuda.py:551] Using backend AttentionBackendEnum.FLASH_ATTN for vit attention
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:01 [mm_encoder_attention.py:372] Using AttentionBackendEnum.FLASH_ATTN for MMEncoderAttention.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [uva.py:65] Total CPU offloaded parameters: 0.86
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [__init__.py:695] Selected TritonFp8BlockScaledMMKernel for Fp8LinearMethod
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [qwen_gdn_linear_attn.py:167] Using Triton/FLA GDN prefill kernel (requested=auto, head_k_dim=128).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [qwen_gdn_linear_attn.py:519] GDN decode kernel: cuda
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:02 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:10 [uva.py:65] Total CPU offloaded parameters: 12.13
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:10 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 28.75 GiB. Available RAM: 7.56 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:10 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (28.75 GiB) exceeds 90% of available RAM (7.56 GiB).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/66 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 2% Completed | 1/66 [00:00<00:16, 3.86it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 3% Completed | 2/66 [00:00<00:17, 3.70it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 5% Completed | 3/66 [00:00<00:16, 3.73it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 4/66 [00:01<00:16, 3.74it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 8% Completed | 5/66 [00:01<00:16, 3.79it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 9% Completed | 6/66 [00:01<00:15, 3.76it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 11% Completed | 7/66 [00:01<00:15, 3.76it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 8/66 [00:02<00:15, 3.77it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 14% Completed | 9/66 [00:02<00:14, 3.80it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 15% Completed | 10/66 [00:02<00:14, 3.82it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 17% Completed | 11/66 [00:02<00:14, 3.85it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 18% Completed | 12/66 [00:03<00:14, 3.84it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 20% Completed | 13/66 [00:03<00:13, 3.83it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 21% Completed | 14/66 [00:03<00:13, 3.82it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 23% Completed | 15/66 [00:03<00:13, 3.77it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 24% Completed | 16/66 [00:04<00:13, 3.74it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 26% Completed | 17/66 [00:04<00:13, 3.74it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 27% Completed | 18/66 [00:04<00:12, 3.76it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 29% Completed | 19/66 [00:05<00:12, 3.78it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 30% Completed | 20/66 [00:05<00:12, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 32% Completed | 21/66 [00:05<00:12, 3.62it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 33% Completed | 22/66 [00:05<00:11, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 35% Completed | 23/66 [00:06<00:11, 3.72it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 36% Completed | 24/66 [00:06<00:11, 3.75it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 25/66 [00:06<00:10, 3.77it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 39% Completed | 26/66 [00:06<00:10, 3.79it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 41% Completed | 27/66 [00:07<00:10, 3.57it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 42% Completed | 28/66 [00:07<00:10, 3.48it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 29/66 [00:07<00:10, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 45% Completed | 30/66 [00:08<00:09, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 47% Completed | 31/66 [00:08<00:09, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 48% Completed | 32/66 [00:08<00:09, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 33/66 [00:08<00:09, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 52% Completed | 34/66 [00:09<00:09, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 53% Completed | 35/66 [00:09<00:08, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 55% Completed | 36/66 [00:09<00:08, 3.40it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 37/66 [00:10<00:08, 3.46it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 58% Completed | 38/66 [00:10<00:08, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 59% Completed | 39/66 [00:10<00:07, 3.55it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 61% Completed | 40/66 [00:10<00:07, 3.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 41/66 [00:11<00:07, 3.55it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 64% Completed | 42/66 [00:11<00:06, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 65% Completed | 43/66 [00:11<00:06, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 67% Completed | 44/66 [00:12<00:06, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 68% Completed | 45/66 [00:12<00:05, 3.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 70% Completed | 46/66 [00:12<00:05, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 71% Completed | 47/66 [00:12<00:05, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 73% Completed | 48/66 [00:13<00:04, 3.60it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 74% Completed | 49/66 [00:13<00:04, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 76% Completed | 50/66 [00:13<00:04, 3.65it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 77% Completed | 51/66 [00:13<00:04, 3.64it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 79% Completed | 52/66 [00:14<00:03, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 80% Completed | 53/66 [00:14<00:03, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 82% Completed | 54/66 [00:14<00:03, 3.49it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 83% Completed | 55/66 [00:15<00:03, 3.54it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 85% Completed | 56/66 [00:15<00:02, 3.45it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 86% Completed | 57/66 [00:15<00:02, 3.53it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 58/66 [00:15<00:02, 3.59it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 89% Completed | 59/66 [00:16<00:01, 3.56it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 91% Completed | 60/66 [00:16<00:01, 3.62it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 92% Completed | 61/66 [00:16<00:01, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 62/66 [00:17<00:01, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 95% Completed | 63/66 [00:17<00:00, 3.69it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 97% Completed | 64/66 [00:17<00:00, 3.72it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:21<00:00, 1.13s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 66/66 [00:21<00:00, 3.03it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:33 [default_loader.py:430] Loading weights took 22.52 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:34 [gpu_model_runner.py:5471] Model loading took 16.36 GiB memory and 32.190280 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:34 [interface.py:918] Setting attention block size to 784 tokens to ensure that attention page size is >= mamba page size.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:34 [interface.py:942] Padding mamba page size by 0.13% to ensure that mamba page size and attention page size are exactly equal.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:34 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:35 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:35 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=16384,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=6144,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=34816,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=5120,K=17408,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:36 [fp8_utils.py:852] Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! Config file not found at C:\Users\zhout\w2\omni029venv\Lib\site-packages\vllm\model_executor\layers\quantization\utils\configs\N=14336,K=5120,device_name=NVIDIA_GeForce_RTX_5090_Laptop_GPU,dtype=fp8_w8a8,block_shape=[128,128].json
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [base.py:148] Available KV cache memory: 1.6 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [kv_cache_utils.py:2032] GPU KV cache size: 2,413 tokens, Maximum concurrency for 512 tokens per request: 4.71x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:45 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700)
|
||||
JIT kernel warmup (2 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (2 compile keys): 100%|██████████| 1/1 [00:00<00:00, 76.91kernel/s, ComputeSlotMappingKernel (2 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [kernel_warmup.py:134] JIT kernel warmup finished in 0.02s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [qwen_triton_warmup.py:270] Warming up Qwen Triton kernels for model_type=qwen3_5_text.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:45 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.8, 19.11 GiB). Actual usage is 17.45 GiB for consumed memory (weights + non-torch), 0.12 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=1506165965` (1.4 GiB) to fit into requested memory, or `--kv-cache-memory=5261064192` (4.9 GiB) to fully utilize gpu memory. Current kv cache memory in use is 1.6 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:46 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:47 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:47 [core.py:368] init engine (profile, create kv cache, warmup model) took 12.27 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:47 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:47 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:47 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:47 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:47 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:47 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-24 17:12:47 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-24 17:12:47 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:47 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-24 17:12:47 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-24 17:12:47 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
[transformers] The `use_fast` parameter is deprecated and will be removed in a future version. Use `backend="torchvision"` instead of `use_fast=True`, or `backend="pil"` instead of `use_fast=False`.
|
||||
INFO 09-24 17:12:55 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-24 17:12:55 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-24 17:12:55 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 85.97 seconds
|
||||
INFO 09-24 17:12:55 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model C:\Users\zhout\w2\Qwen3.8-27B-FP8
|
||||
WARNING 09-24 17:12:55 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:55 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _w8a8_triton_block_scaled_mm. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:56 [jit_monitor.py:141] Triton kernel JIT compilation during inference: layer_norm_fwd_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) INFO 09-24 17:12:57 [prefix_cache.py:160] Omni prefix cache async-write copy stream initialized.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:57 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _causal_conv1d_update_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:12:57 [jit_monitor.py:141] Triton kernel JIT compilation during inference: fused_recurrent_gated_delta_rule_packed_decode_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:13:11 [jit_monitor.py:141] Triton kernel JIT compilation during inference: _bilinear_pos_embed_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=14700) WARNING 09-24 17:13:11 [jit_monitor.py:141] Triton kernel JIT compilation during inference: rotary_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
INFO 09-24 17:13:18 [orchestrator.py:917] [Orchestrator] Aborted request(s) ['public-qwen38-cancel-bf06eb9b']
|
||||
Traceback (most recent call last):
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_qwen38_windows_async.py", line 125, in main
|
||||
raise RuntimeError("aborted request emitted after abort completed")
|
||||
RuntimeError: aborted request emitted after abort completed
|
||||
INFO 09-24 17:13:18 [omni_base.py:781] [AsyncOmni] Shutting down
|
||||
INFO 09-24 17:13:18 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-24 17:13:18 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-24 17:13:18 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-24 17:13:18 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-24 17:13:18 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-24 17:13:18 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-24 17:13:20 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-24 17:13:20 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-24 17:13:20 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-24 17:13:20 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"entrypoint": "AsyncOmni.generate",
|
||||
"started": 1790241070.80365,
|
||||
"config": {
|
||||
"model": "C:\\Users\\zhout\\w2\\Qwen3.8-27B-FP8",
|
||||
"language_model_only": false,
|
||||
"max_model_len": 512,
|
||||
"max_num_seqs": 1,
|
||||
"max_num_batched_tokens": 512,
|
||||
"enforce_eager": true,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"cpu_offload_gb": 12,
|
||||
"linear_backend": "triton",
|
||||
"dtype": "bfloat16",
|
||||
"stage_init_timeout": 180,
|
||||
"init_timeout": 240
|
||||
},
|
||||
"startup_s": 85.97157339996193,
|
||||
"cases": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 3.3566855000099167,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-text",
|
||||
"error": null,
|
||||
"text": "ready",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "ready"
|
||||
},
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 16.921946799964644,
|
||||
"outputs": [
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": false,
|
||||
"stage_id": 0
|
||||
},
|
||||
{
|
||||
"request_id": "public-qwen38-image",
|
||||
"error": null,
|
||||
"text": "Red",
|
||||
"finished": true,
|
||||
"stage_id": 0
|
||||
}
|
||||
],
|
||||
"expected": "Red"
|
||||
}
|
||||
],
|
||||
"abort_check": {
|
||||
"request_id": "public-qwen38-cancel",
|
||||
"submitted": true,
|
||||
"task_active_before_abort": true,
|
||||
"emitted_before_abort": 0,
|
||||
"emissions": [
|
||||
{
|
||||
"at_monotonic": 695938.6635963,
|
||||
"text": "",
|
||||
"finished": true
|
||||
}
|
||||
],
|
||||
"abort_started_monotonic": 695938.6613355,
|
||||
"abort_completed_monotonic": 695938.6635175,
|
||||
"task_error": null,
|
||||
"remaining_request_states": 0
|
||||
},
|
||||
"status": "failed",
|
||||
"error": "RuntimeError('aborted request emitted after abort completed')",
|
||||
"traceback": "Traceback (most recent call last):\n File \"\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\experiments\\probe_qwen38_windows_async.py\", line 125, in main\n raise RuntimeError(\"aborted request emitted after abort completed\")\nRuntimeError: aborted request emitted after abort completed\n",
|
||||
"ended": 1790241200.6279483
|
||||
}
|
||||
@@ -86,7 +86,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI_CUDA; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted Omni whole-policy graph stage with BF16 policy and Cosmos encoder on the RTX 5090 Laptop GPU. One warmup plus 20/20 serial patterned synthetic observation/state/noise requests returned identical finite float32 [1,50,32] actions with terminal events; nearest-rank complete-request wall p50/p95 was 0.382/0.399 s, excluding 43.82 s startup. The 16 GiB host-RAM and 16 GiB CUDA demands were within explicit 28/21 GiB ceilings checked against available memory. Actual CUDA placement was verified, PyTorch peak reserved VRAM reached 7.86 GB, and an 8 GiB VRAM plan was refused before worker launch. An in-flight abort and shutdown released both ledgers with no stale output; a separate public AsyncOmni.generate action request passed. Relative to a separate patterned WSL CPU output, action relative L2 difference was 2.006% with no task tolerance. The earlier one-forward zero-input direct CUDA run was a different experiment. This is synthetic action-policy integration, not robot-task E2E; units/order/step time remain unverified and control_ready=false. | Validate real observations/reference actions and physical action semantics, post-cancel restart, concurrent admission, loading peak, deadlines and sustained latency/power/thermal behavior; [Omni CUDA evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_cuda/README.md), [earlier direct result](../../../e2e_recovery_20260923/evidence/internvla/internvla_finetune_synthetic.json). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST_GGUF_VULKAN; RESTART_AFTER_INFLIGHT_CANCEL_PASS; SEPARATE_4B_BF16_CUDA_PROFILE | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran through an Omni whole-session StageRuntime/StagePool backend on RTX Vulkan0. The server log identifies NVIDIA GeForce RTX 5090 Laptop GPU, 29/29 layers offloaded and a 1051.02 MiB Vulkan0 model buffer. The short and 120-item inventory prompts answered `Paris` and `1511`; one warmup plus 20/20 serial inventory requests answered `1511` with nearest-rank complete-request wall p50/p95 0.294/0.302 s (2,463 prompt tokens, five completion tokens), excluding 4.319 s startup. A public AsyncOmni request also answered `Paris` on the RTX. An over-context request was rejected without truncation, followed by a correct recovery request. In a separate cancellation check, the owned server task started before abort; the worker and host-RAM reservation drained, and a fresh stage with the same artifact and new generation answered `Paris` without stale output. This GGUF/Vulkan profile is distinct from the earlier Spark 4B BF16 vLLM CUDA profile; the precisions, model sizes and workloads preclude a speedup comparison. There is no incremental token stream, automatic same-stage restart, broad token/quality parity, explicit GPU-VRAM peak admission, concurrency or sustained power/thermal qualification. | Validate streaming, larger quality and parity workloads, VRAM/load peaks, concurrent requests and sustained power; [Omni RTX raw evidence](../../../e2e_expansion_20260924/evidence/spark_omni_windows_rtx/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_WAV_RTX_VULKAN_CPU; RESTART_AFTER_INFLIGHT_CANCEL_PASS; SEPARATE_CUDA_STREAM_PLAYBACK_JITTER | The pinned Q8_0 CustomVoice talker and F16 codec ran on RTX Vulkan0 with the code predictor pinned to CPU FP32 through an admitted Omni whole-session stage. The server logs identify the RTX and actual codec placement. Two named text-to-WAV requests returned finite 24 kHz PCM; one warmup plus 20/20 serial 2.72 s outputs had identical PCM and nearest-rank complete-request wall p50/p95 3.153/3.231 s (RTF 1.16/1.19), excluding startup. Pinned Whisper tiny.en transcribed the first RTX WAV exactly (WER 0 as an intelligibility proxy); RTX PCM differs from the Radeon route, so no same-output speedup is claimed. Public AsyncOmni audio returned the same PCM. The owned server entered prefill before cancellation; the worker and separate declared 8 GiB host-RAM/4 GiB GPU-VRAM reservations drained, then a fresh stage with a new generation returned identical PCM without late output. An undersized 3 GiB GPU-VRAM demand was refused before server launch; the Radeon route still passed a one-request regression. This is complete-WAV Vulkan+CPU execution, not incremental playable audio, an NPU route or a measured GPU-memory peak. Separately, the historical complete-model vLLM CUDA stream slowed sharply around minute 26–27: all 17 requests in minutes 27–30 had simulated playback underruns and median RTF 1.331. A current-source instrumented 30-minute CUDA repeat completed 1,232 serial medium requests; its final 119 had p50/p95 RTF 0.185/0.203 and three brief simulated underruns needing at most 26.6 ms extra prebuffer. The late collapse did not recur, but 69 requests across the full windows had simulated underruns; the original cause and underrun-free sustained playback remain open. These CUDA and Vulkan routes use different artifacts, workloads and output behavior. | Validate playable chunk streaming, broader voice/speaker quality, measured VRAM/load peaks, concurrency and an underrun-free sustained run; [RTX Omni raw evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_windows_rtx/README.md), [CUDA sustained runs](../../../e2e_expansion_20260924/evidence/qwen_tts_sustained_drift/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | SCOPED_E2E_OMNI_FP8_TEXT_IMAGE_RTX; NVFP4_POST_LOAD_PTX_STARTUP_FAILED | The separate pinned Qwen/Qwen3.8-27B-FP8 checkpoint (revision 017b9c7) now ran complete native-Windows Omni text and synthetic red-image requests on RTX 5090 Laptop using vLLM 0.29.0+cu134 with the explicit Triton block-FP8 linear backend, BF16 compute, 12 GiB host offload, 0.80 GPU-memory cap, one sequence and 512-token context. One warmup of each kind plus 20/20 serial measured image outputs Red and 20/20 text outputs ready passed; nearest-rank complete-request wall p50/p95 was 4.184/4.380 s image and 3.070/3.214 s text. The profile initialized in 87.37 s, with 31.35 s reported weight load and a 1,974-token GPU KV cache; startup is excluded from request timing. The default FP8 CUTLASS operator lacked an SM120 kernel, and a 0.75 cap refused the 0.33 GiB required KV allocation; both failures are retained. The different NVFP4 artifact still fails in Marlin post-load PTX under driver 610.71. These two toy prompts do not qualify long context, video, broad quality, cancellation, explicit loading-peak admission, concurrency or sustained behavior. | Validate public AsyncOmni and cancellation/restart, explicit host/VRAM admission and load peaks, longer multimodal/quality cases and sustained power; keep NVFP4 driver/toolchain repair separate; [FP8 and NVFP4 raw evidence](../../../e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | SCOPED_E2E_OMNI_FP8_TEXT_IMAGE_RTX; PUBLIC_ASYNC_ABORT_RECOVERY_PASS; NVFP4_POST_LOAD_PTX_STARTUP_FAILED | The separate pinned Qwen/Qwen3.8-27B-FP8 checkpoint (revision 017b9c7) now ran complete native-Windows Omni text and synthetic red-image requests on RTX 5090 Laptop using vLLM 0.29.0+cu134 with the explicit Triton block-FP8 linear backend, BF16 compute, 12 GiB host offload, 0.80 GPU-memory cap, one sequence and 512-token context. One warmup of each kind plus 20/20 serial measured image outputs Red and 20/20 text outputs ready passed; nearest-rank complete-request wall p50/p95 was 4.184/4.380 s image and 3.070/3.214 s text. The profile initialized in 87.37 s, with 31.35 s reported weight load and a 1,974-token GPU KV cache; startup is excluded from request timing. The default FP8 CUTLASS operator lacked an SM120 kernel, and a 0.75 cap refused the 0.33 GiB required KV allocation; both failures are retained. The different NVFP4 artifact still fails in Marlin post-load PTX under driver 610.71. A public AsyncOmni text and image request passed; a long text request was aborted while active, produced one terminal abort marker, cleared request state, and a fresh same-engine request returned ready. The public stream carries cumulative text before its final event. These toy prompts do not qualify long context, video, broad quality, a server-started cancellation boundary, explicit loading-peak admission, concurrency or sustained behavior. | Validate server-started cancellation and separate-stage restart, explicit host/VRAM admission and load peaks, longer multimodal/quality cases and sustained power; keep NVFP4 driver/toolchain repair separate; [FP8 and NVFP4 raw evidence](../../../e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed native Windows three-stage BF16 text, one synthetic image, and one synthetic audio input to text+speech in separate constrained plans with 5 GiB thinker host offload and explicit 2,048/512-token limits. Text and all 32 thinker token IDs matched WSL CUDA; a 5.2 s WAV was nonzero. The red-square image produced the same description and 27 thinker token IDs as WSL plus a nonzero 5.2 s WAV in 17.15 s. The 2 s/440 Hz tone produced the same whistling description and nine IDs as WSL plus a nonzero 2.16 s WAV in 8.26 s. The initial text run needed 900/600 s startup limits after a 300 s timeout; a separate 1-warmup/20-measured serial text run had request wall p50 18.38 s/p95 18.73 s, with sampled host available RAM as low as 0.78 GB and pagefile use up to 11.17 GB. These synthetic inputs do not establish general image/audio or speech quality. | Check real image/audio suites, video input, speech intelligibility/alignment, concurrency, interruption, loading-peak admission and sustained thermal/power behavior before broader qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI_CUDA; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted native Windows Omni whole-policy graph stage with BF16 policy and Cosmos encoder on the RTX 5090 Laptop GPU. After one warmup, 20/20 serial patterned synthetic requests returned identical finite float32 [1,50,32] actions with terminal events; nearest-rank complete-request wall p50/p95 was 0.746/0.806 s, excluding 147.77 s startup. The 16 GiB host-RAM and 16 GiB CUDA demands were within explicit 18/21 GiB ceilings checked against available memory. Actual CUDA placement was verified, PyTorch peak reserved VRAM reached 7.86 GB, and an 8 GiB VRAM plan was refused before worker launch. In-flight cancellation and shutdown released both ledgers with no stale output; a separate public AsyncOmni.generate request returned the same action hash. Relative to the separate native CPU patterned output, action relative L2 difference was 2.340% with no task tolerance; native-vs-WSL CUDA relative L2 was 1.555%. The earlier one-forward zero-input direct CUDA run was a different experiment. This is synthetic action-policy integration, not robot-task E2E; units/order/step time remain unverified and control_ready=false. | Validate real observations/reference actions and physical action semantics, post-cancel restart, concurrent admission, loading peak, deadlines and sustained latency/power/thermal behavior; [Omni Windows CUDA evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_windows_cuda/README.md), [earlier direct result](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST; RESTART_AFTER_INFLIGHT_CANCEL_PASS | The same pinned Spark-X2.5-1.7B Q4_K_M GGUF completed text prefill/decode through a bounded Omni whole-session stage with llama.cpp Vulkan1. The log identifies Radeon 890M and 29/29 layers offloaded there, rather than RTX 5090; an intentionally wrong device name was refused. `Paris` and `1511` passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 2.10/2.14 s. A public AsyncOmni request passed; an oversized context was rejected with a later correct request; in-flight cancellation produced no stale output and released the ledger. The prior standalone run was 2.18/2.20 s. A separate confirmed server-start cancellation of the 120-item request terminated the Vulkan1 worker, left no stale output or reservation, and a fresh StageRuntime returned Paris under a new worker generation with Radeon 890M placement again verified. This is fresh-stage recovery, not same-session continuation. No streaming, GPU-memory peak or sustained power/thermal check is claimed. | Qualify public configuration, broader text/token quality, streaming, concurrent recovery, shared-RAM/GPU peak and sustained use; [restart evidence](../../../e2e_expansion_20260924/evidence/spark_omni_windows_restart/README.md), [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
|
||||
@@ -331,7 +331,7 @@ The [native Windows RTX Spark GGUF follow-up](../../benchmarks/edge_harness/resu
|
||||
|
||||
A [native Windows Qwen3.8-27B NVFP4 attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md) did not reach inference despite loading all three weight shards. The installed Windows vLLM 0.29+cu134 wheel failed during Marlin post-load scale permutation with an unsupported PTX toolchain error under RTX driver 610.71/CUDA UMD 13.3. NVIDIA's PTX compatibility guidance makes a driver/toolchain mismatch likely; an upgraded-driver or compatible-wheel control is still needed before rerunning the 512-token text/image cases. This failure constrains the tested Windows route, not the previously passed WSL checkpoint path.
|
||||
|
||||
The separate official Qwen3.8-27B FP8 checkpoint now has a [scoped native-Windows Omni RTX text/image pass and 20-request serial profile per modality](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md). The installed Windows wheel lacks an SM120 CUTLASS block-FP8 operator, so an explicit Omni-owned Triton linear backend was added and verified. The 0.75 GPU-memory cap correctly refused the 512-token KV requirement; 0.80 with 12 GiB declared host offload admitted the run. This is a different artifact/backend from NVFP4 and covers only short text plus a synthetic red image. The next gates remain public request, cancellation/recovery, loading-peak admission, longer multimodal and quality cases, plus sustained power/thermal behavior.
|
||||
The separate official Qwen3.8-27B FP8 checkpoint now has a [scoped native-Windows Omni RTX text/image pass and 20-request serial profile per modality](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md). The installed Windows wheel lacks an SM120 CUTLASS block-FP8 operator, so an explicit Omni-owned Triton linear backend was added and verified. The 0.75 GPU-memory cap correctly refused the 512-token KV requirement; 0.80 with 12 GiB declared host offload admitted the run. A public AsyncOmni text/image check and same-engine abort/recovery also passed: the canceled long request emitted its terminal abort marker, cleared request state and was followed by a correct fresh request. This is a different artifact/backend from NVFP4 and covers only short text plus a synthetic red image. The next gates remain a server-started cancellation boundary, separate-stage restart, loading-peak admission, longer multimodal and quality cases, plus sustained power/thermal behavior.
|
||||
|
||||
The [native Windows RTX Qwen3-TTS Omni hybrid follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_windows_rtx/README.md) extends the existing whole-session CrispASR stage to the optional discrete-GPU route. The RTX Vulkan0 talker/codec and CPU FP32 predictor produced two named complete WAVs, 20 identical serial measured outputs at 3.153/3.231 s p50/p95 for 2.72 s audio, and public AsyncOmni audio. Pinned Whisper tiny.en transcribed the first sentence exactly; the RTX PCM differs from the Radeon PCM, so the timings are not a same-output acceleration comparison. Separate 8 GiB host-RAM and 4 GiB GPU-VRAM reservations, a prelaunch GPU-budget refusal, in-flight cancellation and fresh-stage restart passed; the original Radeon route passed a regression probe. M2 still needs playable chunks, RTF below 1 for the declared workload, broader speaker quality, measured VRAM/load peaks, concurrency and an underrun-free sustained run. This Vulkan+CPU route does not qualify mobile NPU+GPU co-residency.
|
||||
|
||||
|
||||
Reference in new issue
Block a user