Qualify WSL CPU Qwen3.8 GGUF with bounded no-repack loading
This commit is contained in:
tzhouam committed
1 parent
e74d029380
commit
17b737914b
21 files changed
+4093
-3
No files matched your search
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit pinned Qwen3.8 GGUF Omni WSL CPU complete-request profiling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
MODEL_SHA = "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747"
|
||||
MMPROJ_SHA = "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb"
|
||||
SERVER_SHA = "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c"
|
||||
REVISION = "efbb3b1f70a21d97fd4495240648405f7228554f"
|
||||
|
||||
|
||||
def read(root: Path, name: str) -> dict:
|
||||
return json.loads((root / name).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def rank(values: list[float], fraction: float) -> float:
|
||||
return sorted(values)[math.ceil(len(values) * fraction) - 1]
|
||||
|
||||
|
||||
def digest(path: Path) -> dict[str, str | int]:
|
||||
with path.open("rb") as stream:
|
||||
return {"bytes": path.stat().st_size, "sha256": hashlib.file_digest(stream, "sha256").hexdigest()}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--evidence-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
root = args.evidence_dir
|
||||
for name in ("smoke.json", "smoke2.json"):
|
||||
failed = read(root, name)
|
||||
if failed["status"] != "failed" or "available native host RAM" not in failed["error"]:
|
||||
raise ValueError(f"{name}: low-available-RAM pre-admission refusal missing")
|
||||
repack = read(root, "smoke3.json")
|
||||
if repack["status"] != "failed" or "RSS exceeds stage reservation" not in repack["error"]:
|
||||
raise ValueError("repacked model did not fail the loaded-RSS admission gate")
|
||||
smoke = read(root, "smoke4.json")
|
||||
profile = read(root, "profile20.json")
|
||||
if smoke["status"] != "completed" or profile["status"] != "completed":
|
||||
raise ValueError("WSL no-repack smoke/profile did not complete")
|
||||
plan = profile["execution_plan"]
|
||||
if (plan["backend"] != "external.llamacpp.multimodal.v1"
|
||||
or plan["model_sha256"] != MODEL_SHA
|
||||
or plan["mmproj_sha256"] != MMPROJ_SHA
|
||||
or plan["server_sha256"] != SERVER_SHA
|
||||
or plan["requested_device"] != "cpu"
|
||||
or plan["offloaded_layers"] is not None
|
||||
or not plan["cpu_only_layout"]
|
||||
or plan["model_buffer_devices"]
|
||||
or plan["all_model_buffers"] != ["CPU_Mapped"]
|
||||
or plan["vision_backends"] != ["CPU"]
|
||||
or not plan["disable_repack"]
|
||||
or plan["loaded_rss_bytes"] > profile["reserved_host_ram_bytes"]
|
||||
or profile["available_host_ram_before_bytes"] < profile["reserved_host_ram_bytes"]):
|
||||
raise ValueError("WSL artifact, explicit no-repack CPU placement or admission changed")
|
||||
ledger = profile["ledger_after_shutdown"]
|
||||
if ledger["reserved"].get("host_ram") != 0 or ledger["owners"] or ledger["quarantined"]:
|
||||
raise ValueError("WSL stage did not release its reservation")
|
||||
events = [*profile["checks"]]
|
||||
timing = {}
|
||||
for kind, expected in (("text", "ready"), ("image", "Red")):
|
||||
rows = profile[f"{kind}_measured"]
|
||||
if (len(rows) != 20 or profile[f"{kind}_warmup"]["answer"] != expected
|
||||
or any(row["kind"] != kind or row["answer"] != expected for row in rows)):
|
||||
raise ValueError(f"WSL {kind} sample count/output changed")
|
||||
events.extend([profile[f"{kind}_warmup"], *rows])
|
||||
walls = [row["wall_s"] for row in rows]
|
||||
p50, p95 = rank(walls, 0.5), rank(walls, 0.95)
|
||||
if (profile[f"nearest_rank_p50_{kind}_wall_s"] != p50
|
||||
or profile[f"nearest_rank_p95_{kind}_wall_s"] != p95):
|
||||
raise ValueError(f"WSL {kind} percentile arithmetic changed")
|
||||
timing[kind] = {"n": 20, "p50_s": p50, "p95_s": p95,
|
||||
"min_s": min(walls), "max_s": max(walls)}
|
||||
if (sorted(row["stage_event"]["epoch"] for row in events) != list(range(1, 45))
|
||||
or any(not row["stage_event"]["terminal"] or row["stage_event"]["seq"] != 1
|
||||
or row["stage_event"]["worker_generation"] != plan["worker_generation"]
|
||||
for row in events)):
|
||||
raise ValueError("WSL terminal event epochs missing or duplicated")
|
||||
log = (root / "profile20.server.log").read_text(encoding="utf-8", errors="replace")
|
||||
if ("warning: no usable GPU found" not in log
|
||||
or "CPU_Mapped model buffer size" not in log
|
||||
or "CPU_REPACK model buffer size" in log
|
||||
or "CLIP using CPU backend" not in log):
|
||||
raise ValueError("WSL server log does not confirm CPU/no-repack/vision placement")
|
||||
files = sorted(path for path in root.iterdir() if path.suffix in (".json", ".log")
|
||||
and path.name != "audit_report.json")
|
||||
audit = {
|
||||
"status": "audited_scoped_e2e",
|
||||
"evidence_depth": "P: WSL CPU Omni complete text and synthetic-image requests",
|
||||
"checkpoint": "ggml-org/Qwen3.8-27B-GGUF",
|
||||
"revision": REVISION,
|
||||
"quantization": "Q4_K_M language model + Q8_0 vision projector",
|
||||
"model_sha256": MODEL_SHA,
|
||||
"mmproj_sha256": MMPROJ_SHA,
|
||||
"server_sha256": SERVER_SHA,
|
||||
"host": "Ryzen AI 9 HX 370, Ubuntu 26.04 on WSL2, 30.91 GiB WSL RAM quota",
|
||||
"runtime": "Omni checkout through Linux vLLM 0.29; llama.cpp b10849 CPU-only build, --no-repack",
|
||||
"scope": "2048 context, 128 max new tokens, concurrency one, one 96x96 red PNG and one short text instruction",
|
||||
"startup_s": profile["startup_s"],
|
||||
"reserved_host_ram_bytes": profile["reserved_host_ram_bytes"],
|
||||
"loaded_rss_bytes": plan["loaded_rss_bytes"],
|
||||
"timing": timing,
|
||||
"files": {path.name: digest(path) for path in files},
|
||||
"limits": [
|
||||
"The prompts produce two completion tokens; long-context, 128-token output and broad quality remain unqualified.",
|
||||
"No public AsyncOmni, cancellation/restart, sampled loading/request memory peak, concurrency or sustained power/thermal profile was run.",
|
||||
"The WSL quota, no-repack option and Linux llama.cpp b10849 differ from the native Windows CPU setup; latencies are not a paired OS comparison.",
|
||||
],
|
||||
}
|
||||
(root / "audit_report.json").write_text(json.dumps(audit, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"status": audit["status"], "timing": timing}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -25,6 +25,7 @@ async def main() -> None:
|
||||
parser.add_argument("--" + key.replace("_", "-"), required=True)
|
||||
parser.add_argument("--expected-device-name")
|
||||
parser.add_argument("--ggml-vk-visible-devices")
|
||||
parser.add_argument("--no-repack", action="store_true")
|
||||
parser.add_argument("--reserve-gib", type=int, default=28)
|
||||
parser.add_argument("--repeats", type=int, default=1)
|
||||
args = parser.parse_args()
|
||||
@@ -65,6 +66,7 @@ async def main() -> None:
|
||||
"device": args.device,
|
||||
"expected_device_name": args.expected_device_name,
|
||||
"ggml_vk_visible_devices": args.ggml_vk_visible_devices,
|
||||
"disable_repack": args.no_repack,
|
||||
"context_tokens": 2048,
|
||||
"max_new_tokens": 128,
|
||||
"max_io_bytes": 1 << 20,
|
||||
|
||||
@@ -56,6 +56,16 @@ artifact hashes, CPU/Radeon model and vision placement, and ledger release
|
||||
were checked. These two-token responses do not settle long-context quality,
|
||||
memory peaks, public entrypoint, cancellation or sustained performance.
|
||||
|
||||
The same pinned model also passed a [WSL CPU no-repack Omni profile](evidence/qwen38_gguf_wsl_cpu/README.md).
|
||||
Two 28 GiB pre-admission refusals and a 27 GiB loaded-RSS refusal were
|
||||
retained. Disabling llama.cpp's CPU weight repacking removed a 9,180 MiB
|
||||
buffer, allowing an explicit 24 GiB reservation within WSL's 30.91 GiB quota.
|
||||
One warmup of each type plus 20/20 text and 20/20 image requests passed at
|
||||
nearest-rank p50/p95 **1.934/1.983 s text** and **3.642/3.789 s image**.
|
||||
The Linux CPU-only server build, quota and no-repack setting differ from the
|
||||
native Windows runs, so these are distinct configurations rather than a paired
|
||||
OS speed comparison.
|
||||
|
||||
A [native Windows RTX Qwen3-TTS Omni hybrid run](evidence/qwen_tts_windows_rtx/README.md)
|
||||
adds a separate GGUF/Vulkan complete-WAV path with RTX talker/codec and CPU
|
||||
FP32 code predictor. Two named requests, 20/20 serial outputs, public audio,
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
# Qwen3.8-27B GGUF on WSL CPU
|
||||
|
||||
The same pinned [ggml-org/Qwen3.8-27B-GGUF](https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF) Q4_K_M language model and Q8_0 vision projector as the [native Windows CPU run](../qwen38_gguf_windows_cpu/README.md) passed bounded complete text and image-to-text requests through Omni `StageRuntime` on WSL2 Ubuntu 26.04. The exact checkpoint revision is `efbb3b1f70a21d97fd4495240648405f7228554f`. The local Linux CPU-only llama.cpp server was b10849, commit `e71b80510`, SHA256 `0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c`, loaded by the current checkout with installed vLLM 0.29.0. [Hardware/runtime record](hardware_runtime.json).
|
||||
|
||||
The WSL RAM quota is 30.91 GiB. Two [pre-admission](smoke.json) [attempts](smoke2.json) refused a 28 GiB reservation when available RAM after Python imports was below it. A [27 GiB attempt](smoke3.json) loaded the model but was correctly refused because its CPU-repacked process RSS exceeded that reservation. The explicit `--no-repack` option removed the 9,180 MiB repack buffer without changing weights or precision. A [24 GiB smoke](smoke4.json) then passed `ready` text and `Red` synthetic-image answers; the stage measured 20,026,560,512 bytes of loaded process RSS, verified `CPU_Mapped` model storage and `CLIP using CPU backend`, and fully released the reservation. The CPU-only build emits `warning: no usable GPU found` rather than an offload-count line, so the backend requires positive CPU-buffer evidence before labeling CPU placement.
|
||||
|
||||
The [serial profile](profile20.json) used one 2048-token slot, a 128-token maximum output, deterministic temperature zero, a 96×96 solid-red PNG, and short prompts requesting `ready` and `Red`. After one warmup of each type, **20/20 text** and **20/20 image** requests returned the expected two-token answers. Nearest-rank complete-request p50/p95 was **1.934/1.983 s text** and **3.642/3.789 s image**, concurrency one. Startup was 11.45 s on this run and is excluded from those percentiles; the earlier smoke startup was 25.01 s, so cache state should be treated as uncontrolled. The [server log](profile20.server.log) and [auditor](audit_report.json) verify hashes, CPU model/vision placement, every output and terminal epoch, percentile arithmetic and ledger release.
|
||||
|
||||
Reproduce with [the Omni probe](../../../../experiments/probe_omni_llamacpp_qwen38.py), the checkpoint hashes in the Windows CPU record, `--device cpu --reserve-gib 24 --no-repack --repeats 20`, the Linux server hash above, and fresh log/report paths. Use the vLLM 0.29 environment, `PYTHONPATH` pointing to this checkout, `LD_LIBRARY_PATH` pointing to the llama.cpp build directory, `CUDA_VISIBLE_DEVICES=''`, and `VLLM_ENABLE_V1_MULTIPROCESSING=0`. Run [the auditor](../../../../experiments/audit_qwen38_gguf_wsl.py) against this evidence directory. This is scoped P-level evidence for short text and one synthetic image, not long-context or broad visual quality, public AsyncOmni, cancellation/restart, concurrent serving, loading/request memory peaks or sustained power and thermal behavior. The WSL quota and no-repack setting differ from native Windows, so their latencies are not a paired OS comparison.
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"status": "audited_scoped_e2e",
|
||||
"evidence_depth": "P: WSL CPU Omni complete text and synthetic-image requests",
|
||||
"checkpoint": "ggml-org/Qwen3.8-27B-GGUF",
|
||||
"revision": "efbb3b1f70a21d97fd4495240648405f7228554f",
|
||||
"quantization": "Q4_K_M language model + Q8_0 vision projector",
|
||||
"model_sha256": "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747",
|
||||
"mmproj_sha256": "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb",
|
||||
"server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c",
|
||||
"host": "Ryzen AI 9 HX 370, Ubuntu 26.04 on WSL2, 30.91 GiB WSL RAM quota",
|
||||
"runtime": "Omni checkout through Linux vLLM 0.29; llama.cpp b10849 CPU-only build, --no-repack",
|
||||
"scope": "2048 context, 128 max new tokens, concurrency one, one 96x96 red PNG and one short text instruction",
|
||||
"startup_s": 11.45404783997219,
|
||||
"reserved_host_ram_bytes": 25769803776,
|
||||
"loaded_rss_bytes": 20026531840,
|
||||
"timing": {
|
||||
"text": {
|
||||
"n": 20,
|
||||
"p50_s": 1.9337806559633464,
|
||||
"p95_s": 1.983029014954809,
|
||||
"min_s": 1.8979981419979595,
|
||||
"max_s": 2.0445767869823612
|
||||
},
|
||||
"image": {
|
||||
"n": 20,
|
||||
"p50_s": 3.6416604099795222,
|
||||
"p95_s": 3.788667813001666,
|
||||
"min_s": 3.564662608958315,
|
||||
"max_s": 3.81039645295823
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
"hardware_runtime.json": {
|
||||
"bytes": 627,
|
||||
"sha256": "df65f32090c0f0ff3bf3db6cb2b8e3060138ee3136bff9074d4da60f3eec3958"
|
||||
},
|
||||
"profile20.driver.log": {
|
||||
"bytes": 608,
|
||||
"sha256": "ecc46c7d39d9d8e84cd860579d48b305fecb2a5fd50b1e2475865e82ca84f849"
|
||||
},
|
||||
"profile20.json": {
|
||||
"bytes": 32936,
|
||||
"sha256": "d81554c7f7d8e9e782f5b99b0716a5ca60d3fd1eacd0a5366e413ca6f8931b64"
|
||||
},
|
||||
"profile20.server.log": {
|
||||
"bytes": 144420,
|
||||
"sha256": "0324cff765b7ca890ad994b66a634233ce4ff19a956ea1d54c2c0402d7fa056e"
|
||||
},
|
||||
"smoke.driver.log": {
|
||||
"bytes": 1541,
|
||||
"sha256": "3e9046f4b6cf81c9510e73d3e61d48dc4cdee6414ab81e74950f5aa4a9e8f345"
|
||||
},
|
||||
"smoke.json": {
|
||||
"bytes": 557,
|
||||
"sha256": "d05ba97947654feed445b47cd5d043500f452612d0dd97c0f797181c4e8324cf"
|
||||
},
|
||||
"smoke2.driver.log": {
|
||||
"bytes": 1541,
|
||||
"sha256": "7d6e4c2bf144315044991bbcf7dc5ef6e1d6e1927d693f86b9df188d1e0fc9e4"
|
||||
},
|
||||
"smoke2.json": {
|
||||
"bytes": 557,
|
||||
"sha256": "fb197382bc717af111c54d26464f682897f7fdbd3cb4aafea2b1b5b7377c3486"
|
||||
},
|
||||
"smoke3.driver.log": {
|
||||
"bytes": 6058,
|
||||
"sha256": "402d41f730c3cb64d6caea2522f39263d231139d9b12f78001a1e81809d70da9"
|
||||
},
|
||||
"smoke3.json": {
|
||||
"bytes": 728,
|
||||
"sha256": "6e472ab0816f9c9b1cfe220636b318c1697c94386276869b58bb80c14c464591"
|
||||
},
|
||||
"smoke3.server.log": {
|
||||
"bytes": 19265,
|
||||
"sha256": "cc6919d965657beb5ea5ef279a590449293b6121cf8ea42689f3e8a1e20ff047"
|
||||
},
|
||||
"smoke4.driver.log": {
|
||||
"bytes": 608,
|
||||
"sha256": "88e7a14179c1ecc860b0e4a7c1a74081cb84fc424f42d6a0af0ff109804a7b54"
|
||||
},
|
||||
"smoke4.json": {
|
||||
"bytes": 6366,
|
||||
"sha256": "24dd653a4ffaf392a828a394d243758af83c194f97606f6e064a45319d8f73db"
|
||||
},
|
||||
"smoke4.server.log": {
|
||||
"bytes": 35797,
|
||||
"sha256": "6f765837700c4210378d4128fae8781e3a7fe0616513ce4d88eac073307910ca"
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"The prompts produce two completion tokens; long-context, 128-token output and broad quality remain unqualified.",
|
||||
"No public AsyncOmni, cancellation/restart, sampled loading/request memory peak, concurrency or sustained power/thermal profile was run.",
|
||||
"The WSL quota, no-repack option and Linux llama.cpp b10849 differ from the native Windows CPU setup; latencies are not a paired OS comparison."
|
||||
]
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"captured_utc": "2026-09-24T10:12:23Z",
|
||||
"os": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43",
|
||||
"kernel": "6.18.33.2-microsoft-standard-WSL2",
|
||||
"cpu": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M",
|
||||
"wsl_memory_total_bytes": 33184976896,
|
||||
"wsl_memory_available_after_profile_bytes": 31320154112,
|
||||
"vllm_package": "0.29.0",
|
||||
"omni_package": "0.29.0rc2.dev71+g00510ce38.d20260922",
|
||||
"llama_cpp_build": 10849,
|
||||
"llama_cpp_commit": "e71b80510",
|
||||
"server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c",
|
||||
"gpu_visibility": "CUDA_VISIBLE_DEVICES empty; server -dev none -ngl 0"
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
INFO 09-24 18:09:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 18:09:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 18:09:18 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
INFO 09-24 18:09:19 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 18:09:19 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
|
||||
{"status": "completed", "error": null}
|
||||
+1300
File diff suppressed because it is too large.
Load diff
+1452
File diff suppressed because it is too large.
Load diff
+19
@@ -0,0 +1,19 @@
|
||||
INFO 09-24 18:04:39 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 18:04:39 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 18:04:39 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
{"status": "failed", "error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation"}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 151, in <module>
|
||||
asyncio.run(main())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run
|
||||
return runner.run(main)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run
|
||||
return self._loop.run_until_complete(task)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 55, in main
|
||||
raise MemoryError("available native host RAM is below the declared Qwen3.8 reservation")
|
||||
MemoryError: available native host RAM is below the declared Qwen3.8 reservation
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp",
|
||||
"device": "cpu",
|
||||
"available_host_ram_before_bytes": 29981622272,
|
||||
"reserved_host_ram_bytes": 30064771072,
|
||||
"repeats": 1,
|
||||
"model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf",
|
||||
"mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf",
|
||||
"status": "failed",
|
||||
"error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation"
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
INFO 09-24 18:05:34 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 18:05:34 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 18:05:34 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
{"status": "failed", "error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation"}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 151, in <module>
|
||||
asyncio.run(main())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run
|
||||
return runner.run(main)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run
|
||||
return self._loop.run_until_complete(task)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 55, in main
|
||||
raise MemoryError("available native host RAM is below the declared Qwen3.8 reservation")
|
||||
MemoryError: available native host RAM is below the declared Qwen3.8 reservation
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp",
|
||||
"device": "cpu",
|
||||
"available_host_ram_before_bytes": 29934391296,
|
||||
"reserved_host_ram_bytes": 30064771072,
|
||||
"repeats": 1,
|
||||
"model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf",
|
||||
"mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf",
|
||||
"status": "failed",
|
||||
"error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation"
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
INFO 09-24 18:06:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 18:06:08 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 18:06:08 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
INFO 09-24 18:06:09 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 18:06:09 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s)
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 817, in _initialize_replica
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] return create_graph_client(
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/__init__.py", line 19, in create_graph_client
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] return LlamaCppMultimodalStageClient(metadata, backend, ledger, reservation)
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 466, in __init__
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] super().__init__(metadata, config, ledger, reservation)
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 224, in __init__
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] raise ResourceUnavailable("loaded llama.cpp process RSS exceeds stage reservation")
|
||||
ERROR 09-24 18:06:53 [stage_runtime.py:265] vllm_omni.engine.resource_ledger.ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation
|
||||
{"status": "failed", "error": "ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation"}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 151, in <module>
|
||||
asyncio.run(main())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run
|
||||
return runner.run(main)
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run
|
||||
return self._loop.run_until_complete(task)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 86, in main
|
||||
runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 817, in _initialize_replica
|
||||
return create_graph_client(
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/__init__.py", line 19, in create_graph_client
|
||||
return LlamaCppMultimodalStageClient(metadata, backend, ledger, reservation)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 466, in __init__
|
||||
super().__init__(metadata, config, ledger, reservation)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 224, in __init__
|
||||
raise ResourceUnavailable("loaded llama.cpp process RSS exceeds stage reservation")
|
||||
vllm_omni.engine.resource_ledger.ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp",
|
||||
"device": "cpu",
|
||||
"available_host_ram_before_bytes": 29994790912,
|
||||
"reserved_host_ram_bytes": 28991029248,
|
||||
"repeats": 1,
|
||||
"model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf",
|
||||
"mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf",
|
||||
"status": "failed",
|
||||
"error": "ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 29994790912
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
warning: no usable GPU found, --gpu-layers option will be ignored
|
||||
warning: one possible reason is that llama.cpp was compiled without GPU support
|
||||
warning: consult docs/build.md for compilation instructions
|
||||
0.00.004.616 I cmn common_param: common_params_print_info: build 10849 (e71b80510) with GNU 11.4.0 for Linux x86_64
|
||||
0.00.004.645 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg)
|
||||
0.00.004.645 I cmn common_param: device_info:
|
||||
0.00.004.697 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (31647 MiB, 31647 MiB free)
|
||||
0.00.004.773 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
||||
0.00.004.907 I srv init: using 23 threads for HTTP server
|
||||
0.00.004.927 I srv init: The UI is disabled
|
||||
0.00.004.927 I srv init: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable
|
||||
0.00.004.987 W srv llama_server: -----------------
|
||||
0.00.005.006 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set
|
||||
0.00.005.006 W srv llama_server: this can be a security risk (cross-origin attacks)
|
||||
0.00.005.006 W srv llama_server: more info: https://github.com/ggml-org/llama.cpp/pull/25655
|
||||
0.00.005.006 W srv llama_server: -----------------
|
||||
0.00.005.028 I srv start: binding port with default address family
|
||||
0.00.007.077 I srv load_model: loading model '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf'
|
||||
0.00.007.097 I srv load_model: local path '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf'
|
||||
0.00.070.920 I llama_model_loader: loaded meta data with 40 key-value pairs and 851 tensors from /home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf (version GGUF V3 (latest))
|
||||
0.00.071.014 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
0.00.071.027 I llama_model_loader: - kv 0: general.architecture str = qwen35
|
||||
0.00.071.028 I llama_model_loader: - kv 1: general.type str = model
|
||||
0.00.071.034 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 20
|
||||
0.00.071.041 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000
|
||||
0.00.071.042 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000
|
||||
0.00.071.042 I llama_model_loader: - kv 5: general.name str = Qwen3.8-27B
|
||||
0.00.071.043 I llama_model_loader: - kv 6: general.size_label str = 27B
|
||||
0.00.071.043 I llama_model_loader: - kv 7: general.license str = apache-2.0
|
||||
0.00.071.068 I llama_model_loader: - kv 8: general.tags arr[str,1] = ["image-text-to-text"]
|
||||
0.00.071.069 I llama_model_loader: - kv 9: qwen35.block_count u32 = 64
|
||||
0.00.071.070 I llama_model_loader: - kv 10: qwen35.context_length u32 = 262144
|
||||
0.00.071.070 I llama_model_loader: - kv 11: qwen35.embedding_length u32 = 5120
|
||||
0.00.071.071 I llama_model_loader: - kv 12: qwen35.feed_forward_length u32 = 17408
|
||||
0.00.071.071 I llama_model_loader: - kv 13: qwen35.attention.head_count u32 = 24
|
||||
0.00.071.071 I llama_model_loader: - kv 14: qwen35.attention.head_count_kv u32 = 4
|
||||
0.00.071.074 I llama_model_loader: - kv 15: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0]
|
||||
0.00.071.076 I llama_model_loader: - kv 16: qwen35.rope.freq_base f32 = 10000000.000000
|
||||
0.00.071.077 I llama_model_loader: - kv 17: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
0.00.071.078 I llama_model_loader: - kv 18: qwen35.attention.key_length u32 = 256
|
||||
0.00.071.078 I llama_model_loader: - kv 19: qwen35.attention.value_length u32 = 256
|
||||
0.00.071.079 I llama_model_loader: - kv 20: qwen35.ssm.conv_kernel u32 = 4
|
||||
0.00.071.079 I llama_model_loader: - kv 21: qwen35.ssm.state_size u32 = 128
|
||||
0.00.071.079 I llama_model_loader: - kv 22: qwen35.ssm.group_count u32 = 16
|
||||
0.00.071.079 I llama_model_loader: - kv 23: qwen35.ssm.time_step_rank u32 = 48
|
||||
0.00.071.080 I llama_model_loader: - kv 24: qwen35.ssm.inner_size u32 = 6144
|
||||
0.00.071.084 I llama_model_loader: - kv 25: qwen35.attention.recurrent_layers arr[bool,64] = [true, true, true, false, true, true,...
|
||||
0.00.071.084 I llama_model_loader: - kv 26: qwen35.full_attention_interval u32 = 4
|
||||
0.00.071.084 I llama_model_loader: - kv 27: qwen35.rope.dimension_count u32 = 64
|
||||
0.00.071.085 I llama_model_loader: - kv 28: tokenizer.ggml.model str = gpt2
|
||||
0.00.071.085 I llama_model_loader: - kv 29: tokenizer.ggml.pre str = qwen35
|
||||
0.00.089.105 I llama_model_loader: - kv 30: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
0.00.092.472 I llama_model_loader: - kv 31: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
0.00.108.554 I llama_model_loader: - kv 32: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
|
||||
0.00.108.593 I llama_model_loader: - kv 33: tokenizer.ggml.eos_token_id u32 = 248046
|
||||
0.00.108.594 I llama_model_loader: - kv 34: tokenizer.ggml.padding_token_id u32 = 248044
|
||||
0.00.108.594 I llama_model_loader: - kv 35: tokenizer.ggml.bos_token_id u32 = 248044
|
||||
0.00.108.595 I llama_model_loader: - kv 36: tokenizer.ggml.add_bos_token bool = false
|
||||
0.00.108.599 I llama_model_loader: - kv 37: tokenizer.chat_template str = {%- set image_count = namespace(value...
|
||||
0.00.108.599 I llama_model_loader: - kv 38: general.quantization_version u32 = 2
|
||||
0.00.108.600 I llama_model_loader: - kv 39: general.file_type u32 = 15
|
||||
0.00.108.608 I llama_model_loader: - type f32: 353 tensors
|
||||
0.00.108.609 I llama_model_loader: - type q8_0: 288 tensors
|
||||
0.00.108.609 I llama_model_loader: - type q4_K: 193 tensors
|
||||
0.00.108.609 I llama_model_loader: - type q6_K: 17 tensors
|
||||
0.00.108.613 I print_info: file format = GGUF V3 (latest)
|
||||
0.00.108.616 I print_info: file type = Q4_K - Medium
|
||||
0.00.108.621 I print_info: file size = 17.66 GiB (5.64 BPW)
|
||||
0.00.339.323 I load: 0 unused tokens
|
||||
0.00.404.119 I load: printing all EOG tokens:
|
||||
0.00.404.155 I load: - 248044 ('<|endoftext|>')
|
||||
0.00.404.157 I load: - 248046 ('<|im_end|>')
|
||||
0.00.404.158 I load: - 248063 ('<|fim_pad|>')
|
||||
0.00.404.158 I load: - 248064 ('<|repo_name|>')
|
||||
0.00.404.158 I load: - 248065 ('<|file_sep|>')
|
||||
0.00.404.429 I load: special tokens cache size = 33
|
||||
0.00.461.091 I load: token to piece cache size = 1.7581 MB
|
||||
0.00.461.146 I print_info: arch = qwen35
|
||||
0.00.461.147 I print_info: vocab_only = 0
|
||||
0.00.461.147 I print_info: no_alloc = 0
|
||||
0.00.461.148 I print_info: n_ctx_train = 262144
|
||||
0.00.461.149 I print_info: n_embd_inp = 5120
|
||||
0.00.461.150 I print_info: n_embd = 5120
|
||||
0.00.461.151 I print_info: n_embd_out = 5120
|
||||
0.00.461.151 I print_info: n_layer = 64
|
||||
0.00.461.151 I print_info: n_layer_all = 64
|
||||
0.00.461.169 I print_info: n_head = 24
|
||||
0.00.461.171 I print_info: n_head_kv = 4
|
||||
0.00.461.171 I print_info: n_rot = 64
|
||||
0.00.461.171 I print_info: n_swa = 0
|
||||
0.00.461.173 I print_info: is_swa_any = 0
|
||||
0.00.461.173 I print_info: non_causal_type = 0
|
||||
0.00.461.173 I print_info: n_embd_head_k = 256
|
||||
0.00.461.173 I print_info: n_embd_head_v = 256
|
||||
0.00.461.175 I print_info: n_gqa = 6
|
||||
0.00.461.181 I print_info: n_embd_k_gqa = 1024
|
||||
0.00.461.184 I print_info: n_embd_v_gqa = 1024
|
||||
0.00.461.185 I print_info: f_norm_eps = 0.0e+00
|
||||
0.00.461.187 I print_info: f_norm_rms_eps = 1.0e-06
|
||||
0.00.461.187 I print_info: f_clamp_kqv = 0.0e+00
|
||||
0.00.461.187 I print_info: f_max_alibi_bias = 0.0e+00
|
||||
0.00.461.187 I print_info: f_logit_scale = 0.0e+00
|
||||
0.00.461.188 I print_info: f_attn_scale = 0.0e+00
|
||||
0.00.461.188 I print_info: f_attn_value_scale = 0.0000
|
||||
0.00.461.190 I print_info: n_ff = 17408
|
||||
0.00.461.190 I print_info: n_expert = 0
|
||||
0.00.461.190 I print_info: n_expert_used = 0
|
||||
0.00.461.190 I print_info: n_expert_groups = 0
|
||||
0.00.461.191 I print_info: n_group_used = 0
|
||||
0.00.461.191 I print_info: causal attn = 1
|
||||
0.00.461.191 I print_info: pooling type = -1
|
||||
0.00.461.191 I print_info: rope type = 40
|
||||
0.00.461.192 I print_info: rope scaling = linear
|
||||
0.00.461.193 I print_info: freq_base_train = 10000000.0
|
||||
0.00.461.193 I print_info: freq_scale_train = 1
|
||||
0.00.461.193 I print_info: n_ctx_orig_yarn = 262144
|
||||
0.00.461.193 I print_info: rope_yarn_log_mul = 0.0000
|
||||
0.00.461.193 I print_info: rope_finetuned = unknown
|
||||
0.00.461.194 I print_info: mrope sections = [11, 11, 10, 0]
|
||||
0.00.461.194 I print_info: ssm_d_conv = 4
|
||||
0.00.461.194 I print_info: ssm_d_inner = 6144
|
||||
0.00.461.194 I print_info: ssm_d_state = 128
|
||||
0.00.461.194 I print_info: ssm_dt_rank = 48
|
||||
0.00.461.194 I print_info: ssm_n_group = 16
|
||||
0.00.461.194 I print_info: ssm_dt_b_c_rms = 0
|
||||
0.00.461.195 I print_info: model type = 27B
|
||||
0.00.461.196 I print_info: model params = 26.90 B
|
||||
0.00.461.196 I print_info: general.name = Qwen3.8-27B
|
||||
0.00.461.201 I print_info: vocab type = BPE
|
||||
0.00.461.202 I print_info: n_vocab = 248320
|
||||
0.00.461.202 I print_info: n_merges = 247587
|
||||
0.00.461.203 I print_info: BOS token = 248044 '<|endoftext|>'
|
||||
0.00.461.203 I print_info: EOS token = 248046 '<|im_end|>'
|
||||
0.00.461.203 I print_info: EOT token = 248046 '<|im_end|>'
|
||||
0.00.461.203 I print_info: PAD token = 248044 '<|endoftext|>'
|
||||
0.00.461.204 I print_info: LF token = 198 'Ċ'
|
||||
0.00.461.204 I print_info: FIM PRE token = 248060 '<|fim_prefix|>'
|
||||
0.00.461.204 I print_info: FIM SUF token = 248062 '<|fim_suffix|>'
|
||||
0.00.461.204 I print_info: FIM MID token = 248061 '<|fim_middle|>'
|
||||
0.00.461.204 I print_info: FIM PAD token = 248063 '<|fim_pad|>'
|
||||
0.00.461.204 I print_info: FIM REP token = 248064 '<|repo_name|>'
|
||||
0.00.461.204 I print_info: FIM SEP token = 248065 '<|file_sep|>'
|
||||
0.00.461.205 I print_info: EOG token = 248044 '<|endoftext|>'
|
||||
0.00.461.206 I print_info: EOG token = 248046 '<|im_end|>'
|
||||
0.00.461.217 I print_info: EOG token = 248063 '<|fim_pad|>'
|
||||
0.00.461.218 I print_info: EOG token = 248064 '<|repo_name|>'
|
||||
0.00.461.218 I print_info: EOG token = 248065 '<|file_sep|>'
|
||||
0.00.461.218 I print_info: max token length = 256
|
||||
0.00.461.222 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap)
|
||||
AMX is not ready to be used!
|
||||
0.00.479.707 I load_tensors: CPU_Mapped model buffer size = 18084.41 MiB
|
||||
0.00.479.737 I load_tensors: CPU_REPACK model buffer size = 9180.00 MiB
|
||||
0.14.394.812 I cmn common_init_: added <|endoftext|> logit bias = -inf
|
||||
0.14.394.847 I cmn common_init_: added <|im_end|> logit bias = -inf
|
||||
0.14.394.848 I cmn common_init_: added <|fim_pad|> logit bias = -inf
|
||||
0.14.394.848 I cmn common_init_: added <|repo_name|> logit bias = -inf
|
||||
0.14.394.849 I cmn common_init_: added <|file_sep|> logit bias = -inf
|
||||
0.14.394.957 I llama_context: constructing llama_context
|
||||
0.14.394.962 I llama_context: n_seq_max = 1
|
||||
0.14.394.962 I llama_context: n_ctx = 2048
|
||||
0.14.394.962 I llama_context: n_ctx_seq = 2048
|
||||
0.14.394.962 I llama_context: n_batch = 2048
|
||||
0.14.394.963 I llama_context: n_ubatch = 512
|
||||
0.14.394.963 I llama_context: causal_attn = 1
|
||||
0.14.394.965 I llama_context: flash_attn = auto
|
||||
0.14.394.965 I llama_context: kv_unified = false
|
||||
0.14.394.968 I llama_context: freq_base = 10000000.0
|
||||
0.14.394.969 I llama_context: freq_scale = 1
|
||||
0.14.394.969 I llama_context: n_rs_seq = 0
|
||||
0.14.394.969 I llama_context: n_outputs_max = 1
|
||||
0.14.394.969 I llama_context: n_outputs_max_per_seq = 1
|
||||
0.14.394.970 I llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
|
||||
0.14.395.033 I llama_context: CPU output buffer size = 0.95 MiB
|
||||
0.14.395.178 I llama_kv_cache: CPU KV buffer size = 128.00 MiB
|
||||
0.14.465.763 I llama_kv_cache: size = 128.00 MiB ( 2048 cells, 16 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB
|
||||
0.14.465.794 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256
|
||||
0.14.465.794 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256
|
||||
0.14.522.121 I llama_memory_recurrent: CPU RS buffer size = 149.62 MiB
|
||||
0.14.522.174 I llama_memory_recurrent: size = 149.62 MiB ( 1 cells, 64 layers, 1 seqs 0 rs_seq), R (f32): 5.62 MiB, S (f32): 144.00 MiB, P (f32): 0.00 MiB
|
||||
0.14.522.198 I sched_reserve: reserving ...
|
||||
0.14.530.268 I resolve_fused_ops: Flash Attention enabled
|
||||
0.14.530.294 I resolve_fused_ops: resolving fused DeepSeek V4 HC support:
|
||||
0.14.530.800 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled
|
||||
0.14.531.293 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled
|
||||
0.14.531.806 I resolve_fused_ops: fused DeepSeek V4 HC post enabled
|
||||
0.14.534.131 I sched_reserve: CPU compute buffer size = 134.02 MiB
|
||||
0.14.534.153 I sched_reserve: graph nodes = 3751
|
||||
0.14.534.154 I sched_reserve: graph splits = 1
|
||||
0.14.534.156 I sched_reserve: reserve took 11.95 ms, sched copies = 1
|
||||
0.14.534.221 I cmn init: llama threadpool init, n_threads = 12
|
||||
0.14.534.272 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable)
|
||||
0.15.294.041 I clip_model_loader: model name: Qwen3.8-27B
|
||||
0.15.294.069 I clip_model_loader: description:
|
||||
0.15.294.073 I clip_model_loader: GGUF version: 3
|
||||
0.15.294.075 I clip_model_loader: alignment: 32
|
||||
0.15.294.076 I clip_model_loader: n_tensors: 334
|
||||
0.15.294.077 I clip_model_loader: n_kv: 26
|
||||
0.15.294.078 I
|
||||
0.15.294.082 I clip_model_loader: has vision encoder
|
||||
0.15.294.369 I clip_ctx: CLIP using CPU backend
|
||||
0.15.294.628 W load_hparams: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks
|
||||
0.15.294.650 W load_hparams: if you encounter problems with accuracy, try adding --image-min-tokens 1024
|
||||
0.15.294.650 W load_hparams: more info: https://github.com/ggml-org/llama.cpp/issues/16842
|
||||
|
||||
0.15.294.651 I load_hparams: projector: qwen3vl_merger
|
||||
0.15.294.651 I load_hparams: n_embd: 1152
|
||||
0.15.294.651 I load_hparams: n_head: 16
|
||||
0.15.294.651 I load_hparams: n_ff: 4304
|
||||
0.15.294.651 I load_hparams: n_layer: 27
|
||||
0.15.294.652 I load_hparams: ffn_op: gelu
|
||||
0.15.294.652 I load_hparams: projection_dim: 5120
|
||||
0.15.294.652 I
|
||||
--- vision hparams ---
|
||||
0.15.294.652 I load_hparams: image_size: 768
|
||||
0.15.294.653 I load_hparams: patch_size: 16
|
||||
0.15.294.653 I load_hparams: has_llava_proj: 0
|
||||
0.15.294.653 I load_hparams: minicpmv_version: 0
|
||||
0.15.294.653 I load_hparams: n_merge: 2
|
||||
0.15.294.653 I load_hparams: n_wa_pattern: 0
|
||||
0.15.294.654 I load_hparams: image_min_pixels: 8192
|
||||
0.15.294.654 I load_hparams: image_max_pixels: 4194304
|
||||
0.15.294.654 I
|
||||
0.15.294.659 I load_hparams: model size: 600.08 MiB
|
||||
0.15.294.661 I load_hparams: metadata size: 0.12 MiB
|
||||
0.16.644.354 I get_dummy_batch: warmup with image size = 1472 x 1472
|
||||
0.16.662.404 I get_dummy_batch: warmup with image size = 1472 x 1472
|
||||
0.16.672.021 I reserve_compute_meta: CPU compute buffer size = 248.10 MiB
|
||||
0.16.672.058 I reserve_compute_meta: graph splits = 1, nodes = 823
|
||||
0.16.672.081 I warmup: flash attention is enabled
|
||||
0.16.675.000 I srv load_model: loaded multimodal model, '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf'
|
||||
0.19.587.564 I cmn common_conte: the context does not support partial sequence removal
|
||||
0.19.598.280 I srv load_model: speculative decoding will use checkpoints
|
||||
0.19.598.317 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 2048, kv_unified = 'false'
|
||||
0.19.598.350 I spec common_specu: no implementations specified for speculative decoding
|
||||
0.19.598.354 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048
|
||||
0.19.598.453 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it
|
||||
0.19.598.472 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391
|
||||
0.19.598.473 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192
|
||||
0.19.598.500 W srv init: --cache-idle-slots requires --cache-ram, disabling
|
||||
0.19.606.193 I srv init: init: chat template, example_format: '<|im_start|>system
|
||||
You are a helpful assistant<|im_end|>
|
||||
<|im_start|>user
|
||||
Hello<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
Hi there<|im_end|>
|
||||
<|im_start|>user
|
||||
How are you?<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
'
|
||||
0.19.606.732 I srv init: init: chat template, thinking = 0
|
||||
0.19.606.772 I srv init: preserve_reasoning kwarg: true
|
||||
0.19.606.772 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve)
|
||||
0.19.606.829 I srv llama_server: model loaded
|
||||
0.19.606.853 I srv llama_server: listening on http://127.0.0.1:33725
|
||||
0.19.606.995 I srv update_slots: all slots are idle
|
||||
0.19.770.089 I srv operator(): operator(): cleaning up before exit...
|
||||
0.19.771.302 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
|
||||
0.19.771.323 I common_memory_breakdown_print: | - Host | 18496 = 18084 + 277 + 134 |
|
||||
0.19.771.323 I common_memory_breakdown_print: | - CPU_REPACK | 9180 = 9180 + 0 + 0 |
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
INFO 09-24 18:07:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-24 18:07:56 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-24 18:07:56 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
INFO 09-24 18:07:57 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
WARNING 09-24 18:07:57 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge'
|
||||
{"status": "completed", "error": null}
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp",
|
||||
"device": "cpu",
|
||||
"available_host_ram_before_bytes": 29979381760,
|
||||
"reserved_host_ram_bytes": 25769803776,
|
||||
"repeats": 1,
|
||||
"model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf",
|
||||
"mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf",
|
||||
"startup_s": 25.010701490973588,
|
||||
"execution_plan": {
|
||||
"backend": "external.llamacpp.multimodal.v1",
|
||||
"stage_id": 0,
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"model_sha256": "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747",
|
||||
"mmproj_sha256": "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb",
|
||||
"server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c",
|
||||
"model_alias": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf",
|
||||
"requested_device": "cpu",
|
||||
"expected_device_name": null,
|
||||
"layer_assignments": 0,
|
||||
"offloaded_layers": null,
|
||||
"model_buffer_devices": [],
|
||||
"all_model_buffers": [
|
||||
"CPU_Mapped"
|
||||
],
|
||||
"vision_backends": [
|
||||
"CPU"
|
||||
],
|
||||
"cpu_only_layout": true,
|
||||
"worker_pid": 346310,
|
||||
"loaded_rss_bytes": 20026560512,
|
||||
"reserved_bytes": {
|
||||
"host_ram": 25769803776
|
||||
},
|
||||
"memory_pool": "host_ram",
|
||||
"memory_overhead_bytes": 4294967296,
|
||||
"disable_repack": true,
|
||||
"context_tokens": 2048,
|
||||
"max_new_tokens": 128,
|
||||
"max_io_bytes": 1048576,
|
||||
"max_image_bytes": 262144,
|
||||
"image_token_reserve": 512,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "llama.cpp-owned; reset for each complete request",
|
||||
"evidence": "B"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 1.958196632040199,
|
||||
"answer": "ready",
|
||||
"expected": "ready",
|
||||
"stage_event": {
|
||||
"request_id": "text-493858057328167",
|
||||
"stage_id": 0,
|
||||
"epoch": 1,
|
||||
"seq": 1,
|
||||
"kind": "text",
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"llamacpp_wall_s": 1.946947110001929,
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 19,
|
||||
"total_tokens": 21,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 3.556307617982384,
|
||||
"answer": "Red",
|
||||
"expected": "Red",
|
||||
"stage_event": {
|
||||
"request_id": "image-493860015612375",
|
||||
"stage_id": 0,
|
||||
"epoch": 2,
|
||||
"seq": 1,
|
||||
"kind": "text",
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"llamacpp_wall_s": 3.5440641850000247,
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 37,
|
||||
"total_tokens": 39,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"text_warmup": {
|
||||
"kind": "text",
|
||||
"wall_s": 1.9051429200335406,
|
||||
"answer": "ready",
|
||||
"expected": "ready",
|
||||
"stage_event": {
|
||||
"request_id": "text-493863571993703",
|
||||
"stage_id": 0,
|
||||
"epoch": 3,
|
||||
"seq": 1,
|
||||
"kind": "text",
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"llamacpp_wall_s": 1.8953040880151093,
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 19,
|
||||
"total_tokens": 21,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"text_measured": [
|
||||
{
|
||||
"kind": "text",
|
||||
"wall_s": 1.9479502299800515,
|
||||
"answer": "ready",
|
||||
"expected": "ready",
|
||||
"stage_event": {
|
||||
"request_id": "text-493865477197598",
|
||||
"stage_id": 0,
|
||||
"epoch": 4,
|
||||
"seq": 1,
|
||||
"kind": "text",
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"llamacpp_wall_s": 1.939045864972286,
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 19,
|
||||
"total_tokens": 21,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"nearest_rank_p50_text_wall_s": 1.9479502299800515,
|
||||
"nearest_rank_p95_text_wall_s": 1.9479502299800515,
|
||||
"image_warmup": {
|
||||
"kind": "image",
|
||||
"wall_s": 3.6417471549939364,
|
||||
"answer": "Red",
|
||||
"expected": "Red",
|
||||
"stage_event": {
|
||||
"request_id": "image-493867425252186",
|
||||
"stage_id": 0,
|
||||
"epoch": 5,
|
||||
"seq": 1,
|
||||
"kind": "text",
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"llamacpp_wall_s": 3.629429625987541,
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 37,
|
||||
"total_tokens": 39,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"image_measured": [
|
||||
{
|
||||
"kind": "image",
|
||||
"wall_s": 3.647124751994852,
|
||||
"answer": "Red",
|
||||
"expected": "Red",
|
||||
"stage_event": {
|
||||
"request_id": "image-493871067080776",
|
||||
"stage_id": 0,
|
||||
"epoch": 6,
|
||||
"seq": 1,
|
||||
"kind": "text",
|
||||
"worker_generation": "b9aa9595fd0f48b081c982d1cb597a69",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"llamacpp_wall_s": 3.640121451986488,
|
||||
"usage": {
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens": 37,
|
||||
"total_tokens": 39,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"nearest_rank_p50_image_wall_s": 3.647124751994852,
|
||||
"nearest_rank_p95_image_wall_s": 3.647124751994852,
|
||||
"status": "completed",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 29979381760
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
warning: no usable GPU found, --gpu-layers option will be ignored
|
||||
warning: one possible reason is that llama.cpp was compiled without GPU support
|
||||
warning: consult docs/build.md for compilation instructions
|
||||
0.00.003.118 I cmn common_param: common_params_print_info: build 10849 (e71b80510) with GNU 11.4.0 for Linux x86_64
|
||||
0.00.003.141 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg)
|
||||
0.00.003.142 I cmn common_param: device_info:
|
||||
0.00.003.155 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (31647 MiB, 31647 MiB free)
|
||||
0.00.003.224 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
||||
0.00.003.403 I srv init: using 23 threads for HTTP server
|
||||
0.00.003.422 I srv init: The UI is disabled
|
||||
0.00.003.423 I srv init: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable
|
||||
0.00.003.481 W srv llama_server: -----------------
|
||||
0.00.003.499 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set
|
||||
0.00.003.499 W srv llama_server: this can be a security risk (cross-origin attacks)
|
||||
0.00.003.499 W srv llama_server: more info: https://github.com/ggml-org/llama.cpp/pull/25655
|
||||
0.00.003.500 W srv llama_server: -----------------
|
||||
0.00.003.521 I srv start: binding port with default address family
|
||||
0.00.004.988 I srv load_model: loading model '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf'
|
||||
0.00.005.007 I srv load_model: local path '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf'
|
||||
0.00.049.640 I llama_model_loader: loaded meta data with 40 key-value pairs and 851 tensors from /home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf (version GGUF V3 (latest))
|
||||
0.00.049.705 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
0.00.049.717 I llama_model_loader: - kv 0: general.architecture str = qwen35
|
||||
0.00.049.718 I llama_model_loader: - kv 1: general.type str = model
|
||||
0.00.049.723 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 20
|
||||
0.00.049.729 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000
|
||||
0.00.049.730 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000
|
||||
0.00.049.730 I llama_model_loader: - kv 5: general.name str = Qwen3.8-27B
|
||||
0.00.049.730 I llama_model_loader: - kv 6: general.size_label str = 27B
|
||||
0.00.049.730 I llama_model_loader: - kv 7: general.license str = apache-2.0
|
||||
0.00.049.755 I llama_model_loader: - kv 8: general.tags arr[str,1] = ["image-text-to-text"]
|
||||
0.00.049.756 I llama_model_loader: - kv 9: qwen35.block_count u32 = 64
|
||||
0.00.049.756 I llama_model_loader: - kv 10: qwen35.context_length u32 = 262144
|
||||
0.00.049.756 I llama_model_loader: - kv 11: qwen35.embedding_length u32 = 5120
|
||||
0.00.049.757 I llama_model_loader: - kv 12: qwen35.feed_forward_length u32 = 17408
|
||||
0.00.049.757 I llama_model_loader: - kv 13: qwen35.attention.head_count u32 = 24
|
||||
0.00.049.758 I llama_model_loader: - kv 14: qwen35.attention.head_count_kv u32 = 4
|
||||
0.00.049.760 I llama_model_loader: - kv 15: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0]
|
||||
0.00.049.762 I llama_model_loader: - kv 16: qwen35.rope.freq_base f32 = 10000000.000000
|
||||
0.00.049.763 I llama_model_loader: - kv 17: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
0.00.049.763 I llama_model_loader: - kv 18: qwen35.attention.key_length u32 = 256
|
||||
0.00.049.764 I llama_model_loader: - kv 19: qwen35.attention.value_length u32 = 256
|
||||
0.00.049.764 I llama_model_loader: - kv 20: qwen35.ssm.conv_kernel u32 = 4
|
||||
0.00.049.764 I llama_model_loader: - kv 21: qwen35.ssm.state_size u32 = 128
|
||||
0.00.049.765 I llama_model_loader: - kv 22: qwen35.ssm.group_count u32 = 16
|
||||
0.00.049.765 I llama_model_loader: - kv 23: qwen35.ssm.time_step_rank u32 = 48
|
||||
0.00.049.765 I llama_model_loader: - kv 24: qwen35.ssm.inner_size u32 = 6144
|
||||
0.00.049.769 I llama_model_loader: - kv 25: qwen35.attention.recurrent_layers arr[bool,64] = [true, true, true, false, true, true,...
|
||||
0.00.049.769 I llama_model_loader: - kv 26: qwen35.full_attention_interval u32 = 4
|
||||
0.00.049.769 I llama_model_loader: - kv 27: qwen35.rope.dimension_count u32 = 64
|
||||
0.00.049.769 I llama_model_loader: - kv 28: tokenizer.ggml.model str = gpt2
|
||||
0.00.049.769 I llama_model_loader: - kv 29: tokenizer.ggml.pre str = qwen35
|
||||
0.00.065.537 I llama_model_loader: - kv 30: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
0.00.068.926 I llama_model_loader: - kv 31: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
0.00.084.155 I llama_model_loader: - kv 32: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
|
||||
0.00.084.211 I llama_model_loader: - kv 33: tokenizer.ggml.eos_token_id u32 = 248046
|
||||
0.00.084.212 I llama_model_loader: - kv 34: tokenizer.ggml.padding_token_id u32 = 248044
|
||||
0.00.084.213 I llama_model_loader: - kv 35: tokenizer.ggml.bos_token_id u32 = 248044
|
||||
0.00.084.213 I llama_model_loader: - kv 36: tokenizer.ggml.add_bos_token bool = false
|
||||
0.00.084.219 I llama_model_loader: - kv 37: tokenizer.chat_template str = {%- set image_count = namespace(value...
|
||||
0.00.084.219 I llama_model_loader: - kv 38: general.quantization_version u32 = 2
|
||||
0.00.084.220 I llama_model_loader: - kv 39: general.file_type u32 = 15
|
||||
0.00.084.228 I llama_model_loader: - type f32: 353 tensors
|
||||
0.00.084.229 I llama_model_loader: - type q8_0: 288 tensors
|
||||
0.00.084.230 I llama_model_loader: - type q4_K: 193 tensors
|
||||
0.00.084.230 I llama_model_loader: - type q6_K: 17 tensors
|
||||
0.00.084.235 I print_info: file format = GGUF V3 (latest)
|
||||
0.00.084.238 I print_info: file type = Q4_K - Medium
|
||||
0.00.084.244 I print_info: file size = 17.66 GiB (5.64 BPW)
|
||||
0.00.323.776 I load: 0 unused tokens
|
||||
0.00.387.123 I load: printing all EOG tokens:
|
||||
0.00.387.159 I load: - 248044 ('<|endoftext|>')
|
||||
0.00.387.161 I load: - 248046 ('<|im_end|>')
|
||||
0.00.387.162 I load: - 248063 ('<|fim_pad|>')
|
||||
0.00.387.162 I load: - 248064 ('<|repo_name|>')
|
||||
0.00.387.162 I load: - 248065 ('<|file_sep|>')
|
||||
0.00.387.417 I load: special tokens cache size = 33
|
||||
0.00.441.887 I load: token to piece cache size = 1.7581 MB
|
||||
0.00.441.946 I print_info: arch = qwen35
|
||||
0.00.441.946 I print_info: vocab_only = 0
|
||||
0.00.441.947 I print_info: no_alloc = 0
|
||||
0.00.441.947 I print_info: n_ctx_train = 262144
|
||||
0.00.441.950 I print_info: n_embd_inp = 5120
|
||||
0.00.441.950 I print_info: n_embd = 5120
|
||||
0.00.441.951 I print_info: n_embd_out = 5120
|
||||
0.00.441.951 I print_info: n_layer = 64
|
||||
0.00.441.952 I print_info: n_layer_all = 64
|
||||
0.00.441.971 I print_info: n_head = 24
|
||||
0.00.441.973 I print_info: n_head_kv = 4
|
||||
0.00.441.973 I print_info: n_rot = 64
|
||||
0.00.441.974 I print_info: n_swa = 0
|
||||
0.00.441.975 I print_info: is_swa_any = 0
|
||||
0.00.441.976 I print_info: non_causal_type = 0
|
||||
0.00.441.976 I print_info: n_embd_head_k = 256
|
||||
0.00.441.976 I print_info: n_embd_head_v = 256
|
||||
0.00.441.978 I print_info: n_gqa = 6
|
||||
0.00.441.983 I print_info: n_embd_k_gqa = 1024
|
||||
0.00.441.987 I print_info: n_embd_v_gqa = 1024
|
||||
0.00.441.987 I print_info: f_norm_eps = 0.0e+00
|
||||
0.00.441.989 I print_info: f_norm_rms_eps = 1.0e-06
|
||||
0.00.441.989 I print_info: f_clamp_kqv = 0.0e+00
|
||||
0.00.441.989 I print_info: f_max_alibi_bias = 0.0e+00
|
||||
0.00.441.990 I print_info: f_logit_scale = 0.0e+00
|
||||
0.00.441.990 I print_info: f_attn_scale = 0.0e+00
|
||||
0.00.441.990 I print_info: f_attn_value_scale = 0.0000
|
||||
0.00.441.992 I print_info: n_ff = 17408
|
||||
0.00.441.992 I print_info: n_expert = 0
|
||||
0.00.441.992 I print_info: n_expert_used = 0
|
||||
0.00.441.992 I print_info: n_expert_groups = 0
|
||||
0.00.441.993 I print_info: n_group_used = 0
|
||||
0.00.441.993 I print_info: causal attn = 1
|
||||
0.00.441.993 I print_info: pooling type = -1
|
||||
0.00.441.993 I print_info: rope type = 40
|
||||
0.00.441.993 I print_info: rope scaling = linear
|
||||
0.00.441.994 I print_info: freq_base_train = 10000000.0
|
||||
0.00.441.995 I print_info: freq_scale_train = 1
|
||||
0.00.441.995 I print_info: n_ctx_orig_yarn = 262144
|
||||
0.00.441.995 I print_info: rope_yarn_log_mul = 0.0000
|
||||
0.00.441.995 I print_info: rope_finetuned = unknown
|
||||
0.00.441.995 I print_info: mrope sections = [11, 11, 10, 0]
|
||||
0.00.441.995 I print_info: ssm_d_conv = 4
|
||||
0.00.441.996 I print_info: ssm_d_inner = 6144
|
||||
0.00.441.996 I print_info: ssm_d_state = 128
|
||||
0.00.441.996 I print_info: ssm_dt_rank = 48
|
||||
0.00.441.996 I print_info: ssm_n_group = 16
|
||||
0.00.441.996 I print_info: ssm_dt_b_c_rms = 0
|
||||
0.00.441.997 I print_info: model type = 27B
|
||||
0.00.441.998 I print_info: model params = 26.90 B
|
||||
0.00.441.998 I print_info: general.name = Qwen3.8-27B
|
||||
0.00.442.002 I print_info: vocab type = BPE
|
||||
0.00.442.004 I print_info: n_vocab = 248320
|
||||
0.00.442.004 I print_info: n_merges = 247587
|
||||
0.00.442.004 AMX is not ready to be used!
|
||||
I print_info: BOS token = 248044 '<|endoftext|>'
|
||||
0.00.442.004 I print_info: EOS token = 248046 '<|im_end|>'
|
||||
0.00.442.005 I print_info: EOT token = 248046 '<|im_end|>'
|
||||
0.00.442.005 I print_info: PAD token = 248044 '<|endoftext|>'
|
||||
0.00.442.006 I print_info: LF token = 198 'Ċ'
|
||||
0.00.442.006 I print_info: FIM PRE token = 248060 '<|fim_prefix|>'
|
||||
0.00.442.006 I print_info: FIM SUF token = 248062 '<|fim_suffix|>'
|
||||
0.00.442.006 I print_info: FIM MID token = 248061 '<|fim_middle|>'
|
||||
0.00.442.006 I print_info: FIM PAD token = 248063 '<|fim_pad|>'
|
||||
0.00.442.006 I print_info: FIM REP token = 248064 '<|repo_name|>'
|
||||
0.00.442.007 I print_info: FIM SEP token = 248065 '<|file_sep|>'
|
||||
0.00.442.007 I print_info: EOG token = 248044 '<|endoftext|>'
|
||||
0.00.442.008 I print_info: EOG token = 248046 '<|im_end|>'
|
||||
0.00.442.008 I print_info: EOG token = 248063 '<|fim_pad|>'
|
||||
0.00.442.008 I print_info: EOG token = 248064 '<|repo_name|>'
|
||||
0.00.442.008 I print_info: EOG token = 248065 '<|file_sep|>'
|
||||
0.00.442.008 I print_info: max token length = 256
|
||||
0.00.442.012 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap)
|
||||
0.00.513.398 I load_tensors: CPU_Mapped model buffer size = 18084.41 MiB
|
||||
0.00.518.071 I cmn common_init_: added <|endoftext|> logit bias = -inf
|
||||
0.00.518.100 I cmn common_init_: added <|im_end|> logit bias = -inf
|
||||
0.00.518.101 I cmn common_init_: added <|fim_pad|> logit bias = -inf
|
||||
0.00.518.101 I cmn common_init_: added <|repo_name|> logit bias = -inf
|
||||
0.00.518.102 I cmn common_init_: added <|file_sep|> logit bias = -inf
|
||||
0.00.518.225 I llama_context: constructing llama_context
|
||||
0.00.518.251 I llama_context: n_seq_max = 1
|
||||
0.00.518.251 I llama_context: n_ctx = 2048
|
||||
0.00.518.251 I llama_context: n_ctx_seq = 2048
|
||||
0.00.518.251 I llama_context: n_batch = 2048
|
||||
0.00.518.252 I llama_context: n_ubatch = 512
|
||||
0.00.518.252 I llama_context: causal_attn = 1
|
||||
0.00.518.254 I llama_context: flash_attn = auto
|
||||
0.00.518.254 I llama_context: kv_unified = false
|
||||
0.00.518.258 I llama_context: freq_base = 10000000.0
|
||||
0.00.518.258 I llama_context: freq_scale = 1
|
||||
0.00.518.258 I llama_context: n_rs_seq = 0
|
||||
0.00.518.259 I llama_context: n_outputs_max = 1
|
||||
0.00.518.259 I llama_context: n_outputs_max_per_seq = 1
|
||||
0.00.518.259 I llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized
|
||||
0.00.518.327 I llama_context: CPU output buffer size = 0.95 MiB
|
||||
0.00.518.448 I llama_kv_cache: CPU KV buffer size = 128.00 MiB
|
||||
0.00.605.242 I llama_kv_cache: size = 128.00 MiB ( 2048 cells, 16 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB
|
||||
0.00.605.270 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256
|
||||
0.00.605.271 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256
|
||||
0.00.713.625 I llama_memory_recurrent: CPU RS buffer size = 149.62 MiB
|
||||
0.00.713.683 I llama_memory_recurrent: size = 149.62 MiB ( 1 cells, 64 layers, 1 seqs 0 rs_seq), R (f32): 5.62 MiB, S (f32): 144.00 MiB, P (f32): 0.00 MiB
|
||||
0.00.713.706 I sched_reserve: reserving ...
|
||||
0.00.731.562 I resolve_fused_ops: Flash Attention enabled
|
||||
0.00.731.592 I resolve_fused_ops: resolving fused DeepSeek V4 HC support:
|
||||
0.00.732.337 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled
|
||||
0.00.732.977 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled
|
||||
0.00.733.582 I resolve_fused_ops: fused DeepSeek V4 HC post enabled
|
||||
0.00.736.118 I sched_reserve: CPU compute buffer size = 134.02 MiB
|
||||
0.00.736.144 I sched_reserve: graph nodes = 3751
|
||||
0.00.736.145 I sched_reserve: graph splits = 1
|
||||
0.00.736.148 I sched_reserve: reserve took 22.43 ms, sched copies = 1
|
||||
0.00.736.211 I cmn init: llama threadpool init, n_threads = 12
|
||||
0.00.736.250 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable)
|
||||
0.01.131.297 I clip_model_loader: model name: Qwen3.8-27B
|
||||
0.01.131.335 I clip_model_loader: description:
|
||||
0.01.131.338 I clip_model_loader: GGUF version: 3
|
||||
0.01.131.341 I clip_model_loader: alignment: 32
|
||||
0.01.131.341 I clip_model_loader: n_tensors: 334
|
||||
0.01.131.342 I clip_model_loader: n_kv: 26
|
||||
0.01.131.343 I
|
||||
0.01.131.346 I clip_model_loader: has vision encoder
|
||||
0.01.131.647 I clip_ctx: CLIP using CPU backend
|
||||
0.01.132.241 W load_hparams: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks
|
||||
0.01.132.261 W load_hparams: if you encounter problems with accuracy, try adding --image-min-tokens 1024
|
||||
0.01.132.261 W load_hparams: more info: https://github.com/ggml-org/llama.cpp/issues/16842
|
||||
|
||||
0.01.132.262 I load_hparams: projector: qwen3vl_merger
|
||||
0.01.132.262 I load_hparams: n_embd: 1152
|
||||
0.01.132.263 I load_hparams: n_head: 16
|
||||
0.01.132.263 I load_hparams: n_ff: 4304
|
||||
0.01.132.263 I load_hparams: n_layer: 27
|
||||
0.01.132.263 I load_hparams: ffn_op: gelu
|
||||
0.01.132.263 I load_hparams: projection_dim: 5120
|
||||
0.01.132.264 I
|
||||
--- vision hparams ---
|
||||
0.01.132.264 I load_hparams: image_size: 768
|
||||
0.01.132.264 I load_hparams: patch_size: 16
|
||||
0.01.132.264 I load_hparams: has_llava_proj: 0
|
||||
0.01.132.264 I load_hparams: minicpmv_version: 0
|
||||
0.01.132.264 I load_hparams: n_merge: 2
|
||||
0.01.132.264 I load_hparams: n_wa_pattern: 0
|
||||
0.01.132.264 I load_hparams: image_min_pixels: 8192
|
||||
0.01.132.265 I load_hparams: image_max_pixels: 4194304
|
||||
0.01.132.265 I
|
||||
0.01.132.270 I load_hparams: model size: 600.08 MiB
|
||||
0.01.132.272 I load_hparams: metadata size: 0.12 MiB
|
||||
0.01.644.456 I get_dummy_batch: warmup with image size = 1472 x 1472
|
||||
0.01.674.365 I get_dummy_batch: warmup with image size = 1472 x 1472
|
||||
0.01.683.463 I reserve_compute_meta: CPU compute buffer size = 248.10 MiB
|
||||
0.01.683.507 I reserve_compute_meta: graph splits = 1, nodes = 823
|
||||
0.01.683.534 I warmup: flash attention is enabled
|
||||
0.01.686.331 I srv load_model: loaded multimodal model, '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf'
|
||||
0.02.132.137 I cmn common_conte: the context does not support partial sequence removal
|
||||
0.02.143.173 I srv load_model: speculative decoding will use checkpoints
|
||||
0.02.143.214 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 2048, kv_unified = 'false'
|
||||
0.02.143.233 I spec common_specu: no implementations specified for speculative decoding
|
||||
0.02.143.253 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048
|
||||
0.02.143.332 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it
|
||||
0.02.143.336 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391
|
||||
0.02.143.336 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192
|
||||
0.02.143.362 W srv init: --cache-idle-slots requires --cache-ram, disabling
|
||||
0.02.147.190 I srv init: init: chat template, example_format: '<|im_start|>system
|
||||
You are a helpful assistant<|im_end|>
|
||||
<|im_start|>user
|
||||
Hello<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
Hi there<|im_end|>
|
||||
<|im_start|>user
|
||||
How are you?<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
'
|
||||
0.02.147.691 I srv init: init: chat template, thinking = 0
|
||||
0.02.147.728 I srv init: preserve_reasoning kwarg: true
|
||||
0.02.147.728 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve)
|
||||
0.02.147.772 I srv llama_server: model loaded
|
||||
0.02.147.795 I srv llama_server: listening on http://127.0.0.1:34289
|
||||
0.02.147.957 I srv update_slots: all slots are idle
|
||||
0.02.241.520 I srv server_strea: conv_id= (empty=1)
|
||||
0.02.241.770 I srv operator(): chat format: peg-native
|
||||
0.02.241.991 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty
|
||||
0.02.242.009 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1
|
||||
0.02.242.068 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist
|
||||
0.02.242.098 I slot launch_slot_: id 0 | task -1 | sampler params:
|
||||
repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000
|
||||
dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64
|
||||
top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000
|
||||
mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900
|
||||
0.02.242.099 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0
|
||||
0.02.242.114 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19
|
||||
0.02.242.117 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end)
|
||||
0.03.349.092 I slot operator(): id 0 | task 0 | cached n_tokens = 15, memory_seq_rm [15, end)
|
||||
0.03.349.182 I slot init_sampler: id 0 | task 0 | init sampler, took 0.02 ms, tokens: text = 19, total = 19
|
||||
0.03.444.604 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB)
|
||||
0.04.185.775 I slot print_timing: id 0 | task 0 | prompt eval time = 1632.09 ms / 19 tokens ( 85.90 ms per token, 11.64 tokens per second)
|
||||
0.04.185.821 I slot print_timing: id 0 | task 0 | eval time = 311.54 ms / 2 tokens ( 311.54 ms per token, 3.21 tokens per second)
|
||||
0.04.185.822 I slot print_timing: id 0 | task 0 | total time = 1943.62 ms / 21 tokens
|
||||
0.04.185.831 I slot print_timing: id 0 | task 0 | graphs reused = 1
|
||||
0.04.185.913 I slot release: id 0 | task 0 | stop processing: n_tokens = 20, truncated = 0
|
||||
0.04.185.929 I srv update_slots: all slots are idle
|
||||
0.04.198.796 I srv server_strea: conv_id= (empty=1)
|
||||
0.04.199.218 I srv operator(): chat format: peg-native
|
||||
0.04.199.376 I slot get_availabl: id 0 | task -1 | - checking sim = 0.081 (3/37) > 0.100
|
||||
0.04.199.397 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 493860006106
|
||||
0.04.199.456 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist
|
||||
0.04.199.482 I slot launch_slot_: id 0 | task -1 | sampler params:
|
||||
repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000
|
||||
dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64
|
||||
top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000
|
||||
mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900
|
||||
0.04.199.483 I slot launch_slot_: id 0 | task 4 | processing task, is_child = 0
|
||||
0.04.199.490 I slot operator(): id 0 | task 4 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37
|
||||
0.04.199.492 I slot operator(): id 0 | task 4 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB)
|
||||
0.04.208.853 I slot operator(): id 0 | task 4 | cached n_tokens = 0, memory_seq_rm [0, end)
|
||||
0.05.509.411 I slot operator(): id 0 | task 4 | cached n_tokens = 18, memory_seq_rm [18, end)
|
||||
0.05.509.500 I slot process_mtmd: id 0 | task 4 | encoding mtmd batch from idx = 18, n_chunks = 1
|
||||
0.05.571.320 I decoding image batch 1/1, n_tokens_batch = 9
|
||||
0.05.571.432 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens
|
||||
0.05.571.459 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens
|
||||
0.06.318.400 I image decoded (batch 1/1) in 747 ms
|
||||
0.06.318.890 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens
|
||||
0.06.318.959 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens
|
||||
0.06.912.682 I slot operator(): id 0 | task 4 | cached n_tokens = 33, memory_seq_rm [27, end)
|
||||
0.06.912.760 I slot init_sampler: id 0 | task 4 | init sampler, took 0.01 ms, tokens: text = 28, total = 37
|
||||
0.06.980.991 I slot create_check: id 0 | task 4 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB)
|
||||
0.07.740.128 I slot print_timing: id 0 | task 4 | prompt eval time = 3227.13 ms / 37 tokens ( 87.22 ms per token, 11.47 tokens per second)
|
||||
0.07.740.161 I slot print_timing: id 0 | task 4 | eval time = 313.48 ms / 2 tokens ( 313.48 ms per token, 3.19 tokens per second)
|
||||
0.07.740.162 I slot print_timing: id 0 | task 4 | total time = 3540.61 ms / 39 tokens
|
||||
0.07.740.164 I slot print_timing: id 0 | task 4 | graphs reused = 1
|
||||
0.07.740.217 I slot release: id 0 | task 4 | stop processing: n_tokens = 38, truncated = 0
|
||||
0.07.740.245 I srv update_slots: all slots are idle
|
||||
0.07.755.538 I srv server_strea: conv_id= (empty=1)
|
||||
0.07.755.707 I srv operator(): chat format: peg-native
|
||||
0.07.755.921 I slot get_availabl: id 0 | task -1 | - checking sim = 0.158 (3/19) > 0.100
|
||||
0.07.755.942 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 0.158 (> 0.100 thold), f_keep = 0.079
|
||||
0.07.755.992 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist
|
||||
0.07.756.016 I slot launch_slot_: id 0 | task -1 | sampler params:
|
||||
repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000
|
||||
dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64
|
||||
top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000
|
||||
mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900
|
||||
0.07.756.017 I slot launch_slot_: id 0 | task 9 | processing task, is_child = 0
|
||||
0.07.756.024 I slot operator(): id 0 | task 9 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19
|
||||
0.07.756.026 I slot operator(): id 0 | task 9 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB)
|
||||
0.07.764.483 I slot operator(): id 0 | task 9 | cached n_tokens = 0, memory_seq_rm [0, end)
|
||||
0.08.856.525 I slot operator(): id 0 | task 9 | cached n_tokens = 15, memory_seq_rm [15, end)
|
||||
0.08.856.627 I slot init_sampler: id 0 | task 9 | init sampler, took 0.01 ms, tokens: text = 19, total = 19
|
||||
0.08.921.613 I slot create_check: id 0 | task 9 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB)
|
||||
0.09.648.486 I slot print_timing: id 0 | task 9 | prompt eval time = 1582.94 ms / 19 tokens ( 83.31 ms per token, 12.00 tokens per second)
|
||||
0.09.648.550 I slot print_timing: id 0 | task 9 | eval time = 309.48 ms / 2 tokens ( 309.48 ms per token, 3.23 tokens per second)
|
||||
0.09.648.552 I slot print_timing: id 0 | task 9 | total time = 1892.42 ms / 21 tokens
|
||||
0.09.648.554 I slot print_timing: id 0 | task 9 | graphs reused = 1
|
||||
0.09.648.623 I slot release: id 0 | task 9 | stop processing: n_tokens = 20, truncated = 0
|
||||
0.09.648.652 I srv update_slots: all slots are idle
|
||||
0.09.659.990 I srv server_strea: conv_id= (empty=1)
|
||||
0.09.660.185 I srv operator(): chat format: peg-native
|
||||
0.09.660.340 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100
|
||||
0.09.660.363 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950
|
||||
0.09.660.413 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist
|
||||
0.09.660.417 I slot launch_slot_: id 0 | task -1 | sampler params:
|
||||
repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000
|
||||
dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64
|
||||
top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000
|
||||
mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900
|
||||
0.09.660.418 I slot launch_slot_: id 0 | task 13 | processing task, is_child = 0
|
||||
0.09.660.425 I slot operator(): id 0 | task 13 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19
|
||||
0.09.660.427 I slot operator(): id 0 | task 13 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB)
|
||||
0.09.668.964 I slot operator(): id 0 | task 13 | cached n_tokens = 0, memory_seq_rm [0, end)
|
||||
0.10.799.794 I slot operator(): id 0 | task 13 | cached n_tokens = 15, memory_seq_rm [15, end)
|
||||
0.10.799.894 I slot init_sampler: id 0 | task 13 | init sampler, took 0.01 ms, tokens: text = 19, total = 19
|
||||
0.10.866.949 I slot create_check: id 0 | task 13 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB)
|
||||
0.11.596.501 I slot print_timing: id 0 | task 13 | prompt eval time = 1635.85 ms / 19 tokens ( 86.10 ms per token, 11.61 tokens per second)
|
||||
0.11.596.539 I slot print_timing: id 0 | task 13 | eval time = 300.20 ms / 2 tokens ( 300.20 ms per token, 3.33 tokens per second)
|
||||
0.11.596.540 I slot print_timing: id 0 | task 13 | total time = 1936.05 ms / 21 tokens
|
||||
0.11.596.541 I slot print_timing: id 0 | task 13 | graphs reused = 1
|
||||
0.11.596.593 I slot release: id 0 | task 13 | stop processing: n_tokens = 20, truncated = 0
|
||||
0.11.596.610 I srv update_slots: all slots are idle
|
||||
0.11.608.474 I srv server_strea: conv_id= (empty=1)
|
||||
0.11.608.847 I srv operator(): chat format: peg-native
|
||||
0.11.609.003 I slot get_availabl: id 0 | task -1 | - checking sim = 0.081 (3/37) > 0.100
|
||||
0.11.609.024 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 493867416785
|
||||
0.11.609.076 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist
|
||||
0.11.609.084 I slot launch_slot_: id 0 | task -1 | sampler params:
|
||||
repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000
|
||||
dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64
|
||||
top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000
|
||||
mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900
|
||||
0.11.609.087 I slot launch_slot_: id 0 | task 17 | processing task, is_child = 0
|
||||
0.11.609.094 I slot operator(): id 0 | task 17 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37
|
||||
0.11.609.096 I slot operator(): id 0 | task 17 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB)
|
||||
0.11.618.134 I slot operator(): id 0 | task 17 | cached n_tokens = 0, memory_seq_rm [0, end)
|
||||
0.12.975.520 I slot operator(): id 0 | task 17 | cached n_tokens = 18, memory_seq_rm [18, end)
|
||||
0.12.975.603 I slot process_mtmd: id 0 | task 17 | encoding mtmd batch from idx = 18, n_chunks = 1
|
||||
0.13.039.366 I decoding image batch 1/1, n_tokens_batch = 9
|
||||
0.13.039.573 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens
|
||||
0.13.039.605 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens
|
||||
0.13.801.404 I image decoded (batch 1/1) in 762 ms
|
||||
0.13.801.710 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens
|
||||
0.13.801.767 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens
|
||||
0.14.389.213 I slot operator(): id 0 | task 17 | cached n_tokens = 33, memory_seq_rm [27, end)
|
||||
0.14.389.287 I slot init_sampler: id 0 | task 17 | init sampler, took 0.01 ms, tokens: text = 28, total = 37
|
||||
0.14.500.054 I slot create_check: id 0 | task 17 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB)
|
||||
0.15.235.429 I slot print_timing: id 0 | task 17 | prompt eval time = 3321.23 ms / 37 tokens ( 89.76 ms per token, 11.14 tokens per second)
|
||||
0.15.235.477 I slot print_timing: id 0 | task 17 | eval time = 305.07 ms / 2 tokens ( 305.07 ms per token, 3.28 tokens per second)
|
||||
0.15.235.478 I slot print_timing: id 0 | task 17 | total time = 3626.31 ms / 39 tokens
|
||||
0.15.235.480 I slot print_timing: id 0 | task 17 | graphs reused = 1
|
||||
0.15.235.541 I slot release: id 0 | task 17 | stop processing: n_tokens = 38, truncated = 0
|
||||
0.15.235.562 I srv update_slots: all slots are idle
|
||||
0.15.260.146 I srv server_strea: conv_id= (empty=1)
|
||||
0.15.260.607 I srv operator(): chat format: peg-native
|
||||
0.15.260.769 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100
|
||||
0.15.260.788 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974
|
||||
0.15.260.842 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist
|
||||
0.15.260.866 I slot launch_slot_: id 0 | task -1 | sampler params:
|
||||
repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000
|
||||
dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64
|
||||
top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000
|
||||
mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900
|
||||
0.15.260.867 I slot launch_slot_: id 0 | task 22 | processing task, is_child = 0
|
||||
0.15.260.875 I slot operator(): id 0 | task 22 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37
|
||||
0.15.260.877 I slot operator(): id 0 | task 22 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB)
|
||||
0.15.269.453 I slot operator(): id 0 | task 22 | cached n_tokens = 0, memory_seq_rm [0, end)
|
||||
0.16.610.837 I slot operator(): id 0 | task 22 | cached n_tokens = 18, memory_seq_rm [18, end)
|
||||
0.16.610.908 I slot process_mtmd: id 0 | task 22 | encoding mtmd batch from idx = 18, n_chunks = 1
|
||||
0.16.676.869 I decoding image batch 1/1, n_tokens_batch = 9
|
||||
0.16.676.977 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens
|
||||
0.16.677.009 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens
|
||||
0.17.436.949 I image decoded (batch 1/1) in 760 ms
|
||||
0.17.437.236 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens
|
||||
0.17.437.295 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens
|
||||
0.18.061.847 I slot operator(): id 0 | task 22 | cached n_tokens = 33, memory_seq_rm [27, end)
|
||||
0.18.061.923 I slot init_sampler: id 0 | task 22 | init sampler, took 0.01 ms, tokens: text = 28, total = 37
|
||||
0.18.131.774 I slot create_check: id 0 | task 22 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB)
|
||||
0.18.887.462 I slot print_timing: id 0 | task 22 | prompt eval time = 3290.39 ms / 37 tokens ( 88.93 ms per token, 11.24 tokens per second)
|
||||
0.18.887.504 I slot print_timing: id 0 | task 22 | eval time = 336.17 ms / 2 tokens ( 336.17 ms per token, 2.97 tokens per second)
|
||||
0.18.887.506 I slot print_timing: id 0 | task 22 | total time = 3626.55 ms / 39 tokens
|
||||
0.18.887.507 I slot print_timing: id 0 | task 22 | graphs reused = 1
|
||||
0.18.887.569 I slot release: id 0 | task 22 | stop processing: n_tokens = 38, truncated = 0
|
||||
0.18.887.594 I srv update_slots: all slots are idle
|
||||
0.18.894.469 I srv operator(): operator(): cleaning up before exit...
|
||||
0.18.895.991 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted |
|
||||
0.18.896.031 I common_memory_breakdown_print: | - Host | 18496 = 18084 + 277 + 134 |
|
||||
@@ -16,7 +16,7 @@ The [native Windows RTX Qwen3.8-27B NVFP4 attempt](../../../e2e_expansion_202609
|
||||
|
||||
[A native Windows RTX Qwen3.8-27B FP8 recovery](../../../e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md) completed scoped Omni text and synthetic image-to-text requests via an explicit Triton FP8 linear backend. One warmup plus 20/20 serial measured requests of each kind produced stable outputs at 3.070/3.214 s text and 4.184/4.380 s image p50/p95. The separate NVFP4 Marlin/PTX failure remains unresolved.
|
||||
|
||||
A separate pinned [Qwen3.8-27B Q4_K_M GGUF CPU path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_cpu/README.md) and [Radeon 890M path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_radeon/README.md) now pass complete short text and synthetic red-image requests through Omni on native Windows. Each passed 20/20 serial measured requests per type after one warmup per type. CPU p50/p95 complete-request wall time was 1.656/1.697 s text and 2.925/2.976 s image; Radeon was 3.101/3.181 s text and 5.160/5.220 s image. The backend verified hashes, 28 GiB host admission, actual CPU or Radeon model and vision placement, and released the ledger. These two-token-output tests do not qualify long generation, broad quality or a Radeon speedup; the earlier WSL CPU and AMD NPU cells remain separate.
|
||||
A separate pinned [Qwen3.8-27B Q4_K_M GGUF native Windows CPU path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_cpu/README.md), [Radeon 890M path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_radeon/README.md), and [WSL CPU no-repack path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md) now pass complete short text and synthetic red-image requests through Omni. Each passed 20/20 serial measured requests per type after one warmup per type. Native Windows CPU p50/p95 complete-request wall time was 1.656/1.697 s text and 2.925/2.976 s image; Radeon was 3.101/3.181 s text and 5.160/5.220 s image; WSL CPU no-repack was 1.934/1.983 s text and 3.642/3.789 s image. The backend verified pinned hashes, explicit host admission, actual model and vision placement, and ledger release. The WSL route uses a different llama.cpp build, repack setting and RAM quota, so the timings are not a paired OS comparison. These two-token-output tests do not qualify long generation, broad quality or a Radeon speedup; the AMD NPU cell remains separate component-only evidence.
|
||||
|
||||
A [native Windows RTX Qwen3-TTS Omni hybrid route](../../../e2e_expansion_20260924/evidence/qwen_tts_windows_rtx/README.md) now returns complete 24 kHz WAVs with talker/codec on RTX Vulkan0 and predictor on CPU FP32. One warmup plus 20/20 serial outputs had identical PCM at p50/p95 complete-request wall 3.153/3.231 s for 2.72 s audio. Public audio, explicit host/GPU admission, refusal of an undersized GPU reservation and fresh-stage recovery after server-started cancellation passed. The PCM differs from the Radeon hybrid, and the complete-WAV RTF remains above 1; the existing CUDA sustained playback findings are separate.
|
||||
|
||||
@@ -54,7 +54,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
|
||||
| Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 |
|
||||
|---|---|---|---|---|---|
|
||||
| PC HX370: CPU / WSL | scoped E2E: Omni GGUF or BF16 Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF combined image+audio → text+speech | synthetic policy pass: Omni CPU |
|
||||
| PC HX370: CPU / WSL | scoped E2E: Omni GGUF or BF16 Spark text | scoped E2E: Omni CPU text → complete WAV | scoped E2E: Omni Q4_K_M GGUF CPU text+image (20 each; no repack) | scoped E2E: Omni GGUF combined image+audio → text+speech | synthetic policy pass: Omni CPU |
|
||||
| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text; fresh restart after in-flight cancel | scoped E2E: Omni CPU text → complete WAV | scoped E2E: Omni Q4_K_M GGUF CPU text+image (20 each) | scoped E2E: Omni GGUF image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | scoped E2E: Omni Spark 1.7B GGUF text on RTX Vulkan; fresh restart after in-flight cancel (separate 4B BF16 CUDA profile) | scoped E2E: Omni RTX Vulkan talker/codec + CPU predictor complete WAV and restart (separate CUDA stream has playback jitter) | scoped E2E: Omni FP8 Triton text+image on RTX; separate NVFP4 Windows startup failed | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA |
|
||||
@@ -73,7 +73,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
|---|---|---|---|---|
|
||||
| PC HX370: CPU / WSL | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST; BF16_CPU_ACCEPTANCE_PARITY_PROFILE_AND_RESTART_PASS | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran on a CPU-only Linux llama.cpp server through the current Omni 0.29 `StageRuntime`/`StagePool` whole-session backend. Both named prompts passed (`Paris`, `1511`); one warmup plus 20/20 serial inventory requests had nearest-rank complete-request wall p50/p95 7.82/7.97 s, excluding 1.61 s startup. The stage verified model/server hashes, explicit CPU-only tensor buffers (`CPU_Mapped`, `CPU_REPACK`) and no usable GPU, reserved 4 GiB host RAM, refused an over-context request without truncation, and after in-flight cancellation returned no stale output or reservation. A separate public `AsyncOmni.generate` text request passed. The Linux CPU-only build omits the Windows 0/29 offload line; the verifier was extended to require positive CPU buffer evidence rather than inferring CPU from an absent GPU count. A separate pinned 1.7B BF16 checkpoint was completed and verified against the upstream index. The current Omni vLLM CPU text path then passed 12/12 named prompts at 128 generated tokens, cancellation cleanup and exact 12/12 greedy token-ID parity against standalone vLLM on the same BF16 checkpoint. The fork-branch acceptance load was 30.36 s with approximately 6.45 GiB process-tree RSS load delta; the plan reserved 7.64 GiB. One warmup plus 20 serial BF16 complete requests (28 prompt tokens, 64 output tokens) measured nearest-rank wall p50/p95 5.411/5.751 s and TTFT 0.129/0.146 s on HX370 WSL CPU. Separate neighboring-checkout evidence retained a 19.88 s acceptance load and 4.808/5.089 s profile; it is not the fork-branch timing. The installed vLLM was 0.28.0 with 0.29.0rc2 Omni source, so that version pairing is the tested boundary. The GGUF result and historical INT8 result remain distinct artifacts. A [BF16 restart probe](../../../e2e_expansion_20260924/evidence/spark_bf16_wsl_cpu/restart_branch_report.json) cancelled after eight token events, observed no late output, rejected the retired session handle and reproduced the exact 128-token sequence in a fresh session. None establishes broad text quality, incremental GGUF token streaming, concurrent recovery, loading peak or sustained performance. | Broaden text quality and context, qualify a matching Omni/vLLM version pair, then test incremental streaming, concurrent recovery and sustained load; [GGUF evidence](../../../e2e_expansion_20260923/evidence/spark_omni_wsl_cpu/README.md) and [BF16 evidence](../../../e2e_expansion_20260924/evidence/spark_bf16_wsl_cpu/README.md). |
|
||||
| PC HX370: CPU / WSL | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The earlier vLLM 0.28 CPU path passed four short streams but had four simulated playback underruns in one measured request; its wheel remains mismatched with Omni 0.29. Separately, the pinned BF16 0.6B CustomVoice checkpoint ran two named text-to-complete-WAV requests through an Omni 0.29 whole-session stage with an isolated CPU worker on WSL. One warmup plus 20 serial measured requests returned identical 3.44 s audio; nearest-rank complete-request wall p50/p95 was 8.27/8.47 s (RTF 2.40/2.46), excluding 19.82 s startup. Pinned Whisper tiny.en transcribed the first measured WAV exactly (WER 0, one-output intelligibility proxy). Public `AsyncOmni` audio output passed; the stage verified weights, interpreter and CPU placement, reserved 10 GiB host RAM, refused 4 GiB before launch, and drained in-flight cancellation with zero remaining reservation. WSL PCM differs from the same-checkpoint native Windows outputs despite the declared settings; its cause is unqualified. Sampled process-tree RSS reached 3.39 GB, not a loading peak. This worker delivers only a complete WAV, not a playable stream. | Qualify cross-OS numerical/quality difference, playable streaming without underruns, post-cancel restart, loading peak, concurrency and sustained power/thermal behavior; [WSL Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_wsl_cpu/README.md), [earlier stream evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370: CPU / WSL | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No compatible CPU artifact/deployment for this model was found in the checked model roots. Available Qwen 27B FP8/NVFP4 CUDA artifacts do not establish a CPU route. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / WSL | Qwen3.8-27B | SCOPED_E2E_OMNI_GGUF_CPU_TEXT_IMAGE_PROFILE20 | The pinned ggml-org/Qwen3.8-27B-GGUF Q4_K_M language model and Q8_0 vision projector at revision efbb3b1 ran through an admitted whole-session Omni StageRuntime/llama.cpp b10849 CPU-only backend on WSL. A 28 GiB reservation was twice refused before loading when available WSL RAM was below it; a 27 GiB attempt loaded but exceeded its RSS budget. Explicit --no-repack removed the 9,180 MiB CPU repack buffer without changing weights/precision. With a 24 GiB reservation, a smoke and one warmup of each type plus 20/20 serial measured text outputs ready and 20/20 synthetic image outputs Red passed. Nearest-rank complete-request p50/p95 was 1.934/1.983 s text and 3.642/3.789 s image, excluding 11.45 s startup. The stage verified hashes, CPU model and vision placement, a loaded RSS sample of 20,026,531,840 bytes, and ledger release. This two-token-output case does not qualify long context, broad image quality, public entrypoint, cancellation or sustained behavior. | Test public AsyncOmni, long-context/128-token output, cancellation/restart, sampled memory peak and sustained power; [WSL GGUF profile](../../../e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md). |
|
||||
| PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The BF16/vLLM 0.28 three-stage path previously passed separate text, synthetic red-square image and synthetic 440 Hz audio inputs to text+speech; its 20-request serial **text-only** profile was p50/p95 21.86/22.97 s with swap and shutdown warnings. A separate pinned Q4_K_M/F16 GGUF worker now runs a **combined spoken question+image → text+speech** request through a bounded Omni StageRuntime/StagePool graph stage on WSL CPU, with installed vLLM 0.29 for the controller and CPU-only C++ model execution. All 20 measured requests after one warmup answered “The square in the image is red.”, emitted complete nonzero speech and a terminal audio event; nearest-rank complete-request wall p50/p95 was 22.67/24.83 s, speech duration 1.60–2.92 s, and maximum sampled worker RSS 13.70 GB. The first measured WAV independently transcribed exactly with pinned Whisper tiny.en (WER 0, one-output intelligibility proxy). Ten weight hashes and CPU placement were verified; 24 GiB shared RAM was reserved, 8 GiB was refused before launch, and in-flight cancellation left no stale output or reservation. Host RAM/swap samples covered only part of the profile. One synthetic fixture and variable speech PCM do not establish general quality, playable streaming, loading peak or sustained performance. | Validate real image/audio suites, speech alignment and quality, video input, concurrent admission, post-cancel restart, loading peak and sustained thermal behavior; [GGUF Omni raw evidence](../../../e2e_expansion_20260923/evidence/minicpmo_omni_wsl_cpu/README.md), [BF16/vLLM CPU evidence](../../../e2e_expansion_20260923/README.md), [earlier standalone GGUF run](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_cpu/README.md). |
|
||||
| PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted Omni StageRuntime whole-policy stage with a CPU-only isolated worker. One warmup plus 20 serial patterned synthetic camera/state/noise requests returned identical finite `[1,50,32]` action chunks with terminal action events; nearest-rank complete-request wall p50/p95 was 3.175/3.219 s, excluding 40.03 s startup. The stage verified checkpoint/interpreter hashes and CPU placement, reserved 16 GiB of a declared 30 GiB WSL host-RAM pool, refused 8 GiB before launch, and released the reservation on shutdown. The earlier direct zero-input policy/CUDA comparison was a different fixture and remains separate. No real-observation task-quality tolerance exists. | Validate real observations/reference actions, physical units/order/step time, action deadlines, loading peak, concurrency and sustained power/thermal behavior; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_policy/README.md), [earlier direct policy evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST; RESTART_AFTER_INFLIGHT_CANCEL_PASS | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran in native Windows llama.cpp CPU (`-dev none -ngl 0`) as a whole-session Omni StageRuntime/StagePool backend. `Paris` and the correct `1511` inventory answer passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 6.93/7.05 s. The public AsyncOmni short request passed. The stage verifies artifact hashes and CPU placement (0/29 layers offloaded), reserves 4 GiB host RAM, emits a terminal text event and releases reservation after in-flight cancellation. The separate prior standalone profile was 6.65/6.86 s. A separate confirmed server-start cancellation of the 120-item request terminated the worker, left no stale output or reservation, and a fresh StageRuntime with the same pinned artifact returned Paris under a new worker generation. This is fresh-stage recovery, not same-session continuation. No vLLM CPU attention repair, incremental stream, loading peak or sustained power/thermal gate is claimed. | Validate broader text/token quality, streaming, concurrent recovery, memory peak and sustained load; [restart evidence](../../../e2e_expansion_20260924/evidence/spark_omni_windows_restart/README.md), [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
|
||||
@@ -155,6 +155,8 @@ class LlamaCppTextStageClient(StageClientBase):
|
||||
"--reasoning", "off", "--no-webui", "--fit", "off",
|
||||
"--cache-ram", "0", "-lv", "4",
|
||||
]
|
||||
if config.get("disable_repack", False):
|
||||
command.append("--no-repack")
|
||||
if self._mmproj is not None:
|
||||
command.extend(["--mmproj", str(self._mmproj)])
|
||||
if self._placement == "cpu":
|
||||
@@ -243,6 +245,7 @@ class LlamaCppTextStageClient(StageClientBase):
|
||||
"reserved_bytes": dict(reservation.demands),
|
||||
"memory_pool": self._memory_pool,
|
||||
"memory_overhead_bytes": overhead,
|
||||
"disable_repack": bool(config.get("disable_repack", False)),
|
||||
"context_tokens": self._context_tokens,
|
||||
"max_new_tokens": self._max_new_tokens,
|
||||
"max_io_bytes": self._max_io_bytes,
|
||||
|
||||
Reference in new issue
Block a user