From 17b737914b229ad5c8dddf8027181efefd98a9cb Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 24 Sep 2026 18:14:25 +0800 Subject: [PATCH] Qualify WSL CPU Qwen3.8 GGUF with bounded no-repack loading --- .../experiments/audit_qwen38_gguf_wsl.py | 121 ++ .../experiments/probe_omni_llamacpp_qwen38.py | 2 + .../results/e2e_expansion_20260924/README.md | 10 + .../evidence/qwen38_gguf_wsl_cpu/README.md | 9 + .../qwen38_gguf_wsl_cpu/audit_report.json | 95 ++ .../qwen38_gguf_wsl_cpu/hardware_runtime.json | 14 + .../qwen38_gguf_wsl_cpu/profile20.driver.log | 6 + .../qwen38_gguf_wsl_cpu/profile20.json | 1300 +++++++++++++++ .../qwen38_gguf_wsl_cpu/profile20.server.log | 1452 +++++++++++++++++ .../qwen38_gguf_wsl_cpu/smoke.driver.log | 19 + .../evidence/qwen38_gguf_wsl_cpu/smoke.json | 11 + .../qwen38_gguf_wsl_cpu/smoke2.driver.log | 19 + .../evidence/qwen38_gguf_wsl_cpu/smoke2.json | 11 + .../qwen38_gguf_wsl_cpu/smoke3.driver.log | 62 + .../evidence/qwen38_gguf_wsl_cpu/smoke3.json | 21 + .../qwen38_gguf_wsl_cpu/smoke3.server.log | 267 +++ .../qwen38_gguf_wsl_cpu/smoke4.driver.log | 6 + .../evidence/qwen38_gguf_wsl_cpu/smoke4.json | 236 +++ .../qwen38_gguf_wsl_cpu/smoke4.server.log | 426 +++++ .../evidence/summary/README.md | 6 +- vllm_omni/engine/backends/llamacpp.py | 3 + 21 files changed, 4093 insertions(+), 3 deletions(-) create mode 100644 benchmarks/edge_harness/experiments/audit_qwen38_gguf_wsl.py create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/audit_report.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/hardware_runtime.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.server.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.server.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.server.log diff --git a/benchmarks/edge_harness/experiments/audit_qwen38_gguf_wsl.py b/benchmarks/edge_harness/experiments/audit_qwen38_gguf_wsl.py new file mode 100644 index 000000000..66d6d2991 --- /dev/null +++ b/benchmarks/edge_harness/experiments/audit_qwen38_gguf_wsl.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Audit pinned Qwen3.8 GGUF Omni WSL CPU complete-request profiling.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path + +MODEL_SHA = "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747" +MMPROJ_SHA = "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb" +SERVER_SHA = "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c" +REVISION = "efbb3b1f70a21d97fd4495240648405f7228554f" + + +def read(root: Path, name: str) -> dict: + return json.loads((root / name).read_text(encoding="utf-8")) + + +def rank(values: list[float], fraction: float) -> float: + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def digest(path: Path) -> dict[str, str | int]: + with path.open("rb") as stream: + return {"bytes": path.stat().st_size, "sha256": hashlib.file_digest(stream, "sha256").hexdigest()} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-dir", type=Path, required=True) + args = parser.parse_args() + root = args.evidence_dir + for name in ("smoke.json", "smoke2.json"): + failed = read(root, name) + if failed["status"] != "failed" or "available native host RAM" not in failed["error"]: + raise ValueError(f"{name}: low-available-RAM pre-admission refusal missing") + repack = read(root, "smoke3.json") + if repack["status"] != "failed" or "RSS exceeds stage reservation" not in repack["error"]: + raise ValueError("repacked model did not fail the loaded-RSS admission gate") + smoke = read(root, "smoke4.json") + profile = read(root, "profile20.json") + if smoke["status"] != "completed" or profile["status"] != "completed": + raise ValueError("WSL no-repack smoke/profile did not complete") + plan = profile["execution_plan"] + if (plan["backend"] != "external.llamacpp.multimodal.v1" + or plan["model_sha256"] != MODEL_SHA + or plan["mmproj_sha256"] != MMPROJ_SHA + or plan["server_sha256"] != SERVER_SHA + or plan["requested_device"] != "cpu" + or plan["offloaded_layers"] is not None + or not plan["cpu_only_layout"] + or plan["model_buffer_devices"] + or plan["all_model_buffers"] != ["CPU_Mapped"] + or plan["vision_backends"] != ["CPU"] + or not plan["disable_repack"] + or plan["loaded_rss_bytes"] > profile["reserved_host_ram_bytes"] + or profile["available_host_ram_before_bytes"] < profile["reserved_host_ram_bytes"]): + raise ValueError("WSL artifact, explicit no-repack CPU placement or admission changed") + ledger = profile["ledger_after_shutdown"] + if ledger["reserved"].get("host_ram") != 0 or ledger["owners"] or ledger["quarantined"]: + raise ValueError("WSL stage did not release its reservation") + events = [*profile["checks"]] + timing = {} + for kind, expected in (("text", "ready"), ("image", "Red")): + rows = profile[f"{kind}_measured"] + if (len(rows) != 20 or profile[f"{kind}_warmup"]["answer"] != expected + or any(row["kind"] != kind or row["answer"] != expected for row in rows)): + raise ValueError(f"WSL {kind} sample count/output changed") + events.extend([profile[f"{kind}_warmup"], *rows]) + walls = [row["wall_s"] for row in rows] + p50, p95 = rank(walls, 0.5), rank(walls, 0.95) + if (profile[f"nearest_rank_p50_{kind}_wall_s"] != p50 + or profile[f"nearest_rank_p95_{kind}_wall_s"] != p95): + raise ValueError(f"WSL {kind} percentile arithmetic changed") + timing[kind] = {"n": 20, "p50_s": p50, "p95_s": p95, + "min_s": min(walls), "max_s": max(walls)} + if (sorted(row["stage_event"]["epoch"] for row in events) != list(range(1, 45)) + or any(not row["stage_event"]["terminal"] or row["stage_event"]["seq"] != 1 + or row["stage_event"]["worker_generation"] != plan["worker_generation"] + for row in events)): + raise ValueError("WSL terminal event epochs missing or duplicated") + log = (root / "profile20.server.log").read_text(encoding="utf-8", errors="replace") + if ("warning: no usable GPU found" not in log + or "CPU_Mapped model buffer size" not in log + or "CPU_REPACK model buffer size" in log + or "CLIP using CPU backend" not in log): + raise ValueError("WSL server log does not confirm CPU/no-repack/vision placement") + files = sorted(path for path in root.iterdir() if path.suffix in (".json", ".log") + and path.name != "audit_report.json") + audit = { + "status": "audited_scoped_e2e", + "evidence_depth": "P: WSL CPU Omni complete text and synthetic-image requests", + "checkpoint": "ggml-org/Qwen3.8-27B-GGUF", + "revision": REVISION, + "quantization": "Q4_K_M language model + Q8_0 vision projector", + "model_sha256": MODEL_SHA, + "mmproj_sha256": MMPROJ_SHA, + "server_sha256": SERVER_SHA, + "host": "Ryzen AI 9 HX 370, Ubuntu 26.04 on WSL2, 30.91 GiB WSL RAM quota", + "runtime": "Omni checkout through Linux vLLM 0.29; llama.cpp b10849 CPU-only build, --no-repack", + "scope": "2048 context, 128 max new tokens, concurrency one, one 96x96 red PNG and one short text instruction", + "startup_s": profile["startup_s"], + "reserved_host_ram_bytes": profile["reserved_host_ram_bytes"], + "loaded_rss_bytes": plan["loaded_rss_bytes"], + "timing": timing, + "files": {path.name: digest(path) for path in files}, + "limits": [ + "The prompts produce two completion tokens; long-context, 128-token output and broad quality remain unqualified.", + "No public AsyncOmni, cancellation/restart, sampled loading/request memory peak, concurrency or sustained power/thermal profile was run.", + "The WSL quota, no-repack option and Linux llama.cpp b10849 differ from the native Windows CPU setup; latencies are not a paired OS comparison.", + ], + } + (root / "audit_report.json").write_text(json.dumps(audit, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": audit["status"], "timing": timing})) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py b/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py index 94c0c2fb1..f3844921e 100644 --- a/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py +++ b/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py @@ -25,6 +25,7 @@ async def main() -> None: parser.add_argument("--" + key.replace("_", "-"), required=True) parser.add_argument("--expected-device-name") parser.add_argument("--ggml-vk-visible-devices") + parser.add_argument("--no-repack", action="store_true") parser.add_argument("--reserve-gib", type=int, default=28) parser.add_argument("--repeats", type=int, default=1) args = parser.parse_args() @@ -65,6 +66,7 @@ async def main() -> None: "device": args.device, "expected_device_name": args.expected_device_name, "ggml_vk_visible_devices": args.ggml_vk_visible_devices, + "disable_repack": args.no_repack, "context_tokens": 2048, "max_new_tokens": 128, "max_io_bytes": 1 << 20, diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md index 97a7bddce..3714d2641 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md @@ -56,6 +56,16 @@ artifact hashes, CPU/Radeon model and vision placement, and ledger release were checked. These two-token responses do not settle long-context quality, memory peaks, public entrypoint, cancellation or sustained performance. +The same pinned model also passed a [WSL CPU no-repack Omni profile](evidence/qwen38_gguf_wsl_cpu/README.md). +Two 28 GiB pre-admission refusals and a 27 GiB loaded-RSS refusal were +retained. Disabling llama.cpp's CPU weight repacking removed a 9,180 MiB +buffer, allowing an explicit 24 GiB reservation within WSL's 30.91 GiB quota. +One warmup of each type plus 20/20 text and 20/20 image requests passed at +nearest-rank p50/p95 **1.934/1.983 s text** and **3.642/3.789 s image**. +The Linux CPU-only server build, quota and no-repack setting differ from the +native Windows runs, so these are distinct configurations rather than a paired +OS speed comparison. + A [native Windows RTX Qwen3-TTS Omni hybrid run](evidence/qwen_tts_windows_rtx/README.md) adds a separate GGUF/Vulkan complete-WAV path with RTX talker/codec and CPU FP32 code predictor. Two named requests, 20/20 serial outputs, public audio, diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md new file mode 100644 index 000000000..7b4788ffa --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md @@ -0,0 +1,9 @@ +# Qwen3.8-27B GGUF on WSL CPU + +The same pinned [ggml-org/Qwen3.8-27B-GGUF](https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF) Q4_K_M language model and Q8_0 vision projector as the [native Windows CPU run](../qwen38_gguf_windows_cpu/README.md) passed bounded complete text and image-to-text requests through Omni `StageRuntime` on WSL2 Ubuntu 26.04. The exact checkpoint revision is `efbb3b1f70a21d97fd4495240648405f7228554f`. The local Linux CPU-only llama.cpp server was b10849, commit `e71b80510`, SHA256 `0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c`, loaded by the current checkout with installed vLLM 0.29.0. [Hardware/runtime record](hardware_runtime.json). + +The WSL RAM quota is 30.91 GiB. Two [pre-admission](smoke.json) [attempts](smoke2.json) refused a 28 GiB reservation when available RAM after Python imports was below it. A [27 GiB attempt](smoke3.json) loaded the model but was correctly refused because its CPU-repacked process RSS exceeded that reservation. The explicit `--no-repack` option removed the 9,180 MiB repack buffer without changing weights or precision. A [24 GiB smoke](smoke4.json) then passed `ready` text and `Red` synthetic-image answers; the stage measured 20,026,560,512 bytes of loaded process RSS, verified `CPU_Mapped` model storage and `CLIP using CPU backend`, and fully released the reservation. The CPU-only build emits `warning: no usable GPU found` rather than an offload-count line, so the backend requires positive CPU-buffer evidence before labeling CPU placement. + +The [serial profile](profile20.json) used one 2048-token slot, a 128-token maximum output, deterministic temperature zero, a 96×96 solid-red PNG, and short prompts requesting `ready` and `Red`. After one warmup of each type, **20/20 text** and **20/20 image** requests returned the expected two-token answers. Nearest-rank complete-request p50/p95 was **1.934/1.983 s text** and **3.642/3.789 s image**, concurrency one. Startup was 11.45 s on this run and is excluded from those percentiles; the earlier smoke startup was 25.01 s, so cache state should be treated as uncontrolled. The [server log](profile20.server.log) and [auditor](audit_report.json) verify hashes, CPU model/vision placement, every output and terminal epoch, percentile arithmetic and ledger release. + +Reproduce with [the Omni probe](../../../../experiments/probe_omni_llamacpp_qwen38.py), the checkpoint hashes in the Windows CPU record, `--device cpu --reserve-gib 24 --no-repack --repeats 20`, the Linux server hash above, and fresh log/report paths. Use the vLLM 0.29 environment, `PYTHONPATH` pointing to this checkout, `LD_LIBRARY_PATH` pointing to the llama.cpp build directory, `CUDA_VISIBLE_DEVICES=''`, and `VLLM_ENABLE_V1_MULTIPROCESSING=0`. Run [the auditor](../../../../experiments/audit_qwen38_gguf_wsl.py) against this evidence directory. This is scoped P-level evidence for short text and one synthetic image, not long-context or broad visual quality, public AsyncOmni, cancellation/restart, concurrent serving, loading/request memory peaks or sustained power and thermal behavior. The WSL quota and no-repack setting differ from native Windows, so their latencies are not a paired OS comparison. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/audit_report.json new file mode 100644 index 000000000..5d2f13f5e --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/audit_report.json @@ -0,0 +1,95 @@ +{ + "status": "audited_scoped_e2e", + "evidence_depth": "P: WSL CPU Omni complete text and synthetic-image requests", + "checkpoint": "ggml-org/Qwen3.8-27B-GGUF", + "revision": "efbb3b1f70a21d97fd4495240648405f7228554f", + "quantization": "Q4_K_M language model + Q8_0 vision projector", + "model_sha256": "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747", + "mmproj_sha256": "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb", + "server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c", + "host": "Ryzen AI 9 HX 370, Ubuntu 26.04 on WSL2, 30.91 GiB WSL RAM quota", + "runtime": "Omni checkout through Linux vLLM 0.29; llama.cpp b10849 CPU-only build, --no-repack", + "scope": "2048 context, 128 max new tokens, concurrency one, one 96x96 red PNG and one short text instruction", + "startup_s": 11.45404783997219, + "reserved_host_ram_bytes": 25769803776, + "loaded_rss_bytes": 20026531840, + "timing": { + "text": { + "n": 20, + "p50_s": 1.9337806559633464, + "p95_s": 1.983029014954809, + "min_s": 1.8979981419979595, + "max_s": 2.0445767869823612 + }, + "image": { + "n": 20, + "p50_s": 3.6416604099795222, + "p95_s": 3.788667813001666, + "min_s": 3.564662608958315, + "max_s": 3.81039645295823 + } + }, + "files": { + "hardware_runtime.json": { + "bytes": 627, + "sha256": "df65f32090c0f0ff3bf3db6cb2b8e3060138ee3136bff9074d4da60f3eec3958" + }, + "profile20.driver.log": { + "bytes": 608, + "sha256": "ecc46c7d39d9d8e84cd860579d48b305fecb2a5fd50b1e2475865e82ca84f849" + }, + "profile20.json": { + "bytes": 32936, + "sha256": "d81554c7f7d8e9e782f5b99b0716a5ca60d3fd1eacd0a5366e413ca6f8931b64" + }, + "profile20.server.log": { + "bytes": 144420, + "sha256": "0324cff765b7ca890ad994b66a634233ce4ff19a956ea1d54c2c0402d7fa056e" + }, + "smoke.driver.log": { + "bytes": 1541, + "sha256": "3e9046f4b6cf81c9510e73d3e61d48dc4cdee6414ab81e74950f5aa4a9e8f345" + }, + "smoke.json": { + "bytes": 557, + "sha256": "d05ba97947654feed445b47cd5d043500f452612d0dd97c0f797181c4e8324cf" + }, + "smoke2.driver.log": { + "bytes": 1541, + "sha256": "7d6e4c2bf144315044991bbcf7dc5ef6e1d6e1927d693f86b9df188d1e0fc9e4" + }, + "smoke2.json": { + "bytes": 557, + "sha256": "fb197382bc717af111c54d26464f682897f7fdbd3cb4aafea2b1b5b7377c3486" + }, + "smoke3.driver.log": { + "bytes": 6058, + "sha256": "402d41f730c3cb64d6caea2522f39263d231139d9b12f78001a1e81809d70da9" + }, + "smoke3.json": { + "bytes": 728, + "sha256": "6e472ab0816f9c9b1cfe220636b318c1697c94386276869b58bb80c14c464591" + }, + "smoke3.server.log": { + "bytes": 19265, + "sha256": "cc6919d965657beb5ea5ef279a590449293b6121cf8ea42689f3e8a1e20ff047" + }, + "smoke4.driver.log": { + "bytes": 608, + "sha256": "88e7a14179c1ecc860b0e4a7c1a74081cb84fc424f42d6a0af0ff109804a7b54" + }, + "smoke4.json": { + "bytes": 6366, + "sha256": "24dd653a4ffaf392a828a394d243758af83c194f97606f6e064a45319d8f73db" + }, + "smoke4.server.log": { + "bytes": 35797, + "sha256": "6f765837700c4210378d4128fae8781e3a7fe0616513ce4d88eac073307910ca" + } + }, + "limits": [ + "The prompts produce two completion tokens; long-context, 128-token output and broad quality remain unqualified.", + "No public AsyncOmni, cancellation/restart, sampled loading/request memory peak, concurrency or sustained power/thermal profile was run.", + "The WSL quota, no-repack option and Linux llama.cpp b10849 differ from the native Windows CPU setup; latencies are not a paired OS comparison." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/hardware_runtime.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/hardware_runtime.json new file mode 100644 index 000000000..ec524ff54 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/hardware_runtime.json @@ -0,0 +1,14 @@ +{ + "captured_utc": "2026-09-24T10:12:23Z", + "os": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "kernel": "6.18.33.2-microsoft-standard-WSL2", + "cpu": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M", + "wsl_memory_total_bytes": 33184976896, + "wsl_memory_available_after_profile_bytes": 31320154112, + "vllm_package": "0.29.0", + "omni_package": "0.29.0rc2.dev71+g00510ce38.d20260922", + "llama_cpp_build": 10849, + "llama_cpp_commit": "e71b80510", + "server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c", + "gpu_visibility": "CUDA_VISIBLE_DEVICES empty; server -dev none -ngl 0" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.driver.log new file mode 100644 index 000000000..2043eb7ed --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.driver.log @@ -0,0 +1,6 @@ +INFO 09-24 18:09:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 18:09:18 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 18:09:18 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-24 18:09:19 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 18:09:19 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge' +{"status": "completed", "error": null} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.json new file mode 100644 index 000000000..e63187510 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.json @@ -0,0 +1,1300 @@ +{ + "scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp", + "device": "cpu", + "available_host_ram_before_bytes": 30044393472, + "reserved_host_ram_bytes": 25769803776, + "repeats": 20, + "model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf", + "startup_s": 11.45404783997219, + "execution_plan": { + "backend": "external.llamacpp.multimodal.v1", + "stage_id": 0, + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "model_sha256": "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747", + "mmproj_sha256": "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb", + "server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c", + "model_alias": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "requested_device": "cpu", + "expected_device_name": null, + "layer_assignments": 0, + "offloaded_layers": null, + "model_buffer_devices": [], + "all_model_buffers": [ + "CPU_Mapped" + ], + "vision_backends": [ + "CPU" + ], + "cpu_only_layout": true, + "worker_pid": 346530, + "loaded_rss_bytes": 20026531840, + "reserved_bytes": { + "host_ram": 25769803776 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 4294967296, + "disable_repack": true, + "context_tokens": 2048, + "max_new_tokens": 128, + "max_io_bytes": 1048576, + "max_image_bytes": 262144, + "image_token_reserve": 512, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "kind": "text", + "wall_s": 1.96138950396562, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493926168433292", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9454641510383226, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.5819617649540305, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493928129927976", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.57794605800882, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "text_warmup": { + "kind": "text", + "wall_s": 1.9073279140284285, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493931711939905", + "stage_id": 0, + "epoch": 3, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.8940413969685324, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + "text_measured": [ + { + "kind": "text", + "wall_s": 1.8979981419979595, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493933619356709", + "stage_id": 0, + "epoch": 4, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.8895888809929602, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9062753170146607, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493935517455692", + "stage_id": 0, + "epoch": 5, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9033502800157294, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.919998081983067, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493937423780372", + "stage_id": 0, + "epoch": 6, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9099342639674433, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.950439628039021, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493939343829792", + "stage_id": 0, + "epoch": 7, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9384903000318445, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9337806559633464, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493941294346105", + "stage_id": 0, + "epoch": 8, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9258593799895607, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9696584059856832, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493943228189851", + "stage_id": 0, + "epoch": 9, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9657905130297877, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9485176269663498, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493945197928239", + "stage_id": 0, + "epoch": 10, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9415814050007612, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9467064820346422, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493947146521459", + "stage_id": 0, + "epoch": 11, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9439987280056812, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.952661233022809, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493949093270391", + "stage_id": 0, + "epoch": 12, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9507893080008216, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.939775890961755, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493951046021134", + "stage_id": 0, + "epoch": 13, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9299813079996966, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9129057030077092, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493952985843343", + "stage_id": 0, + "epoch": 14, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.906580634007696, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9593236920190975, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493954898834438", + "stage_id": 0, + "epoch": 15, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9523620010004379, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9085550550371408, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493956858244994", + "stage_id": 0, + "epoch": 16, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9002185669960454, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9612433839938603, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493958766849684", + "stage_id": 0, + "epoch": 17, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9563226600294001, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.929774140007794, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493960728200421", + "stage_id": 0, + "epoch": 18, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.917872660967987, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9313761689700186, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493962658049012", + "stage_id": 0, + "epoch": 19, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9214814149891026, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9099857420078479, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493964589480506", + "stage_id": 0, + "epoch": 20, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9014397810096852, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.9190144650056027, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493966499669964", + "stage_id": 0, + "epoch": 21, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9141177500132471, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 1.983029014954809, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493968418768739", + "stage_id": 0, + "epoch": 22, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.9778126499732025, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "text", + "wall_s": 2.0445767869823612, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493970401875241", + "stage_id": 0, + "epoch": 23, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0370960899745114, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_text_wall_s": 1.9337806559633464, + "nearest_rank_p95_text_wall_s": 1.983029014954809, + "image_warmup": { + "kind": "image", + "wall_s": 3.6845831210375763, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493972446566015", + "stage_id": 0, + "epoch": 24, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6721105839824304, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + "image_measured": [ + { + "kind": "image", + "wall_s": 3.6826974829891697, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493976131215302", + "stage_id": 0, + "epoch": 25, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6805525199742988, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.611066625046078, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493979813959483", + "stage_id": 0, + "epoch": 26, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6059130809735507, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.6321385559858754, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493983425107423", + "stage_id": 0, + "epoch": 27, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6198520539910533, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.564662608958315, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493987057297537", + "stage_id": 0, + "epoch": 28, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.553073022980243, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.6644317519967444, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493990622029679", + "stage_id": 0, + "epoch": 29, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.658718942955602, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.5895186600391753, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493994286524861", + "stage_id": 0, + "epoch": 30, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.5841297489823774, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.6193048440036364, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493997876133672", + "stage_id": 0, + "epoch": 31, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6111764350207523, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.6640040000202134, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494001495504771", + "stage_id": 0, + "epoch": 32, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6607264730264433, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.724349104973953, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494005159558615", + "stage_id": 0, + "epoch": 33, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.7122389409923926, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.7457145350053906, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494008883961552", + "stage_id": 0, + "epoch": 34, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.7399401150178164, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.566754536004737, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494012629919409", + "stage_id": 0, + "epoch": 35, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.563712653995026, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.6416604099795222, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494016196732086", + "stage_id": 0, + "epoch": 36, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.629788402991835, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.5681937360204756, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494019838452119", + "stage_id": 0, + "epoch": 37, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.565549254999496, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.570742601004895, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494023406706411", + "stage_id": 0, + "epoch": 38, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.5585884419851936, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.628626898978837, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494026977527280", + "stage_id": 0, + "epoch": 39, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.6227733539999463, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.788667813001666, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494030606237417", + "stage_id": 0, + "epoch": 40, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.786554278980475, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.81039645295823, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494034394976195", + "stage_id": 0, + "epoch": 41, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.801060899044387, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.725606572057586, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494038205449815", + "stage_id": 0, + "epoch": 42, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.7208398359944113, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.671434965042863, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494041931131130", + "stage_id": 0, + "epoch": 43, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.65063127403846, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.6708527610171586, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-494045602610729", + "stage_id": 0, + "epoch": 44, + "seq": 1, + "kind": "text", + "worker_generation": "9aa973dd3066419cb8f52465b95b8706", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.66373421397293, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_image_wall_s": 3.6416604099795222, + "nearest_rank_p95_image_wall_s": 3.788667813001666, + "status": "completed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 30044393472 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.server.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.server.log new file mode 100644 index 000000000..bfeb99ce9 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/profile20.server.log @@ -0,0 +1,1452 @@ +warning: no usable GPU found, --gpu-layers option will be ignored +warning: one possible reason is that llama.cpp was compiled without GPU support +warning: consult docs/build.md for compilation instructions +0.00.002.635 I cmn common_param: common_params_print_info: build 10849 (e71b80510) with GNU 11.4.0 for Linux x86_64 +0.00.002.657 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.002.657 I cmn common_param: device_info: +0.00.002.669 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (31647 MiB, 31647 MiB free) +0.00.002.732 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.002.842 I srv init: using 23 threads for HTTP server +0.00.002.861 I srv init: The UI is disabled +0.00.002.862 I srv init: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.002.916 W srv llama_server: ----------------- +0.00.002.934 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set +0.00.002.934 W srv llama_server: this can be a security risk (cross-origin attacks) +0.00.002.934 W srv llama_server: more info: https://github.com/ggml-org/llama.cpp/pull/25655 +0.00.002.935 W srv llama_server: ----------------- +0.00.002.957 I srv start: binding port with default address family +0.00.004.558 I srv load_model: loading model '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf' +0.00.004.578 I srv load_model: local path '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf' +0.00.046.634 I llama_model_loader: loaded meta data with 40 key-value pairs and 851 tensors from /home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.046.699 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.046.723 I llama_model_loader: - kv 0: general.architecture str = qwen35 +0.00.046.725 I llama_model_loader: - kv 1: general.type str = model +0.00.046.729 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +0.00.046.735 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.046.736 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.046.736 I llama_model_loader: - kv 5: general.name str = Qwen3.8-27B +0.00.046.736 I llama_model_loader: - kv 6: general.size_label str = 27B +0.00.046.736 I llama_model_loader: - kv 7: general.license str = apache-2.0 +0.00.046.758 I llama_model_loader: - kv 8: general.tags arr[str,1] = ["image-text-to-text"] +0.00.046.758 I llama_model_loader: - kv 9: qwen35.block_count u32 = 64 +0.00.046.759 I llama_model_loader: - kv 10: qwen35.context_length u32 = 262144 +0.00.046.759 I llama_model_loader: - kv 11: qwen35.embedding_length u32 = 5120 +0.00.046.760 I llama_model_loader: - kv 12: qwen35.feed_forward_length u32 = 17408 +0.00.046.760 I llama_model_loader: - kv 13: qwen35.attention.head_count u32 = 24 +0.00.046.760 I llama_model_loader: - kv 14: qwen35.attention.head_count_kv u32 = 4 +0.00.046.763 I llama_model_loader: - kv 15: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +0.00.046.764 I llama_model_loader: - kv 16: qwen35.rope.freq_base f32 = 10000000.000000 +0.00.046.765 I llama_model_loader: - kv 17: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.046.766 I llama_model_loader: - kv 18: qwen35.attention.key_length u32 = 256 +0.00.046.766 I llama_model_loader: - kv 19: qwen35.attention.value_length u32 = 256 +0.00.046.766 I llama_model_loader: - kv 20: qwen35.ssm.conv_kernel u32 = 4 +0.00.046.767 I llama_model_loader: - kv 21: qwen35.ssm.state_size u32 = 128 +0.00.046.767 I llama_model_loader: - kv 22: qwen35.ssm.group_count u32 = 16 +0.00.046.767 I llama_model_loader: - kv 23: qwen35.ssm.time_step_rank u32 = 48 +0.00.046.767 I llama_model_loader: - kv 24: qwen35.ssm.inner_size u32 = 6144 +0.00.046.771 I llama_model_loader: - kv 25: qwen35.attention.recurrent_layers arr[bool,64] = [true, true, true, false, true, true,... +0.00.046.771 I llama_model_loader: - kv 26: qwen35.full_attention_interval u32 = 4 +0.00.046.771 I llama_model_loader: - kv 27: qwen35.rope.dimension_count u32 = 64 +0.00.046.771 I llama_model_loader: - kv 28: tokenizer.ggml.model str = gpt2 +0.00.046.772 I llama_model_loader: - kv 29: tokenizer.ggml.pre str = qwen35 +0.00.063.411 I llama_model_loader: - kv 30: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +0.00.066.784 I llama_model_loader: - kv 31: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +0.00.081.056 I llama_model_loader: - kv 32: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +0.00.081.085 I llama_model_loader: - kv 33: tokenizer.ggml.eos_token_id u32 = 248046 +0.00.081.086 I llama_model_loader: - kv 34: tokenizer.ggml.padding_token_id u32 = 248044 +0.00.081.086 I llama_model_loader: - kv 35: tokenizer.ggml.bos_token_id u32 = 248044 +0.00.081.087 I llama_model_loader: - kv 36: tokenizer.ggml.add_bos_token bool = false +0.00.081.091 I llama_model_loader: - kv 37: tokenizer.chat_template str = {%- set image_count = namespace(value... +0.00.081.091 I llama_model_loader: - kv 38: general.quantization_version u32 = 2 +0.00.081.092 I llama_model_loader: - kv 39: general.file_type u32 = 15 +0.00.081.099 I llama_model_loader: - type f32: 353 tensors +0.00.081.100 I llama_model_loader: - type q8_0: 288 tensors +0.00.081.100 I llama_model_loader: - type q4_K: 193 tensors +0.00.081.100 I llama_model_loader: - type q6_K: 17 tensors +0.00.081.104 I print_info: file format = GGUF V3 (latest) +0.00.081.107 I print_info: file type = Q4_K - Medium +0.00.081.113 I print_info: file size = 17.66 GiB (5.64 BPW) +0.00.288.349 I load: 0 unused tokens +0.00.348.739 I load: printing all EOG tokens: +0.00.348.773 I load: - 248044 ('<|endoftext|>') +0.00.348.775 I load: - 248046 ('<|im_end|>') +0.00.348.776 I load: - 248063 ('<|fim_pad|>') +0.00.348.776 I load: - 248064 ('<|repo_name|>') +0.00.348.776 I load: - 248065 ('<|file_sep|>') +0.00.349.029 I load: special tokens cache size = 33 +0.00.403.678 I load: token to piece cache size = 1.7581 MB +0.00.403.732 I print_info: arch = qwen35 +0.00.403.733 I print_info: vocab_only = 0 +0.00.403.733 I print_info: no_alloc = 0 +0.00.403.733 I print_info: n_ctx_train = 262144 +0.00.403.735 I print_info: n_embd_inp = 5120 +0.00.403.735 I print_info: n_embd = 5120 +0.00.403.737 I print_info: n_embd_out = 5120 +0.00.403.737 I print_info: n_layer = 64 +0.00.403.737 I print_info: n_layer_all = 64 +0.00.403.753 I print_info: n_head = 24 +0.00.403.756 I print_info: n_head_kv = 4 +0.00.403.756 I print_info: n_rot = 64 +0.00.403.756 I print_info: n_swa = 0 +0.00.403.758 I print_info: is_swa_any = 0 +0.00.403.758 I print_info: non_causal_type = 0 +0.00.403.758 I print_info: n_embd_head_k = 256 +0.00.403.758 I print_info: n_embd_head_v = 256 +0.00.403.760 I print_info: n_gqa = 6 +0.00.403.765 I print_info: n_embd_k_gqa = 1024 +0.00.403.769 I print_info: n_embd_v_gqa = 1024 +0.00.403.770 I print_info: f_norm_eps = 0.0e+00 +0.00.403.771 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.403.771 I print_info: f_clamp_kqv = 0.0e+00 +0.00.403.772 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.403.772 I print_info: f_logit_scale = 0.0e+00 +0.00.403.772 I print_info: f_attn_scale = 0.0e+00 +0.00.403.772 I print_info: f_attn_value_scale = 0.0000 +0.00.403.774 I print_info: n_ff = 17408 +0.00.403.774 I print_info: n_expert = 0 +0.00.403.774 I print_info: n_expert_used = 0 +0.00.403.774 I print_info: n_expert_groups = 0 +0.00.403.775 I print_info: n_group_used = 0 +0.00.403.775 I print_info: causal attn = 1 +0.00.403.775 I print_info: pooling type = -1 +0.00.403.775 I print_info: rope type = 40 +0.00.403.775 I print_info: rope scaling = linear +0.00.403.776 I print_info: freq_base_train = 10000000.0 +0.00.403.776 I print_info: freq_scale_train = 1 +0.00.403.777 I print_info: n_ctx_orig_yarn = 262144 +0.00.403.777 I print_info: rope_yarn_log_mul = 0.0000 +0.00.403.777 I print_info: rope_finetuned = unknown +0.00.403.777 I print_info: mrope sections = [11, 11, 10, 0] +0.00.403.777 I print_info: ssm_d_conv = 4 +0.00.403.777 I print_info: ssm_d_inner = 6144 +0.00.403.777 I print_info: ssm_d_state = 128 +0.00.403.778 I print_info: ssm_dt_rank = 48 +0.00.403.778 I print_info: ssm_n_group = 16 +0.00.403.778 I print_info: ssm_dt_b_c_rms = 0 +0.00.403.779 I print_info: model type = 27B +0.00.403.780 I print_info: model params = 26.90 B +0.00.403.781 I print_info: general.name = Qwen3.8-27B +0.00.403.785 I print_info: vocab type = BPE +0.00.403.787 I print_info: n_vocab = 248320 +0.00.403.787 I print_info: n_merges = 247587 +0.00.403.787 I print_info: BOS token = 248044 '<|endoftext|>' +0.00.403.787 I print_info: EOS token = 248046 '<|im_end|>' +AMX is not ready to be used! +0.00.403.787 I print_info: EOT token = 248046 '<|im_end|>' +0.00.403.787 I print_info: PAD token = 248044 '<|endoftext|>' +0.00.403.788 I print_info: LF token = 198 'Ċ' +0.00.403.788 I print_info: FIM PRE token = 248060 '<|fim_prefix|>' +0.00.403.788 I print_info: FIM SUF token = 248062 '<|fim_suffix|>' +0.00.403.788 I print_info: FIM MID token = 248061 '<|fim_middle|>' +0.00.403.788 I print_info: FIM PAD token = 248063 '<|fim_pad|>' +0.00.403.789 I print_info: FIM REP token = 248064 '<|repo_name|>' +0.00.403.789 I print_info: FIM SEP token = 248065 '<|file_sep|>' +0.00.403.789 I print_info: EOG token = 248044 '<|endoftext|>' +0.00.403.790 I print_info: EOG token = 248046 '<|im_end|>' +0.00.403.790 I print_info: EOG token = 248063 '<|fim_pad|>' +0.00.403.790 I print_info: EOG token = 248064 '<|repo_name|>' +0.00.403.790 I print_info: EOG token = 248065 '<|file_sep|>' +0.00.403.790 I print_info: max token length = 256 +0.00.403.794 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap) +0.00.476.439 I load_tensors: CPU_Mapped model buffer size = 18084.41 MiB +0.00.481.450 I cmn common_init_: added <|endoftext|> logit bias = -inf +0.00.481.477 I cmn common_init_: added <|im_end|> logit bias = -inf +0.00.481.478 I cmn common_init_: added <|fim_pad|> logit bias = -inf +0.00.481.478 I cmn common_init_: added <|repo_name|> logit bias = -inf +0.00.481.479 I cmn common_init_: added <|file_sep|> logit bias = -inf +0.00.481.576 I llama_context: constructing llama_context +0.00.481.600 I llama_context: n_seq_max = 1 +0.00.481.601 I llama_context: n_ctx = 2048 +0.00.481.601 I llama_context: n_ctx_seq = 2048 +0.00.481.601 I llama_context: n_batch = 2048 +0.00.481.601 I llama_context: n_ubatch = 512 +0.00.481.601 I llama_context: causal_attn = 1 +0.00.481.603 I llama_context: flash_attn = auto +0.00.481.604 I llama_context: kv_unified = false +0.00.481.606 I llama_context: freq_base = 10000000.0 +0.00.481.607 I llama_context: freq_scale = 1 +0.00.481.607 I llama_context: n_rs_seq = 0 +0.00.481.607 I llama_context: n_outputs_max = 1 +0.00.481.607 I llama_context: n_outputs_max_per_seq = 1 +0.00.481.607 I llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.00.481.665 I llama_context: CPU output buffer size = 0.95 MiB +0.00.481.777 I llama_kv_cache: CPU KV buffer size = 128.00 MiB +0.00.574.990 I llama_kv_cache: size = 128.00 MiB ( 2048 cells, 16 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB +0.00.575.031 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.00.575.031 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.00.689.964 I llama_memory_recurrent: CPU RS buffer size = 149.62 MiB +0.00.690.016 I llama_memory_recurrent: size = 149.62 MiB ( 1 cells, 64 layers, 1 seqs 0 rs_seq), R (f32): 5.62 MiB, S (f32): 144.00 MiB, P (f32): 0.00 MiB +0.00.690.041 I sched_reserve: reserving ... +0.00.707.877 I resolve_fused_ops: Flash Attention enabled +0.00.707.902 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.00.708.434 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.00.708.946 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.00.709.471 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.00.711.709 I sched_reserve: CPU compute buffer size = 134.02 MiB +0.00.711.732 I sched_reserve: graph nodes = 3751 +0.00.711.732 I sched_reserve: graph splits = 1 +0.00.711.735 I sched_reserve: reserve took 21.69 ms, sched copies = 1 +0.00.711.802 I cmn init: llama threadpool init, n_threads = 12 +0.00.711.840 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.01.078.344 I clip_model_loader: model name: Qwen3.8-27B +0.01.078.372 I clip_model_loader: description: +0.01.078.376 I clip_model_loader: GGUF version: 3 +0.01.078.378 I clip_model_loader: alignment: 32 +0.01.078.379 I clip_model_loader: n_tensors: 334 +0.01.078.380 I clip_model_loader: n_kv: 26 +0.01.078.381 I +0.01.078.385 I clip_model_loader: has vision encoder +0.01.078.682 I clip_ctx: CLIP using CPU backend +0.01.079.232 W load_hparams: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks +0.01.079.253 W load_hparams: if you encounter problems with accuracy, try adding --image-min-tokens 1024 +0.01.079.253 W load_hparams: more info: https://github.com/ggml-org/llama.cpp/issues/16842 + +0.01.079.253 I load_hparams: projector: qwen3vl_merger +0.01.079.254 I load_hparams: n_embd: 1152 +0.01.079.254 I load_hparams: n_head: 16 +0.01.079.254 I load_hparams: n_ff: 4304 +0.01.079.254 I load_hparams: n_layer: 27 +0.01.079.254 I load_hparams: ffn_op: gelu +0.01.079.255 I load_hparams: projection_dim: 5120 +0.01.079.255 I +--- vision hparams --- +0.01.079.255 I load_hparams: image_size: 768 +0.01.079.255 I load_hparams: patch_size: 16 +0.01.079.256 I load_hparams: has_llava_proj: 0 +0.01.079.256 I load_hparams: minicpmv_version: 0 +0.01.079.256 I load_hparams: n_merge: 2 +0.01.079.256 I load_hparams: n_wa_pattern: 0 +0.01.079.256 I load_hparams: image_min_pixels: 8192 +0.01.079.256 I load_hparams: image_max_pixels: 4194304 +0.01.079.256 I +0.01.079.262 I load_hparams: model size: 600.08 MiB +0.01.079.265 I load_hparams: metadata size: 0.12 MiB +0.01.588.744 I get_dummy_batch: warmup with image size = 1472 x 1472 +0.01.617.639 I get_dummy_batch: warmup with image size = 1472 x 1472 +0.01.626.262 I reserve_compute_meta: CPU compute buffer size = 248.10 MiB +0.01.626.295 I reserve_compute_meta: graph splits = 1, nodes = 823 +0.01.626.325 I warmup: flash attention is enabled +0.01.629.053 I srv load_model: loaded multimodal model, '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf' +0.01.987.746 I cmn common_conte: the context does not support partial sequence removal +0.01.998.861 I srv load_model: speculative decoding will use checkpoints +0.01.998.910 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 2048, kv_unified = 'false' +0.01.998.929 I spec common_specu: no implementations specified for speculative decoding +0.01.998.933 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048 +0.01.999.009 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.01.999.009 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.01.999.010 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.01.999.035 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.02.003.744 I srv init: init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + + + + +' +0.02.004.248 I srv init: init: chat template, thinking = 0 +0.02.004.286 I srv init: preserve_reasoning kwarg: true +0.02.004.286 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.02.004.337 I srv llama_server: model loaded +0.02.004.361 I srv llama_server: listening on http://127.0.0.1:39703 +0.02.004.517 I srv update_slots: all slots are idle +0.02.036.055 I srv server_strea: conv_id= (empty=1) +0.02.036.309 I srv operator(): chat format: peg-native +0.02.036.489 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.02.036.509 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.02.036.567 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.02.036.596 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.02.036.597 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.02.036.610 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.02.036.613 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.03.139.629 I slot operator(): id 0 | task 0 | cached n_tokens = 15, memory_seq_rm [15, end) +0.03.139.722 I slot init_sampler: id 0 | task 0 | init sampler, took 0.02 ms, tokens: text = 19, total = 19 +0.03.238.850 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.03.978.368 I slot print_timing: id 0 | task 0 | prompt eval time = 1634.44 ms / 19 tokens ( 86.02 ms per token, 11.62 tokens per second) +0.03.978.408 I slot print_timing: id 0 | task 0 | eval time = 307.28 ms / 2 tokens ( 307.28 ms per token, 3.25 tokens per second) +0.03.978.409 I slot print_timing: id 0 | task 0 | total time = 1941.72 ms / 21 tokens +0.03.978.417 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.03.978.488 I slot release: id 0 | task 0 | stop processing: n_tokens = 20, truncated = 0 +0.03.978.500 I srv update_slots: all slots are idle +0.03.988.171 I srv server_strea: conv_id= (empty=1) +0.03.988.679 I srv operator(): chat format: peg-native +0.03.988.895 I slot get_availabl: id 0 | task -1 | - checking sim = 0.081 (3/37) > 0.100 +0.03.988.915 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 493928124752 +0.03.988.976 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.03.989.000 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.03.989.001 I slot launch_slot_: id 0 | task 4 | processing task, is_child = 0 +0.03.989.008 I slot operator(): id 0 | task 4 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.03.989.011 I slot operator(): id 0 | task 4 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.03.997.421 I slot operator(): id 0 | task 4 | cached n_tokens = 0, memory_seq_rm [0, end) +0.05.325.768 I slot operator(): id 0 | task 4 | cached n_tokens = 18, memory_seq_rm [18, end) +0.05.325.873 I slot process_mtmd: id 0 | task 4 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.05.390.733 I decoding image batch 1/1, n_tokens_batch = 9 +0.05.390.827 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.05.390.852 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.06.136.295 I image decoded (batch 1/1) in 745 ms +0.06.136.681 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.06.136.743 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.06.721.526 I slot operator(): id 0 | task 4 | cached n_tokens = 33, memory_seq_rm [27, end) +0.06.721.594 I slot init_sampler: id 0 | task 4 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.06.830.512 I slot create_check: id 0 | task 4 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.07.563.176 I slot print_timing: id 0 | task 4 | prompt eval time = 3271.27 ms / 37 tokens ( 88.41 ms per token, 11.31 tokens per second) +0.07.563.235 I slot print_timing: id 0 | task 4 | eval time = 302.87 ms / 2 tokens ( 302.87 ms per token, 3.30 tokens per second) +0.07.563.236 I slot print_timing: id 0 | task 4 | total time = 3574.14 ms / 39 tokens +0.07.563.238 I slot print_timing: id 0 | task 4 | graphs reused = 1 +0.07.563.299 I slot release: id 0 | task 4 | stop processing: n_tokens = 38, truncated = 0 +0.07.563.327 I srv update_slots: all slots are idle +0.07.578.070 I srv server_strea: conv_id= (empty=1) +0.07.578.268 I srv operator(): chat format: peg-native +0.07.578.411 I slot get_availabl: id 0 | task -1 | - checking sim = 0.158 (3/19) > 0.100 +0.07.578.432 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 0.158 (> 0.100 thold), f_keep = 0.079 +0.07.578.483 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.07.578.507 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.07.578.508 I slot launch_slot_: id 0 | task 9 | processing task, is_child = 0 +0.07.578.515 I slot operator(): id 0 | task 9 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.07.578.518 I slot operator(): id 0 | task 9 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.07.586.999 I slot operator(): id 0 | task 9 | cached n_tokens = 0, memory_seq_rm [0, end) +0.08.676.356 I slot operator(): id 0 | task 9 | cached n_tokens = 15, memory_seq_rm [15, end) +0.08.676.427 I slot init_sampler: id 0 | task 9 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.08.742.375 I slot create_check: id 0 | task 9 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.09.469.484 I slot print_timing: id 0 | task 9 | prompt eval time = 1586.12 ms / 19 tokens ( 83.48 ms per token, 11.98 tokens per second) +0.09.469.535 I slot print_timing: id 0 | task 9 | eval time = 304.82 ms / 2 tokens ( 304.82 ms per token, 3.28 tokens per second) +0.09.469.536 I slot print_timing: id 0 | task 9 | total time = 1890.94 ms / 21 tokens +0.09.469.538 I slot print_timing: id 0 | task 9 | graphs reused = 1 +0.09.469.601 I slot release: id 0 | task 9 | stop processing: n_tokens = 20, truncated = 0 +0.09.469.624 I srv update_slots: all slots are idle +0.09.476.701 I srv server_strea: conv_id= (empty=1) +0.09.476.951 I srv operator(): chat format: peg-native +0.09.477.145 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.09.477.147 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.09.477.213 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.09.477.243 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.09.477.244 I slot launch_slot_: id 0 | task 13 | processing task, is_child = 0 +0.09.477.253 I slot operator(): id 0 | task 13 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.09.477.256 I slot operator(): id 0 | task 13 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.09.485.559 I slot operator(): id 0 | task 13 | cached n_tokens = 0, memory_seq_rm [0, end) +0.10.574.876 I slot operator(): id 0 | task 13 | cached n_tokens = 15, memory_seq_rm [15, end) +0.10.574.957 I slot init_sampler: id 0 | task 13 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.10.633.347 I slot create_check: id 0 | task 13 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.11.363.642 I slot print_timing: id 0 | task 13 | prompt eval time = 1576.40 ms / 19 tokens ( 82.97 ms per token, 12.05 tokens per second) +0.11.363.679 I slot print_timing: id 0 | task 13 | eval time = 309.96 ms / 2 tokens ( 309.96 ms per token, 3.23 tokens per second) +0.11.363.680 I slot print_timing: id 0 | task 13 | total time = 1886.36 ms / 21 tokens +0.11.363.682 I slot print_timing: id 0 | task 13 | graphs reused = 1 +0.11.363.733 I slot release: id 0 | task 13 | stop processing: n_tokens = 20, truncated = 0 +0.11.363.752 I srv update_slots: all slots are idle +0.11.374.167 I srv server_strea: conv_id= (empty=1) +0.11.374.370 I srv operator(): chat format: peg-native +0.11.374.524 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.11.374.546 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.11.374.593 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.11.374.618 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.11.374.619 I slot launch_slot_: id 0 | task 17 | processing task, is_child = 0 +0.11.374.625 I slot operator(): id 0 | task 17 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.11.374.628 I slot operator(): id 0 | task 17 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.11.384.069 I slot operator(): id 0 | task 17 | cached n_tokens = 0, memory_seq_rm [0, end) +0.12.467.916 I slot operator(): id 0 | task 17 | cached n_tokens = 15, memory_seq_rm [15, end) +0.12.467.985 I slot init_sampler: id 0 | task 17 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.12.526.758 I slot create_check: id 0 | task 17 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.13.275.122 I slot print_timing: id 0 | task 17 | prompt eval time = 1581.93 ms / 19 tokens ( 83.26 ms per token, 12.01 tokens per second) +0.13.275.167 I slot print_timing: id 0 | task 17 | eval time = 318.53 ms / 2 tokens ( 318.53 ms per token, 3.14 tokens per second) +0.13.275.168 I slot print_timing: id 0 | task 17 | total time = 1900.46 ms / 21 tokens +0.13.275.169 I slot print_timing: id 0 | task 17 | graphs reused = 1 +0.13.275.225 I slot release: id 0 | task 17 | stop processing: n_tokens = 20, truncated = 0 +0.13.275.244 I srv update_slots: all slots are idle +0.13.280.896 I srv server_strea: conv_id= (empty=1) +0.13.281.084 I srv operator(): chat format: peg-native +0.13.281.269 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.13.281.294 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.13.281.353 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.13.281.360 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.13.281.361 I slot launch_slot_: id 0 | task 21 | processing task, is_child = 0 +0.13.281.369 I slot operator(): id 0 | task 21 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.13.281.371 I slot operator(): id 0 | task 21 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.13.289.992 I slot operator(): id 0 | task 21 | cached n_tokens = 0, memory_seq_rm [0, end) +0.14.385.847 I slot operator(): id 0 | task 21 | cached n_tokens = 15, memory_seq_rm [15, end) +0.14.385.917 I slot init_sampler: id 0 | task 21 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.14.445.826 I slot create_check: id 0 | task 21 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.15.187.849 I slot print_timing: id 0 | task 21 | prompt eval time = 1589.08 ms / 19 tokens ( 83.64 ms per token, 11.96 tokens per second) +0.15.187.889 I slot print_timing: id 0 | task 21 | eval time = 317.37 ms / 2 tokens ( 317.37 ms per token, 3.15 tokens per second) +0.15.187.890 I slot print_timing: id 0 | task 21 | total time = 1906.45 ms / 21 tokens +0.15.187.892 I slot print_timing: id 0 | task 21 | graphs reused = 1 +0.15.187.943 I slot release: id 0 | task 21 | stop processing: n_tokens = 20, truncated = 0 +0.15.187.960 I srv update_slots: all slots are idle +0.15.199.696 I srv server_strea: conv_id= (empty=1) +0.15.199.865 I srv operator(): chat format: peg-native +0.15.200.021 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.15.200.041 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.15.200.092 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.15.200.116 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.15.200.117 I slot launch_slot_: id 0 | task 25 | processing task, is_child = 0 +0.15.200.124 I slot operator(): id 0 | task 25 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.15.200.127 I slot operator(): id 0 | task 25 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.15.208.766 I slot operator(): id 0 | task 25 | cached n_tokens = 0, memory_seq_rm [0, end) +0.16.320.966 I slot operator(): id 0 | task 25 | cached n_tokens = 15, memory_seq_rm [15, end) +0.16.321.028 I slot init_sampler: id 0 | task 25 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.16.388.482 I slot create_check: id 0 | task 25 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.17.135.810 I slot print_timing: id 0 | task 25 | prompt eval time = 1610.34 ms / 19 tokens ( 84.75 ms per token, 11.80 tokens per second) +0.17.135.854 I slot print_timing: id 0 | task 25 | eval time = 325.32 ms / 2 tokens ( 325.32 ms per token, 3.07 tokens per second) +0.17.135.855 I slot print_timing: id 0 | task 25 | total time = 1935.65 ms / 21 tokens +0.17.135.857 I slot print_timing: id 0 | task 25 | graphs reused = 1 +0.17.135.914 I slot release: id 0 | task 25 | stop processing: n_tokens = 20, truncated = 0 +0.17.135.932 I srv update_slots: all slots are idle +0.17.151.940 I srv server_strea: conv_id= (empty=1) +0.17.152.122 I srv operator(): chat format: peg-native +0.17.152.283 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.17.152.284 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.17.152.339 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.17.152.343 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.17.152.344 I slot launch_slot_: id 0 | task 29 | processing task, is_child = 0 +0.17.152.350 I slot operator(): id 0 | task 29 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.17.152.353 I slot operator(): id 0 | task 29 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.17.161.864 I slot operator(): id 0 | task 29 | cached n_tokens = 0, memory_seq_rm [0, end) +0.18.278.193 I slot operator(): id 0 | task 29 | cached n_tokens = 15, memory_seq_rm [15, end) +0.18.278.268 I slot init_sampler: id 0 | task 29 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.18.347.342 I slot create_check: id 0 | task 29 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.19.075.350 I slot print_timing: id 0 | task 29 | prompt eval time = 1616.57 ms / 19 tokens ( 85.08 ms per token, 11.75 tokens per second) +0.19.075.389 I slot print_timing: id 0 | task 29 | eval time = 306.40 ms / 2 tokens ( 306.40 ms per token, 3.26 tokens per second) +0.19.075.390 I slot print_timing: id 0 | task 29 | total time = 1922.97 ms / 21 tokens +0.19.075.392 I slot print_timing: id 0 | task 29 | graphs reused = 1 +0.19.075.441 I slot release: id 0 | task 29 | stop processing: n_tokens = 20, truncated = 0 +0.19.075.460 I srv update_slots: all slots are idle +0.19.084.988 I srv server_strea: conv_id= (empty=1) +0.19.085.226 I srv operator(): chat format: peg-native +0.19.085.392 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.19.085.415 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.19.085.469 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.19.085.494 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.19.085.496 I slot launch_slot_: id 0 | task 33 | processing task, is_child = 0 +0.19.085.503 I slot operator(): id 0 | task 33 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.19.085.505 I slot operator(): id 0 | task 33 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.19.094.507 I slot operator(): id 0 | task 33 | cached n_tokens = 0, memory_seq_rm [0, end) +0.20.222.825 I slot operator(): id 0 | task 33 | cached n_tokens = 15, memory_seq_rm [15, end) +0.20.222.893 I slot init_sampler: id 0 | task 33 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.20.295.242 I slot create_check: id 0 | task 33 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.21.048.266 I slot print_timing: id 0 | task 33 | prompt eval time = 1646.26 ms / 19 tokens ( 86.65 ms per token, 11.54 tokens per second) +0.21.048.305 I slot print_timing: id 0 | task 33 | eval time = 316.48 ms / 2 tokens ( 316.48 ms per token, 3.16 tokens per second) +0.21.048.306 I slot print_timing: id 0 | task 33 | total time = 1962.73 ms / 21 tokens +0.21.048.308 I slot print_timing: id 0 | task 33 | graphs reused = 1 +0.21.048.359 I slot release: id 0 | task 33 | stop processing: n_tokens = 20, truncated = 0 +0.21.048.379 I srv update_slots: all slots are idle +0.21.055.374 I srv server_strea: conv_id= (empty=1) +0.21.055.650 I srv operator(): chat format: peg-native +0.21.055.874 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.21.055.913 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.21.056.045 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.21.056.082 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.21.056.084 I slot launch_slot_: id 0 | task 37 | processing task, is_child = 0 +0.21.056.097 I slot operator(): id 0 | task 37 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.21.056.101 I slot operator(): id 0 | task 37 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.21.066.705 I slot operator(): id 0 | task 37 | cached n_tokens = 0, memory_seq_rm [0, end) +0.22.167.774 I slot operator(): id 0 | task 37 | cached n_tokens = 15, memory_seq_rm [15, end) +0.22.167.847 I slot init_sampler: id 0 | task 37 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.22.276.706 I slot create_check: id 0 | task 37 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.22.993.668 I slot print_timing: id 0 | task 37 | prompt eval time = 1636.88 ms / 19 tokens ( 86.15 ms per token, 11.61 tokens per second) +0.22.993.711 I slot print_timing: id 0 | task 37 | eval time = 300.66 ms / 2 tokens ( 300.66 ms per token, 3.33 tokens per second) +0.22.993.712 I slot print_timing: id 0 | task 37 | total time = 1937.54 ms / 21 tokens +0.22.993.714 I slot print_timing: id 0 | task 37 | graphs reused = 1 +0.22.993.792 I slot release: id 0 | task 37 | stop processing: n_tokens = 20, truncated = 0 +0.22.993.815 I srv update_slots: all slots are idle +0.23.003.279 I srv server_strea: conv_id= (empty=1) +0.23.003.515 I srv operator(): chat format: peg-native +0.23.003.684 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.23.003.704 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.23.003.761 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.23.003.785 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.23.003.786 I slot launch_slot_: id 0 | task 41 | processing task, is_child = 0 +0.23.003.793 I slot operator(): id 0 | task 41 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.23.003.796 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.23.012.566 I slot operator(): id 0 | task 41 | cached n_tokens = 0, memory_seq_rm [0, end) +0.24.096.858 I slot operator(): id 0 | task 41 | cached n_tokens = 15, memory_seq_rm [15, end) +0.24.096.934 I slot init_sampler: id 0 | task 41 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.24.205.607 I slot create_check: id 0 | task 41 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.24.944.381 I slot print_timing: id 0 | task 41 | prompt eval time = 1621.99 ms / 19 tokens ( 85.37 ms per token, 11.71 tokens per second) +0.24.944.448 I slot print_timing: id 0 | task 41 | eval time = 318.56 ms / 2 tokens ( 318.56 ms per token, 3.14 tokens per second) +0.24.944.451 I slot print_timing: id 0 | task 41 | total time = 1940.55 ms / 21 tokens +0.24.944.454 I slot print_timing: id 0 | task 41 | graphs reused = 1 +0.24.944.525 I slot release: id 0 | task 41 | stop processing: n_tokens = 20, truncated = 0 +0.24.944.551 I srv update_slots: all slots are idle +0.24.950.513 I srv server_strea: conv_id= (empty=1) +0.24.950.801 I srv operator(): chat format: peg-native +0.24.951.045 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.24.951.079 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.24.951.150 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.24.951.156 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.24.951.157 I slot launch_slot_: id 0 | task 45 | processing task, is_child = 0 +0.24.951.166 I slot operator(): id 0 | task 45 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.24.951.169 I slot operator(): id 0 | task 45 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.24.960.325 I slot operator(): id 0 | task 45 | cached n_tokens = 0, memory_seq_rm [0, end) +0.26.097.799 I slot operator(): id 0 | task 45 | cached n_tokens = 15, memory_seq_rm [15, end) +0.26.097.879 I slot init_sampler: id 0 | task 45 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.26.160.053 I slot create_check: id 0 | task 45 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.26.898.389 I slot print_timing: id 0 | task 45 | prompt eval time = 1626.00 ms / 19 tokens ( 85.58 ms per token, 11.69 tokens per second) +0.26.898.426 I slot print_timing: id 0 | task 45 | eval time = 321.19 ms / 2 tokens ( 321.19 ms per token, 3.11 tokens per second) +0.26.898.426 I slot print_timing: id 0 | task 45 | total time = 1947.19 ms / 21 tokens +0.26.898.428 I slot print_timing: id 0 | task 45 | graphs reused = 1 +0.26.898.480 I slot release: id 0 | task 45 | stop processing: n_tokens = 20, truncated = 0 +0.26.898.501 I srv update_slots: all slots are idle +0.26.902.984 I srv server_strea: conv_id= (empty=1) +0.26.903.179 I srv operator(): chat format: peg-native +0.26.903.347 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.26.903.372 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.26.903.428 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.26.903.434 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.26.903.436 I slot launch_slot_: id 0 | task 49 | processing task, is_child = 0 +0.26.903.441 I slot operator(): id 0 | task 49 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.26.903.444 I slot operator(): id 0 | task 49 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.26.912.906 I slot operator(): id 0 | task 49 | cached n_tokens = 0, memory_seq_rm [0, end) +0.28.027.385 I slot operator(): id 0 | task 49 | cached n_tokens = 15, memory_seq_rm [15, end) +0.28.027.457 I slot init_sampler: id 0 | task 49 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.28.100.339 I slot create_check: id 0 | task 49 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.28.830.243 I slot print_timing: id 0 | task 49 | prompt eval time = 1615.48 ms / 19 tokens ( 85.03 ms per token, 11.76 tokens per second) +0.28.830.297 I slot print_timing: id 0 | task 49 | eval time = 311.29 ms / 2 tokens ( 311.29 ms per token, 3.21 tokens per second) +0.28.830.299 I slot print_timing: id 0 | task 49 | total time = 1926.77 ms / 21 tokens +0.28.830.301 I slot print_timing: id 0 | task 49 | graphs reused = 1 +0.28.830.367 I slot release: id 0 | task 49 | stop processing: n_tokens = 20, truncated = 0 +0.28.830.388 I srv update_slots: all slots are idle +0.28.842.589 I srv server_strea: conv_id= (empty=1) +0.28.842.759 I srv operator(): chat format: peg-native +0.28.842.913 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.28.842.934 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.28.842.983 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.28.843.007 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.28.843.012 I slot launch_slot_: id 0 | task 53 | processing task, is_child = 0 +0.28.843.019 I slot operator(): id 0 | task 53 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.28.843.022 I slot operator(): id 0 | task 53 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.28.851.758 I slot operator(): id 0 | task 53 | cached n_tokens = 0, memory_seq_rm [0, end) +0.29.957.377 I slot operator(): id 0 | task 53 | cached n_tokens = 15, memory_seq_rm [15, end) +0.29.957.454 I slot init_sampler: id 0 | task 53 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.30.015.358 I slot create_check: id 0 | task 53 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.30.746.022 I slot print_timing: id 0 | task 53 | prompt eval time = 1591.72 ms / 19 tokens ( 83.77 ms per token, 11.94 tokens per second) +0.30.746.076 I slot print_timing: id 0 | task 53 | eval time = 311.25 ms / 2 tokens ( 311.25 ms per token, 3.21 tokens per second) +0.30.746.077 I slot print_timing: id 0 | task 53 | total time = 1902.97 ms / 21 tokens +0.30.746.079 I slot print_timing: id 0 | task 53 | graphs reused = 1 +0.30.746.135 I slot release: id 0 | task 53 | stop processing: n_tokens = 20, truncated = 0 +0.30.746.156 I srv update_slots: all slots are idle +0.30.755.829 I srv server_strea: conv_id= (empty=1) +0.30.756.094 I srv operator(): chat format: peg-native +0.30.756.280 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.30.756.301 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.30.756.360 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.30.756.384 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.30.756.385 I slot launch_slot_: id 0 | task 57 | processing task, is_child = 0 +0.30.756.392 I slot operator(): id 0 | task 57 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.30.756.395 I slot operator(): id 0 | task 57 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.30.764.800 I slot operator(): id 0 | task 57 | cached n_tokens = 0, memory_seq_rm [0, end) +0.31.872.125 I slot operator(): id 0 | task 57 | cached n_tokens = 15, memory_seq_rm [15, end) +0.31.872.212 I slot init_sampler: id 0 | task 57 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.31.976.269 I slot create_check: id 0 | task 57 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.32.705.249 I slot print_timing: id 0 | task 57 | prompt eval time = 1643.60 ms / 19 tokens ( 86.51 ms per token, 11.56 tokens per second) +0.32.705.306 I slot print_timing: id 0 | task 57 | eval time = 305.22 ms / 2 tokens ( 305.22 ms per token, 3.28 tokens per second) +0.32.705.308 I slot print_timing: id 0 | task 57 | total time = 1948.82 ms / 21 tokens +0.32.705.310 I slot print_timing: id 0 | task 57 | graphs reused = 1 +0.32.705.379 I slot release: id 0 | task 57 | stop processing: n_tokens = 20, truncated = 0 +0.32.705.404 I srv update_slots: all slots are idle +0.32.715.281 I srv server_strea: conv_id= (empty=1) +0.32.715.529 I srv operator(): chat format: peg-native +0.32.715.687 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.32.715.708 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.32.715.766 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.32.715.791 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.32.715.792 I slot launch_slot_: id 0 | task 61 | processing task, is_child = 0 +0.32.715.799 I slot operator(): id 0 | task 61 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.32.715.802 I slot operator(): id 0 | task 61 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.32.726.811 I slot operator(): id 0 | task 61 | cached n_tokens = 0, memory_seq_rm [0, end) +0.33.822.635 I slot operator(): id 0 | task 61 | cached n_tokens = 15, memory_seq_rm [15, end) +0.33.822.712 I slot init_sampler: id 0 | task 61 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.33.881.730 I slot create_check: id 0 | task 61 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.34.613.155 I slot print_timing: id 0 | task 61 | prompt eval time = 1582.90 ms / 19 tokens ( 83.31 ms per token, 12.00 tokens per second) +0.34.613.191 I slot print_timing: id 0 | task 61 | eval time = 314.42 ms / 2 tokens ( 314.42 ms per token, 3.18 tokens per second) +0.34.613.192 I slot print_timing: id 0 | task 61 | total time = 1897.33 ms / 21 tokens +0.34.613.193 I slot print_timing: id 0 | task 61 | graphs reused = 1 +0.34.613.247 I slot release: id 0 | task 61 | stop processing: n_tokens = 20, truncated = 0 +0.34.613.265 I srv update_slots: all slots are idle +0.34.623.717 I srv server_strea: conv_id= (empty=1) +0.34.623.965 I srv operator(): chat format: peg-native +0.34.624.236 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.34.624.266 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.34.624.354 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.34.624.392 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.34.624.393 I slot launch_slot_: id 0 | task 65 | processing task, is_child = 0 +0.34.624.411 I slot operator(): id 0 | task 65 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.34.624.415 I slot operator(): id 0 | task 65 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.34.633.498 I slot operator(): id 0 | task 65 | cached n_tokens = 0, memory_seq_rm [0, end) +0.35.757.807 I slot operator(): id 0 | task 65 | cached n_tokens = 15, memory_seq_rm [15, end) +0.35.757.883 I slot init_sampler: id 0 | task 65 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.35.836.094 I slot create_check: id 0 | task 65 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.36.577.238 I slot print_timing: id 0 | task 65 | prompt eval time = 1640.30 ms / 19 tokens ( 86.33 ms per token, 11.58 tokens per second) +0.36.577.292 I slot print_timing: id 0 | task 65 | eval time = 312.49 ms / 2 tokens ( 312.49 ms per token, 3.20 tokens per second) +0.36.577.295 I slot print_timing: id 0 | task 65 | total time = 1952.80 ms / 21 tokens +0.36.577.297 I slot print_timing: id 0 | task 65 | graphs reused = 1 +0.36.577.369 I slot release: id 0 | task 65 | stop processing: n_tokens = 20, truncated = 0 +0.36.577.391 I srv update_slots: all slots are idle +0.36.585.251 I srv server_strea: conv_id= (empty=1) +0.36.585.500 I srv operator(): chat format: peg-native +0.36.585.708 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.36.585.732 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.36.585.821 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.36.585.848 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.36.585.849 I slot launch_slot_: id 0 | task 69 | processing task, is_child = 0 +0.36.585.856 I slot operator(): id 0 | task 69 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.36.585.859 I slot operator(): id 0 | task 69 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.36.594.621 I slot operator(): id 0 | task 69 | cached n_tokens = 0, memory_seq_rm [0, end) +0.37.711.474 I slot operator(): id 0 | task 69 | cached n_tokens = 15, memory_seq_rm [15, end) +0.37.711.542 I slot init_sampler: id 0 | task 69 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.37.772.090 I slot create_check: id 0 | task 69 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.38.500.223 I slot print_timing: id 0 | task 69 | prompt eval time = 1606.67 ms / 19 tokens ( 84.56 ms per token, 11.83 tokens per second) +0.38.500.271 I slot print_timing: id 0 | task 69 | eval time = 307.67 ms / 2 tokens ( 307.67 ms per token, 3.25 tokens per second) +0.38.500.273 I slot print_timing: id 0 | task 69 | total time = 1914.34 ms / 21 tokens +0.38.500.276 I slot print_timing: id 0 | task 69 | graphs reused = 1 +0.38.500.345 I slot release: id 0 | task 69 | stop processing: n_tokens = 20, truncated = 0 +0.38.500.368 I srv update_slots: all slots are idle +0.38.514.471 I srv server_strea: conv_id= (empty=1) +0.38.514.640 I srv operator(): chat format: peg-native +0.38.514.801 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.38.514.822 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.38.514.874 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.38.514.881 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.38.514.882 I slot launch_slot_: id 0 | task 73 | processing task, is_child = 0 +0.38.514.889 I slot operator(): id 0 | task 73 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.38.514.892 I slot operator(): id 0 | task 73 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.38.524.215 I slot operator(): id 0 | task 73 | cached n_tokens = 0, memory_seq_rm [0, end) +0.39.635.358 I slot operator(): id 0 | task 73 | cached n_tokens = 15, memory_seq_rm [15, end) +0.39.635.427 I slot init_sampler: id 0 | task 73 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.39.701.331 I slot create_check: id 0 | task 73 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.40.433.925 I slot print_timing: id 0 | task 73 | prompt eval time = 1610.30 ms / 19 tokens ( 84.75 ms per token, 11.80 tokens per second) +0.40.433.962 I slot print_timing: id 0 | task 73 | eval time = 308.71 ms / 2 tokens ( 308.71 ms per token, 3.24 tokens per second) +0.40.433.963 I slot print_timing: id 0 | task 73 | total time = 1919.01 ms / 21 tokens +0.40.433.965 I slot print_timing: id 0 | task 73 | graphs reused = 1 +0.40.434.021 I slot release: id 0 | task 73 | stop processing: n_tokens = 20, truncated = 0 +0.40.434.040 I srv update_slots: all slots are idle +0.40.445.467 I srv server_strea: conv_id= (empty=1) +0.40.445.642 I srv operator(): chat format: peg-native +0.40.445.831 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.40.445.853 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.40.445.903 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.40.445.964 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.40.445.968 I slot launch_slot_: id 0 | task 77 | processing task, is_child = 0 +0.40.445.975 I slot operator(): id 0 | task 77 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.40.445.979 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.40.455.093 I slot operator(): id 0 | task 77 | cached n_tokens = 0, memory_seq_rm [0, end) +0.41.558.525 I slot operator(): id 0 | task 77 | cached n_tokens = 15, memory_seq_rm [15, end) +0.41.558.606 I slot init_sampler: id 0 | task 77 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.41.617.999 I slot create_check: id 0 | task 77 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.42.344.673 I slot print_timing: id 0 | task 77 | prompt eval time = 1584.45 ms / 19 tokens ( 83.39 ms per token, 11.99 tokens per second) +0.42.344.705 I slot print_timing: id 0 | task 77 | eval time = 314.22 ms / 2 tokens ( 314.22 ms per token, 3.18 tokens per second) +0.42.344.707 I slot print_timing: id 0 | task 77 | total time = 1898.67 ms / 21 tokens +0.42.344.708 I slot print_timing: id 0 | task 77 | graphs reused = 1 +0.42.344.758 I slot release: id 0 | task 77 | stop processing: n_tokens = 20, truncated = 0 +0.42.344.776 I srv update_slots: all slots are idle +0.42.356.129 I srv server_strea: conv_id= (empty=1) +0.42.356.347 I srv operator(): chat format: peg-native +0.42.356.506 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.42.356.532 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.42.356.586 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.42.356.590 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.42.356.591 I slot launch_slot_: id 0 | task 81 | processing task, is_child = 0 +0.42.356.597 I slot operator(): id 0 | task 81 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.42.356.600 I slot operator(): id 0 | task 81 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.42.365.648 I slot operator(): id 0 | task 81 | cached n_tokens = 0, memory_seq_rm [0, end) +0.43.479.858 I slot operator(): id 0 | task 81 | cached n_tokens = 15, memory_seq_rm [15, end) +0.43.479.931 I slot init_sampler: id 0 | task 81 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.43.537.497 I slot create_check: id 0 | task 81 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.44.267.951 I slot print_timing: id 0 | task 81 | prompt eval time = 1597.53 ms / 19 tokens ( 84.08 ms per token, 11.89 tokens per second) +0.44.267.997 I slot print_timing: id 0 | task 81 | eval time = 313.80 ms / 2 tokens ( 313.80 ms per token, 3.19 tokens per second) +0.44.267.998 I slot print_timing: id 0 | task 81 | total time = 1911.32 ms / 21 tokens +0.44.268.000 I slot print_timing: id 0 | task 81 | graphs reused = 1 +0.44.268.059 I slot release: id 0 | task 81 | stop processing: n_tokens = 20, truncated = 0 +0.44.268.078 I srv update_slots: all slots are idle +0.44.276.546 I srv server_strea: conv_id= (empty=1) +0.44.276.813 I srv operator(): chat format: peg-native +0.44.277.041 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.44.277.064 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.44.277.165 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.44.277.190 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.44.277.191 I slot launch_slot_: id 0 | task 85 | processing task, is_child = 0 +0.44.277.201 I slot operator(): id 0 | task 85 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.44.277.204 I slot operator(): id 0 | task 85 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.44.286.743 I slot operator(): id 0 | task 85 | cached n_tokens = 0, memory_seq_rm [0, end) +0.45.420.234 I slot operator(): id 0 | task 85 | cached n_tokens = 15, memory_seq_rm [15, end) +0.45.420.308 I slot init_sampler: id 0 | task 85 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.45.478.490 I slot create_check: id 0 | task 85 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.46.251.077 I slot print_timing: id 0 | task 85 | prompt eval time = 1641.39 ms / 19 tokens ( 86.39 ms per token, 11.58 tokens per second) +0.46.251.126 I slot print_timing: id 0 | task 85 | eval time = 332.46 ms / 2 tokens ( 332.46 ms per token, 3.01 tokens per second) +0.46.251.127 I slot print_timing: id 0 | task 85 | total time = 1973.84 ms / 21 tokens +0.46.251.129 I slot print_timing: id 0 | task 85 | graphs reused = 1 +0.46.251.184 I slot release: id 0 | task 85 | stop processing: n_tokens = 20, truncated = 0 +0.46.251.204 I srv update_slots: all slots are idle +0.46.259.475 I srv server_strea: conv_id= (empty=1) +0.46.259.685 I srv operator(): chat format: peg-native +0.46.259.842 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.46.259.872 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.46.259.926 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.46.259.931 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.46.259.932 I slot launch_slot_: id 0 | task 89 | processing task, is_child = 0 +0.46.259.938 I slot operator(): id 0 | task 89 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.46.259.941 I slot operator(): id 0 | task 89 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.46.269.221 I slot operator(): id 0 | task 89 | cached n_tokens = 0, memory_seq_rm [0, end) +0.47.425.779 I slot operator(): id 0 | task 89 | cached n_tokens = 15, memory_seq_rm [15, end) +0.47.425.844 I slot init_sampler: id 0 | task 89 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.47.483.545 I slot create_check: id 0 | task 89 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.48.293.539 I slot print_timing: id 0 | task 89 | prompt eval time = 1717.56 ms / 19 tokens ( 90.40 ms per token, 11.06 tokens per second) +0.48.293.595 I slot print_timing: id 0 | task 89 | eval time = 316.01 ms / 2 tokens ( 316.01 ms per token, 3.16 tokens per second) +0.48.293.596 I slot print_timing: id 0 | task 89 | total time = 2033.57 ms / 21 tokens +0.48.293.599 I slot print_timing: id 0 | task 89 | graphs reused = 1 +0.48.293.661 I slot release: id 0 | task 89 | stop processing: n_tokens = 20, truncated = 0 +0.48.293.683 I srv update_slots: all slots are idle +0.48.304.303 I srv server_strea: conv_id= (empty=1) +0.48.304.757 I srv operator(): chat format: peg-native +0.48.304.926 I slot get_availabl: id 0 | task -1 | - checking sim = 0.081 (3/37) > 0.100 +0.48.304.947 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 493972439925 +0.48.305.002 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.48.305.007 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.48.305.009 I slot launch_slot_: id 0 | task 93 | processing task, is_child = 0 +0.48.305.015 I slot operator(): id 0 | task 93 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.48.305.018 I slot operator(): id 0 | task 93 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.48.314.723 I slot operator(): id 0 | task 93 | cached n_tokens = 0, memory_seq_rm [0, end) +0.49.689.598 I slot operator(): id 0 | task 93 | cached n_tokens = 18, memory_seq_rm [18, end) +0.49.689.675 I slot process_mtmd: id 0 | task 93 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.49.752.626 I decoding image batch 1/1, n_tokens_batch = 9 +0.49.752.743 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.49.752.774 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.50.513.989 I image decoded (batch 1/1) in 762 ms +0.50.514.306 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.50.514.364 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.51.125.691 I slot operator(): id 0 | task 93 | cached n_tokens = 33, memory_seq_rm [27, end) +0.51.125.824 I slot init_sampler: id 0 | task 93 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.51.200.980 I slot create_check: id 0 | task 93 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.51.973.348 I slot print_timing: id 0 | task 93 | prompt eval time = 3342.37 ms / 37 tokens ( 90.33 ms per token, 11.07 tokens per second) +0.51.973.389 I slot print_timing: id 0 | task 93 | eval time = 325.93 ms / 2 tokens ( 325.93 ms per token, 3.07 tokens per second) +0.51.973.390 I slot print_timing: id 0 | task 93 | total time = 3668.30 ms / 39 tokens +0.51.973.392 I slot print_timing: id 0 | task 93 | graphs reused = 1 +0.51.973.442 I slot release: id 0 | task 93 | stop processing: n_tokens = 38, truncated = 0 +0.51.973.465 I srv update_slots: all slots are idle +0.51.997.465 I srv server_strea: conv_id= (empty=1) +0.51.997.895 I srv operator(): chat format: peg-native +0.51.998.077 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +0.51.998.103 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +0.51.998.162 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.51.998.188 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.51.998.189 I slot launch_slot_: id 0 | task 98 | processing task, is_child = 0 +0.51.998.197 I slot operator(): id 0 | task 98 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.51.998.200 I slot operator(): id 0 | task 98 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.52.007.449 I slot operator(): id 0 | task 98 | cached n_tokens = 0, memory_seq_rm [0, end) +0.53.463.508 I slot operator(): id 0 | task 98 | cached n_tokens = 18, memory_seq_rm [18, end) +0.53.463.587 I slot process_mtmd: id 0 | task 98 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.53.528.274 I decoding image batch 1/1, n_tokens_batch = 9 +0.53.528.376 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.53.528.405 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.54.278.130 I image decoded (batch 1/1) in 750 ms +0.54.278.431 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.54.278.488 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.54.871.483 I slot operator(): id 0 | task 98 | cached n_tokens = 33, memory_seq_rm [27, end) +0.54.871.559 I slot init_sampler: id 0 | task 98 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.54.945.870 I slot create_check: id 0 | task 98 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.55.666.518 I slot print_timing: id 0 | task 98 | prompt eval time = 3366.65 ms / 37 tokens ( 90.99 ms per token, 10.99 tokens per second) +0.55.666.560 I slot print_timing: id 0 | task 98 | eval time = 301.64 ms / 2 tokens ( 301.64 ms per token, 3.32 tokens per second) +0.55.666.561 I slot print_timing: id 0 | task 98 | total time = 3668.29 ms / 39 tokens +0.55.666.562 I slot print_timing: id 0 | task 98 | graphs reused = 1 +0.55.666.621 I slot release: id 0 | task 98 | stop processing: n_tokens = 38, truncated = 0 +0.55.666.645 I srv update_slots: all slots are idle +0.55.671.208 I srv server_strea: conv_id= (empty=1) +0.55.671.990 I srv operator(): chat format: peg-native +0.55.672.278 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +0.55.672.314 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +0.55.672.387 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.55.672.393 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.55.672.396 I slot launch_slot_: id 0 | task 103 | processing task, is_child = 0 +0.55.672.407 I slot operator(): id 0 | task 103 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.55.672.410 I slot operator(): id 0 | task 103 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.55.682.345 I slot operator(): id 0 | task 103 | cached n_tokens = 0, memory_seq_rm [0, end) +0.57.034.827 I slot operator(): id 0 | task 103 | cached n_tokens = 18, memory_seq_rm [18, end) +0.57.034.942 I slot process_mtmd: id 0 | task 103 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.57.099.755 I decoding image batch 1/1, n_tokens_batch = 9 +0.57.099.858 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.57.099.886 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.57.869.264 I image decoded (batch 1/1) in 769 ms +0.57.869.676 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.57.869.788 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.58.459.113 I slot operator(): id 0 | task 103 | cached n_tokens = 33, memory_seq_rm [27, end) +0.58.459.185 I slot init_sampler: id 0 | task 103 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.58.525.629 I slot create_check: id 0 | task 103 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.59.274.193 I slot print_timing: id 0 | task 103 | prompt eval time = 3278.62 ms / 37 tokens ( 88.61 ms per token, 11.29 tokens per second) +0.59.274.243 I slot print_timing: id 0 | task 103 | eval time = 323.14 ms / 2 tokens ( 323.14 ms per token, 3.09 tokens per second) +0.59.274.244 I slot print_timing: id 0 | task 103 | total time = 3601.75 ms / 39 tokens +0.59.274.246 I slot print_timing: id 0 | task 103 | graphs reused = 1 +0.59.274.311 I slot release: id 0 | task 103 | stop processing: n_tokens = 38, truncated = 0 +0.59.274.346 I srv update_slots: all slots are idle +0.59.282.340 I srv server_strea: conv_id= (empty=1) +0.59.282.931 I srv operator(): chat format: peg-native +0.59.283.086 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +0.59.283.107 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +0.59.283.160 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.59.283.187 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.59.283.188 I slot launch_slot_: id 0 | task 108 | processing task, is_child = 0 +0.59.283.194 I slot operator(): id 0 | task 108 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.59.283.196 I slot operator(): id 0 | task 108 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.59.292.247 I slot operator(): id 0 | task 108 | cached n_tokens = 0, memory_seq_rm [0, end) +1.00.649.903 I slot operator(): id 0 | task 108 | cached n_tokens = 18, memory_seq_rm [18, end) +1.00.649.975 I slot process_mtmd: id 0 | task 108 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.00.727.852 I decoding image batch 1/1, n_tokens_batch = 9 +1.00.727.965 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.00.727.998 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.01.479.357 I image decoded (batch 1/1) in 751 ms +1.01.479.608 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.01.479.661 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.02.060.967 I slot operator(): id 0 | task 108 | cached n_tokens = 33, memory_seq_rm [27, end) +1.02.061.082 I slot init_sampler: id 0 | task 108 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.02.173.412 I slot create_check: id 0 | task 108 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.02.899.843 I slot print_timing: id 0 | task 108 | prompt eval time = 3311.90 ms / 37 tokens ( 89.51 ms per token, 11.17 tokens per second) +1.02.899.873 I slot print_timing: id 0 | task 108 | eval time = 304.72 ms / 2 tokens ( 304.72 ms per token, 3.28 tokens per second) +1.02.899.874 I slot print_timing: id 0 | task 108 | total time = 3616.62 ms / 39 tokens +1.02.899.875 I slot print_timing: id 0 | task 108 | graphs reused = 1 +1.02.899.922 I slot release: id 0 | task 108 | stop processing: n_tokens = 38, truncated = 0 +1.02.899.942 I srv update_slots: all slots are idle +1.02.913.425 I srv server_strea: conv_id= (empty=1) +1.02.913.860 I srv operator(): chat format: peg-native +1.02.914.011 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.02.914.034 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.02.914.087 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.02.914.111 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.02.914.112 I slot launch_slot_: id 0 | task 113 | processing task, is_child = 0 +1.02.914.119 I slot operator(): id 0 | task 113 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.02.914.121 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.02.922.855 I slot operator(): id 0 | task 113 | cached n_tokens = 0, memory_seq_rm [0, end) +1.04.258.140 I slot operator(): id 0 | task 113 | cached n_tokens = 18, memory_seq_rm [18, end) +1.04.258.215 I slot process_mtmd: id 0 | task 113 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.04.323.688 I decoding image batch 1/1, n_tokens_batch = 9 +1.04.323.813 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.04.323.845 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.05.088.205 I image decoded (batch 1/1) in 764 ms +1.05.088.519 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.05.088.582 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.05.674.615 I slot operator(): id 0 | task 113 | cached n_tokens = 33, memory_seq_rm [27, end) +1.05.674.686 I slot init_sampler: id 0 | task 113 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.05.735.757 I slot create_check: id 0 | task 113 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.06.464.374 I slot print_timing: id 0 | task 113 | prompt eval time = 3242.15 ms / 37 tokens ( 87.63 ms per token, 11.41 tokens per second) +1.06.464.421 I slot print_timing: id 0 | task 113 | eval time = 308.08 ms / 2 tokens ( 308.08 ms per token, 3.25 tokens per second) +1.06.464.422 I slot print_timing: id 0 | task 113 | total time = 3550.23 ms / 39 tokens +1.06.464.424 I slot print_timing: id 0 | task 113 | graphs reused = 1 +1.06.464.483 I slot release: id 0 | task 113 | stop processing: n_tokens = 38, truncated = 0 +1.06.464.509 I srv update_slots: all slots are idle +1.06.478.298 I srv server_strea: conv_id= (empty=1) +1.06.478.639 I srv operator(): chat format: peg-native +1.06.478.787 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.06.478.809 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.06.478.861 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.06.478.886 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.06.478.888 I slot launch_slot_: id 0 | task 118 | processing task, is_child = 0 +1.06.478.894 I slot operator(): id 0 | task 118 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.06.478.897 I slot operator(): id 0 | task 118 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.06.487.882 I slot operator(): id 0 | task 118 | cached n_tokens = 0, memory_seq_rm [0, end) +1.07.828.449 I slot operator(): id 0 | task 118 | cached n_tokens = 18, memory_seq_rm [18, end) +1.07.828.519 I slot process_mtmd: id 0 | task 118 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.07.901.448 I decoding image batch 1/1, n_tokens_batch = 9 +1.07.901.558 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.07.901.587 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.08.671.970 I image decoded (batch 1/1) in 771 ms +1.08.672.252 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.08.672.305 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.09.258.199 I slot operator(): id 0 | task 118 | cached n_tokens = 33, memory_seq_rm [27, end) +1.09.258.331 I slot init_sampler: id 0 | task 118 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.09.330.618 I slot create_check: id 0 | task 118 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.10.134.876 I slot print_timing: id 0 | task 118 | prompt eval time = 3322.39 ms / 37 tokens ( 89.79 ms per token, 11.14 tokens per second) +1.10.134.906 I slot print_timing: id 0 | task 118 | eval time = 333.56 ms / 2 tokens ( 333.56 ms per token, 3.00 tokens per second) +1.10.134.907 I slot print_timing: id 0 | task 118 | total time = 3655.95 ms / 39 tokens +1.10.134.908 I slot print_timing: id 0 | task 118 | graphs reused = 1 +1.10.134.955 I slot release: id 0 | task 118 | stop processing: n_tokens = 38, truncated = 0 +1.10.134.973 I srv update_slots: all slots are idle +1.10.143.626 I srv server_strea: conv_id= (empty=1) +1.10.144.127 I srv operator(): chat format: peg-native +1.10.144.273 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.10.144.294 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.10.144.343 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.10.144.347 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.10.144.349 I slot launch_slot_: id 0 | task 123 | processing task, is_child = 0 +1.10.144.355 I slot operator(): id 0 | task 123 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.10.144.357 I slot operator(): id 0 | task 123 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.10.153.319 I slot operator(): id 0 | task 123 | cached n_tokens = 0, memory_seq_rm [0, end) +1.11.479.327 I slot operator(): id 0 | task 123 | cached n_tokens = 18, memory_seq_rm [18, end) +1.11.479.417 I slot process_mtmd: id 0 | task 123 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.11.539.678 I decoding image batch 1/1, n_tokens_batch = 9 +1.11.539.993 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.11.540.075 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.12.295.264 I image decoded (batch 1/1) in 755 ms +1.12.295.562 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.12.295.617 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.12.905.275 I slot operator(): id 0 | task 123 | cached n_tokens = 33, memory_seq_rm [27, end) +1.12.905.349 I slot init_sampler: id 0 | task 123 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.12.973.291 I slot create_check: id 0 | task 123 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.13.724.831 I slot print_timing: id 0 | task 123 | prompt eval time = 3263.55 ms / 37 tokens ( 88.20 ms per token, 11.34 tokens per second) +1.13.724.866 I slot print_timing: id 0 | task 123 | eval time = 316.89 ms / 2 tokens ( 316.89 ms per token, 3.16 tokens per second) +1.13.724.867 I slot print_timing: id 0 | task 123 | total time = 3580.45 ms / 39 tokens +1.13.724.869 I slot print_timing: id 0 | task 123 | graphs reused = 1 +1.13.724.922 I slot release: id 0 | task 123 | stop processing: n_tokens = 38, truncated = 0 +1.13.724.946 I srv update_slots: all slots are idle +1.13.732.981 I srv server_strea: conv_id= (empty=1) +1.13.733.481 I srv operator(): chat format: peg-native +1.13.733.631 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.13.733.651 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.13.733.702 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.13.733.709 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.13.733.710 I slot launch_slot_: id 0 | task 128 | processing task, is_child = 0 +1.13.733.716 I slot operator(): id 0 | task 128 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.13.733.718 I slot operator(): id 0 | task 128 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.13.742.548 I slot operator(): id 0 | task 128 | cached n_tokens = 0, memory_seq_rm [0, end) +1.15.073.881 I slot operator(): id 0 | task 128 | cached n_tokens = 18, memory_seq_rm [18, end) +1.15.073.963 I slot process_mtmd: id 0 | task 128 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.15.135.199 I decoding image batch 1/1, n_tokens_batch = 9 +1.15.135.296 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.15.135.327 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.15.907.445 I image decoded (batch 1/1) in 772 ms +1.15.907.814 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.15.907.880 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.16.515.456 I slot operator(): id 0 | task 128 | cached n_tokens = 33, memory_seq_rm [27, end) +1.16.515.580 I slot init_sampler: id 0 | task 128 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.16.583.556 I slot create_check: id 0 | task 128 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.17.341.733 I slot print_timing: id 0 | task 128 | prompt eval time = 3283.17 ms / 37 tokens ( 88.73 ms per token, 11.27 tokens per second) +1.17.341.877 I slot print_timing: id 0 | task 128 | eval time = 324.82 ms / 2 tokens ( 324.82 ms per token, 3.08 tokens per second) +1.17.341.879 I slot print_timing: id 0 | task 128 | total time = 3607.98 ms / 39 tokens +1.17.341.882 I slot print_timing: id 0 | task 128 | graphs reused = 1 +1.17.341.955 I slot release: id 0 | task 128 | stop processing: n_tokens = 38, truncated = 0 +1.17.341.982 I srv update_slots: all slots are idle +1.17.352.310 I srv server_strea: conv_id= (empty=1) +1.17.352.874 I srv operator(): chat format: peg-native +1.17.353.025 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.17.353.046 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.17.353.101 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.17.353.106 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.17.353.107 I slot launch_slot_: id 0 | task 133 | processing task, is_child = 0 +1.17.353.114 I slot operator(): id 0 | task 133 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.17.353.117 I slot operator(): id 0 | task 133 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.17.363.798 I slot operator(): id 0 | task 133 | cached n_tokens = 0, memory_seq_rm [0, end) +1.18.721.272 I slot operator(): id 0 | task 133 | cached n_tokens = 18, memory_seq_rm [18, end) +1.18.721.356 I slot process_mtmd: id 0 | task 133 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.18.788.854 I decoding image batch 1/1, n_tokens_batch = 9 +1.18.788.968 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.18.789.001 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.19.532.223 I image decoded (batch 1/1) in 743 ms +1.19.532.490 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.19.532.544 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.20.136.101 I slot operator(): id 0 | task 133 | cached n_tokens = 33, memory_seq_rm [27, end) +1.20.136.176 I slot init_sampler: id 0 | task 133 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.20.248.124 I slot create_check: id 0 | task 133 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.21.010.975 I slot print_timing: id 0 | task 133 | prompt eval time = 3317.15 ms / 37 tokens ( 89.65 ms per token, 11.15 tokens per second) +1.21.011.022 I slot print_timing: id 0 | task 133 | eval time = 340.69 ms / 2 tokens ( 340.69 ms per token, 2.94 tokens per second) +1.21.011.023 I slot print_timing: id 0 | task 133 | total time = 3657.83 ms / 39 tokens +1.21.011.025 I slot print_timing: id 0 | task 133 | graphs reused = 1 +1.21.011.084 I slot release: id 0 | task 133 | stop processing: n_tokens = 38, truncated = 0 +1.21.011.108 I srv update_slots: all slots are idle +1.21.025.349 I srv server_strea: conv_id= (empty=1) +1.21.025.797 I srv operator(): chat format: peg-native +1.21.025.954 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.21.025.977 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.21.026.033 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.21.026.057 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.21.026.058 I slot launch_slot_: id 0 | task 138 | processing task, is_child = 0 +1.21.026.065 I slot operator(): id 0 | task 138 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.21.026.067 I slot operator(): id 0 | task 138 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.21.035.001 I slot operator(): id 0 | task 138 | cached n_tokens = 0, memory_seq_rm [0, end) +1.22.409.156 I slot operator(): id 0 | task 138 | cached n_tokens = 18, memory_seq_rm [18, end) +1.22.409.224 I slot process_mtmd: id 0 | task 138 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.22.476.620 I decoding image batch 1/1, n_tokens_batch = 9 +1.22.476.722 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.22.476.754 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.23.277.568 I image decoded (batch 1/1) in 801 ms +1.23.277.989 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.23.278.062 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.23.907.479 I slot operator(): id 0 | task 138 | cached n_tokens = 33, memory_seq_rm [27, end) +1.23.907.546 I slot init_sampler: id 0 | task 138 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.23.969.562 I slot create_check: id 0 | task 138 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.24.726.009 I slot print_timing: id 0 | task 138 | prompt eval time = 3376.79 ms / 37 tokens ( 91.26 ms per token, 10.96 tokens per second) +1.24.726.055 I slot print_timing: id 0 | task 138 | eval time = 323.12 ms / 2 tokens ( 323.12 ms per token, 3.09 tokens per second) +1.24.726.057 I slot print_timing: id 0 | task 138 | total time = 3699.91 ms / 39 tokens +1.24.726.059 I slot print_timing: id 0 | task 138 | graphs reused = 1 +1.24.726.130 I slot release: id 0 | task 138 | stop processing: n_tokens = 38, truncated = 0 +1.24.726.162 I srv update_slots: all slots are idle +1.24.740.259 I srv server_strea: conv_id= (empty=1) +1.24.740.725 I srv operator(): chat format: peg-native +1.24.740.924 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.24.740.949 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.24.741.004 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.24.741.011 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.24.741.013 I slot launch_slot_: id 0 | task 143 | processing task, is_child = 0 +1.24.741.020 I slot operator(): id 0 | task 143 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.24.741.023 I slot operator(): id 0 | task 143 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.24.749.954 I slot operator(): id 0 | task 143 | cached n_tokens = 0, memory_seq_rm [0, end) +1.26.136.102 I slot operator(): id 0 | task 143 | cached n_tokens = 18, memory_seq_rm [18, end) +1.26.136.180 I slot process_mtmd: id 0 | task 143 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.26.203.550 I decoding image batch 1/1, n_tokens_batch = 9 +1.26.203.675 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.26.203.714 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.27.038.576 I image decoded (batch 1/1) in 835 ms +1.27.038.874 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.27.038.932 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.27.635.979 I slot operator(): id 0 | task 143 | cached n_tokens = 33, memory_seq_rm [27, end) +1.27.636.063 I slot init_sampler: id 0 | task 143 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.27.752.243 I slot create_check: id 0 | task 143 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.28.477.843 I slot print_timing: id 0 | task 143 | prompt eval time = 3426.87 ms / 37 tokens ( 92.62 ms per token, 10.80 tokens per second) +1.28.477.894 I slot print_timing: id 0 | task 143 | eval time = 309.92 ms / 2 tokens ( 309.92 ms per token, 3.23 tokens per second) +1.28.477.896 I slot print_timing: id 0 | task 143 | total time = 3736.79 ms / 39 tokens +1.28.477.899 I slot print_timing: id 0 | task 143 | graphs reused = 1 +1.28.477.970 I slot release: id 0 | task 143 | stop processing: n_tokens = 38, truncated = 0 +1.28.478.002 I srv update_slots: all slots are idle +1.28.487.117 I srv server_strea: conv_id= (empty=1) +1.28.487.457 I srv operator(): chat format: peg-native +1.28.487.620 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.28.487.641 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.28.487.696 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.28.487.721 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.28.487.722 I slot launch_slot_: id 0 | task 148 | processing task, is_child = 0 +1.28.487.728 I slot operator(): id 0 | task 148 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.28.487.731 I slot operator(): id 0 | task 148 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.28.497.038 I slot operator(): id 0 | task 148 | cached n_tokens = 0, memory_seq_rm [0, end) +1.29.829.966 I slot operator(): id 0 | task 148 | cached n_tokens = 18, memory_seq_rm [18, end) +1.29.830.029 I slot process_mtmd: id 0 | task 148 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.29.893.047 I decoding image batch 1/1, n_tokens_batch = 9 +1.29.893.152 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.29.893.181 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.30.631.972 I image decoded (batch 1/1) in 739 ms +1.30.632.341 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.30.632.416 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.31.235.210 I slot operator(): id 0 | task 148 | cached n_tokens = 33, memory_seq_rm [27, end) +1.31.235.291 I slot init_sampler: id 0 | task 148 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.31.314.207 I slot create_check: id 0 | task 148 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.32.048.149 I slot print_timing: id 0 | task 148 | prompt eval time = 3252.40 ms / 37 tokens ( 87.90 ms per token, 11.38 tokens per second) +1.32.048.183 I slot print_timing: id 0 | task 148 | eval time = 307.99 ms / 2 tokens ( 307.99 ms per token, 3.25 tokens per second) +1.32.048.184 I slot print_timing: id 0 | task 148 | total time = 3560.39 ms / 39 tokens +1.32.048.186 I slot print_timing: id 0 | task 148 | graphs reused = 1 +1.32.048.239 I slot release: id 0 | task 148 | stop processing: n_tokens = 38, truncated = 0 +1.32.048.261 I srv update_slots: all slots are idle +1.32.053.540 I srv server_strea: conv_id= (empty=1) +1.32.054.044 I srv operator(): chat format: peg-native +1.32.054.251 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.32.054.276 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.32.054.369 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.32.054.400 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.32.054.401 I slot launch_slot_: id 0 | task 153 | processing task, is_child = 0 +1.32.054.408 I slot operator(): id 0 | task 153 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.32.054.410 I slot operator(): id 0 | task 153 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.32.063.493 I slot operator(): id 0 | task 153 | cached n_tokens = 0, memory_seq_rm [0, end) +1.33.416.304 I slot operator(): id 0 | task 153 | cached n_tokens = 18, memory_seq_rm [18, end) +1.33.416.379 I slot process_mtmd: id 0 | task 153 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.33.482.199 I decoding image batch 1/1, n_tokens_batch = 9 +1.33.482.338 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.33.482.369 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.34.251.004 I image decoded (batch 1/1) in 769 ms +1.34.251.396 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.34.251.456 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.34.855.793 I slot operator(): id 0 | task 153 | cached n_tokens = 33, memory_seq_rm [27, end) +1.34.855.882 I slot init_sampler: id 0 | task 153 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.34.933.110 I slot create_check: id 0 | task 153 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.35.680.570 I slot print_timing: id 0 | task 153 | prompt eval time = 3312.31 ms / 37 tokens ( 89.52 ms per token, 11.17 tokens per second) +1.35.680.605 I slot print_timing: id 0 | task 153 | eval time = 313.82 ms / 2 tokens ( 313.82 ms per token, 3.19 tokens per second) +1.35.680.606 I slot print_timing: id 0 | task 153 | total time = 3626.13 ms / 39 tokens +1.35.680.608 I slot print_timing: id 0 | task 153 | graphs reused = 1 +1.35.680.660 I slot release: id 0 | task 153 | stop processing: n_tokens = 38, truncated = 0 +1.35.680.682 I srv update_slots: all slots are idle +1.35.694.731 I srv server_strea: conv_id= (empty=1) +1.35.695.069 I srv operator(): chat format: peg-native +1.35.695.221 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.35.695.242 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.35.695.296 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.35.695.301 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.35.695.323 I slot launch_slot_: id 0 | task 158 | processing task, is_child = 0 +1.35.695.330 I slot operator(): id 0 | task 158 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.35.695.333 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.35.704.186 I slot operator(): id 0 | task 158 | cached n_tokens = 0, memory_seq_rm [0, end) +1.37.030.993 I slot operator(): id 0 | task 158 | cached n_tokens = 18, memory_seq_rm [18, end) +1.37.031.057 I slot process_mtmd: id 0 | task 158 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.37.094.882 I decoding image batch 1/1, n_tokens_batch = 9 +1.37.094.985 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.37.095.014 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.37.845.857 I image decoded (batch 1/1) in 751 ms +1.37.846.197 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.37.846.258 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.38.459.420 I slot operator(): id 0 | task 158 | cached n_tokens = 33, memory_seq_rm [27, end) +1.38.459.493 I slot init_sampler: id 0 | task 158 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.38.527.727 I slot create_check: id 0 | task 158 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.39.257.951 I slot print_timing: id 0 | task 158 | prompt eval time = 3251.55 ms / 37 tokens ( 87.88 ms per token, 11.38 tokens per second) +1.39.257.982 I slot print_timing: id 0 | task 158 | eval time = 311.05 ms / 2 tokens ( 311.05 ms per token, 3.21 tokens per second) +1.39.257.983 I slot print_timing: id 0 | task 158 | total time = 3562.59 ms / 39 tokens +1.39.257.984 I slot print_timing: id 0 | task 158 | graphs reused = 1 +1.39.258.038 I slot release: id 0 | task 158 | stop processing: n_tokens = 38, truncated = 0 +1.39.258.065 I srv update_slots: all slots are idle +1.39.264.001 I srv server_strea: conv_id= (empty=1) +1.39.264.805 I srv operator(): chat format: peg-native +1.39.265.004 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.39.265.030 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.39.265.093 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.39.265.099 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.39.265.100 I slot launch_slot_: id 0 | task 163 | processing task, is_child = 0 +1.39.265.107 I slot operator(): id 0 | task 163 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.39.265.109 I slot operator(): id 0 | task 163 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.39.274.184 I slot operator(): id 0 | task 163 | cached n_tokens = 0, memory_seq_rm [0, end) +1.40.613.468 I slot operator(): id 0 | task 163 | cached n_tokens = 18, memory_seq_rm [18, end) +1.40.613.540 I slot process_mtmd: id 0 | task 163 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.40.676.828 I decoding image batch 1/1, n_tokens_batch = 9 +1.40.676.933 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.40.676.963 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.41.423.163 I image decoded (batch 1/1) in 746 ms +1.41.423.525 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.41.423.588 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.42.011.943 I slot operator(): id 0 | task 163 | cached n_tokens = 33, memory_seq_rm [27, end) +1.42.012.022 I slot init_sampler: id 0 | task 163 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.42.085.864 I slot create_check: id 0 | task 163 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.42.819.844 I slot print_timing: id 0 | task 163 | prompt eval time = 3237.18 ms / 37 tokens ( 87.49 ms per token, 11.43 tokens per second) +1.42.819.890 I slot print_timing: id 0 | task 163 | eval time = 317.52 ms / 2 tokens ( 317.52 ms per token, 3.15 tokens per second) +1.42.819.891 I slot print_timing: id 0 | task 163 | total time = 3554.71 ms / 39 tokens +1.42.819.893 I slot print_timing: id 0 | task 163 | graphs reused = 1 +1.42.819.947 I slot release: id 0 | task 163 | stop processing: n_tokens = 38, truncated = 0 +1.42.819.969 I srv update_slots: all slots are idle +1.42.834.279 I srv server_strea: conv_id= (empty=1) +1.42.834.704 I srv operator(): chat format: peg-native +1.42.834.857 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.42.834.878 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.42.834.934 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.42.834.959 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.42.834.960 I slot launch_slot_: id 0 | task 168 | processing task, is_child = 0 +1.42.834.967 I slot operator(): id 0 | task 168 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.42.834.970 I slot operator(): id 0 | task 168 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.42.843.749 I slot operator(): id 0 | task 168 | cached n_tokens = 0, memory_seq_rm [0, end) +1.44.195.385 I slot operator(): id 0 | task 168 | cached n_tokens = 18, memory_seq_rm [18, end) +1.44.195.451 I slot process_mtmd: id 0 | task 168 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.44.262.799 I decoding image batch 1/1, n_tokens_batch = 9 +1.44.262.908 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.44.262.940 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.45.012.740 I image decoded (batch 1/1) in 749 ms +1.45.013.010 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.45.013.075 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.45.611.774 I slot operator(): id 0 | task 168 | cached n_tokens = 33, memory_seq_rm [27, end) +1.45.611.852 I slot init_sampler: id 0 | task 168 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.45.722.050 I slot create_check: id 0 | task 168 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.46.454.475 I slot print_timing: id 0 | task 168 | prompt eval time = 3311.63 ms / 37 tokens ( 89.50 ms per token, 11.17 tokens per second) +1.46.454.517 I slot print_timing: id 0 | task 168 | eval time = 307.84 ms / 2 tokens ( 307.84 ms per token, 3.25 tokens per second) +1.46.454.518 I slot print_timing: id 0 | task 168 | total time = 3619.48 ms / 39 tokens +1.46.454.519 I slot print_timing: id 0 | task 168 | graphs reused = 1 +1.46.454.579 I slot release: id 0 | task 168 | stop processing: n_tokens = 38, truncated = 0 +1.46.454.601 I srv update_slots: all slots are idle +1.46.463.431 I srv server_strea: conv_id= (empty=1) +1.46.463.873 I srv operator(): chat format: peg-native +1.46.464.027 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.46.464.049 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.46.464.105 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.46.464.109 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.46.464.111 I slot launch_slot_: id 0 | task 173 | processing task, is_child = 0 +1.46.464.117 I slot operator(): id 0 | task 173 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.46.464.120 I slot operator(): id 0 | task 173 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.46.475.443 I slot operator(): id 0 | task 173 | cached n_tokens = 0, memory_seq_rm [0, end) +1.47.934.169 I slot operator(): id 0 | task 173 | cached n_tokens = 18, memory_seq_rm [18, end) +1.47.934.275 I slot process_mtmd: id 0 | task 173 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.48.003.598 I decoding image batch 1/1, n_tokens_batch = 9 +1.48.003.702 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.48.003.731 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.48.785.145 I image decoded (batch 1/1) in 782 ms +1.48.785.461 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.48.785.522 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.49.382.974 I slot operator(): id 0 | task 173 | cached n_tokens = 33, memory_seq_rm [27, end) +1.49.383.053 I slot init_sampler: id 0 | task 173 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.49.492.775 I slot create_check: id 0 | task 173 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.50.247.339 I slot print_timing: id 0 | task 173 | prompt eval time = 3465.47 ms / 37 tokens ( 93.66 ms per token, 10.68 tokens per second) +1.50.247.374 I slot print_timing: id 0 | task 173 | eval time = 317.72 ms / 2 tokens ( 317.72 ms per token, 3.15 tokens per second) +1.50.247.375 I slot print_timing: id 0 | task 173 | total time = 3783.19 ms / 39 tokens +1.50.247.377 I slot print_timing: id 0 | task 173 | graphs reused = 1 +1.50.247.440 I slot release: id 0 | task 173 | stop processing: n_tokens = 38, truncated = 0 +1.50.247.464 I srv update_slots: all slots are idle +1.50.252.880 I srv server_strea: conv_id= (empty=1) +1.50.253.887 I srv operator(): chat format: peg-native +1.50.254.156 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.50.254.180 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.50.254.276 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.50.254.306 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.50.254.307 I slot launch_slot_: id 0 | task 178 | processing task, is_child = 0 +1.50.254.318 I slot operator(): id 0 | task 178 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.50.254.321 I slot operator(): id 0 | task 178 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.50.265.222 I slot operator(): id 0 | task 178 | cached n_tokens = 0, memory_seq_rm [0, end) +1.51.652.421 I slot operator(): id 0 | task 178 | cached n_tokens = 18, memory_seq_rm [18, end) +1.51.652.491 I slot process_mtmd: id 0 | task 178 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.51.720.283 I decoding image batch 1/1, n_tokens_batch = 9 +1.51.720.395 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.51.720.426 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.52.511.661 I image decoded (batch 1/1) in 791 ms +1.52.511.967 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.52.512.033 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.53.139.844 I slot operator(): id 0 | task 178 | cached n_tokens = 33, memory_seq_rm [27, end) +1.53.139.916 I slot init_sampler: id 0 | task 178 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.53.252.309 I slot create_check: id 0 | task 178 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.54.050.759 I slot print_timing: id 0 | task 178 | prompt eval time = 3465.59 ms / 37 tokens ( 93.66 ms per token, 10.68 tokens per second) +1.54.050.807 I slot print_timing: id 0 | task 178 | eval time = 330.82 ms / 2 tokens ( 330.82 ms per token, 3.02 tokens per second) +1.54.050.808 I slot print_timing: id 0 | task 178 | total time = 3796.41 ms / 39 tokens +1.54.050.810 I slot print_timing: id 0 | task 178 | graphs reused = 1 +1.54.050.869 I slot release: id 0 | task 178 | stop processing: n_tokens = 38, truncated = 0 +1.54.050.893 I srv update_slots: all slots are idle +1.54.061.822 I srv server_strea: conv_id= (empty=1) +1.54.062.176 I srv operator(): chat format: peg-native +1.54.062.333 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.54.062.355 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.54.062.411 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.54.062.415 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.54.062.417 I slot launch_slot_: id 0 | task 183 | processing task, is_child = 0 +1.54.062.424 I slot operator(): id 0 | task 183 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.54.062.426 I slot operator(): id 0 | task 183 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.54.071.499 I slot operator(): id 0 | task 183 | cached n_tokens = 0, memory_seq_rm [0, end) +1.55.546.075 I slot operator(): id 0 | task 183 | cached n_tokens = 18, memory_seq_rm [18, end) +1.55.546.141 I slot process_mtmd: id 0 | task 183 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.55.612.433 I decoding image batch 1/1, n_tokens_batch = 9 +1.55.612.545 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.55.612.576 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.56.372.809 I image decoded (batch 1/1) in 761 ms +1.56.373.156 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.56.373.215 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +1.56.973.858 I slot operator(): id 0 | task 183 | cached n_tokens = 33, memory_seq_rm [27, end) +1.56.973.943 I slot init_sampler: id 0 | task 183 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +1.57.054.977 I slot create_check: id 0 | task 183 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +1.57.780.392 I slot print_timing: id 0 | task 183 | prompt eval time = 3412.78 ms / 37 tokens ( 92.24 ms per token, 10.84 tokens per second) +1.57.780.438 I slot print_timing: id 0 | task 183 | eval time = 305.15 ms / 2 tokens ( 305.15 ms per token, 3.28 tokens per second) +1.57.780.439 I slot print_timing: id 0 | task 183 | total time = 3717.94 ms / 39 tokens +1.57.780.441 I slot print_timing: id 0 | task 183 | graphs reused = 1 +1.57.780.497 I slot release: id 0 | task 183 | stop processing: n_tokens = 38, truncated = 0 +1.57.780.521 I srv update_slots: all slots are idle +1.57.796.654 I srv server_strea: conv_id= (empty=1) +1.57.797.019 I srv operator(): chat format: peg-native +1.57.797.169 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +1.57.797.190 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +1.57.797.244 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.57.797.251 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.57.797.252 I slot launch_slot_: id 0 | task 188 | processing task, is_child = 0 +1.57.797.259 I slot operator(): id 0 | task 188 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +1.57.797.261 I slot operator(): id 0 | task 188 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +1.57.806.365 I slot operator(): id 0 | task 188 | cached n_tokens = 0, memory_seq_rm [0, end) +1.59.171.028 I slot operator(): id 0 | task 188 | cached n_tokens = 18, memory_seq_rm [18, end) +1.59.171.103 I slot process_mtmd: id 0 | task 188 | encoding mtmd batch from idx = 18, n_chunks = 1 +1.59.234.347 I decoding image batch 1/1, n_tokens_batch = 9 +1.59.234.454 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +1.59.234.486 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +2.00.000.182 I image decoded (batch 1/1) in 766 ms +2.00.000.522 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +2.00.000.580 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +2.00.600.222 I slot operator(): id 0 | task 188 | cached n_tokens = 33, memory_seq_rm [27, end) +2.00.600.291 I slot init_sampler: id 0 | task 188 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +2.00.684.712 I slot create_check: id 0 | task 188 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +2.01.445.174 I slot print_timing: id 0 | task 188 | prompt eval time = 3345.63 ms / 37 tokens ( 90.42 ms per token, 11.06 tokens per second) +2.01.445.207 I slot print_timing: id 0 | task 188 | eval time = 302.26 ms / 2 tokens ( 302.26 ms per token, 3.31 tokens per second) +2.01.445.208 I slot print_timing: id 0 | task 188 | total time = 3647.89 ms / 39 tokens +2.01.445.210 I slot print_timing: id 0 | task 188 | graphs reused = 1 +2.01.445.264 I slot release: id 0 | task 188 | stop processing: n_tokens = 38, truncated = 0 +2.01.445.287 I srv update_slots: all slots are idle +2.01.459.223 I srv server_strea: conv_id= (empty=1) +2.01.459.785 I srv operator(): chat format: peg-native +2.01.459.940 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +2.01.459.962 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +2.01.460.017 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +2.01.460.042 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +2.01.460.043 I slot launch_slot_: id 0 | task 193 | processing task, is_child = 0 +2.01.460.050 I slot operator(): id 0 | task 193 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +2.01.460.053 I slot operator(): id 0 | task 193 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +2.01.468.904 I slot operator(): id 0 | task 193 | cached n_tokens = 0, memory_seq_rm [0, end) +2.02.810.775 I slot operator(): id 0 | task 193 | cached n_tokens = 18, memory_seq_rm [18, end) +2.02.810.851 I slot process_mtmd: id 0 | task 193 | encoding mtmd batch from idx = 18, n_chunks = 1 +2.02.888.593 I decoding image batch 1/1, n_tokens_batch = 9 +2.02.888.702 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +2.02.888.736 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +2.03.668.555 I image decoded (batch 1/1) in 780 ms +2.03.668.895 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +2.03.668.956 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +2.04.317.239 I slot operator(): id 0 | task 193 | cached n_tokens = 33, memory_seq_rm [27, end) +2.04.317.341 I slot init_sampler: id 0 | task 193 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +2.04.390.177 I slot create_check: id 0 | task 193 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +2.05.120.585 I slot print_timing: id 0 | task 193 | prompt eval time = 3356.68 ms / 37 tokens ( 90.72 ms per token, 11.02 tokens per second) +2.05.120.624 I slot print_timing: id 0 | task 193 | eval time = 303.83 ms / 2 tokens ( 303.83 ms per token, 3.29 tokens per second) +2.05.120.625 I slot print_timing: id 0 | task 193 | total time = 3660.50 ms / 39 tokens +2.05.120.626 I slot print_timing: id 0 | task 193 | graphs reused = 1 +2.05.120.680 I slot release: id 0 | task 193 | stop processing: n_tokens = 38, truncated = 0 +2.05.120.702 I srv update_slots: all slots are idle +2.05.127.438 I srv operator(): operator(): cleaning up before exit... +2.05.129.077 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +2.05.129.099 I common_memory_breakdown_print: | - Host | 18496 = 18084 + 277 + 134 | diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.driver.log new file mode 100644 index 000000000..6a88c35a0 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.driver.log @@ -0,0 +1,19 @@ +INFO 09-24 18:04:39 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 18:04:39 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 18:04:39 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +{"status": "failed", "error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation"} +Traceback (most recent call last): + File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 151, in + asyncio.run(main()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 55, in main + raise MemoryError("available native host RAM is below the declared Qwen3.8 reservation") +MemoryError: available native host RAM is below the declared Qwen3.8 reservation diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.json new file mode 100644 index 000000000..213049660 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke.json @@ -0,0 +1,11 @@ +{ + "scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp", + "device": "cpu", + "available_host_ram_before_bytes": 29981622272, + "reserved_host_ram_bytes": 30064771072, + "repeats": 1, + "model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf", + "status": "failed", + "error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.driver.log new file mode 100644 index 000000000..dc9179e6d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.driver.log @@ -0,0 +1,19 @@ +INFO 09-24 18:05:34 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 18:05:34 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 18:05:34 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +{"status": "failed", "error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation"} +Traceback (most recent call last): + File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 151, in + asyncio.run(main()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 55, in main + raise MemoryError("available native host RAM is below the declared Qwen3.8 reservation") +MemoryError: available native host RAM is below the declared Qwen3.8 reservation diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.json new file mode 100644 index 000000000..9b1236235 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke2.json @@ -0,0 +1,11 @@ +{ + "scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp", + "device": "cpu", + "available_host_ram_before_bytes": 29934391296, + "reserved_host_ram_bytes": 30064771072, + "repeats": 1, + "model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf", + "status": "failed", + "error": "MemoryError: available native host RAM is below the declared Qwen3.8 reservation" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.driver.log new file mode 100644 index 000000000..99249b553 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.driver.log @@ -0,0 +1,62 @@ +INFO 09-24 18:06:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 18:06:08 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 18:06:08 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-24 18:06:09 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 18:06:09 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge' +ERROR 09-24 18:06:53 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s) +ERROR 09-24 18:06:53 [stage_runtime.py:265] Traceback (most recent call last): +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-24 18:06:53 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-24 18:06:53 [stage_runtime.py:265] raise primary_exc +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-24 18:06:53 [stage_runtime.py:265] client = self._initialize_replica( +ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 817, in _initialize_replica +ERROR 09-24 18:06:53 [stage_runtime.py:265] return create_graph_client( +ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^ +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/__init__.py", line 19, in create_graph_client +ERROR 09-24 18:06:53 [stage_runtime.py:265] return LlamaCppMultimodalStageClient(metadata, backend, ledger, reservation) +ERROR 09-24 18:06:53 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 466, in __init__ +ERROR 09-24 18:06:53 [stage_runtime.py:265] super().__init__(metadata, config, ledger, reservation) +ERROR 09-24 18:06:53 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 224, in __init__ +ERROR 09-24 18:06:53 [stage_runtime.py:265] raise ResourceUnavailable("loaded llama.cpp process RSS exceeds stage reservation") +ERROR 09-24 18:06:53 [stage_runtime.py:265] vllm_omni.engine.resource_ledger.ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation +{"status": "failed", "error": "ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation"} +Traceback (most recent call last): + File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 151, in + asyncio.run(main()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/experiments/probe_omni_llamacpp_qwen38.py", line 86, in main + runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 817, in _initialize_replica + return create_graph_client( + ^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/__init__.py", line 19, in create_graph_client + return LlamaCppMultimodalStageClient(metadata, backend, ledger, reservation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 466, in __init__ + super().__init__(metadata, config, ledger, reservation) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/backends/llamacpp.py", line 224, in __init__ + raise ResourceUnavailable("loaded llama.cpp process RSS exceeds stage reservation") +vllm_omni.engine.resource_ledger.ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.json new file mode 100644 index 000000000..1a17712e4 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.json @@ -0,0 +1,21 @@ +{ + "scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp", + "device": "cpu", + "available_host_ram_before_bytes": 29994790912, + "reserved_host_ram_bytes": 28991029248, + "repeats": 1, + "model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf", + "status": "failed", + "error": "ResourceUnavailable: loaded llama.cpp process RSS exceeds stage reservation", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 29994790912 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.server.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.server.log new file mode 100644 index 000000000..f41d2d661 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke3.server.log @@ -0,0 +1,267 @@ +warning: no usable GPU found, --gpu-layers option will be ignored +warning: one possible reason is that llama.cpp was compiled without GPU support +warning: consult docs/build.md for compilation instructions +0.00.004.616 I cmn common_param: common_params_print_info: build 10849 (e71b80510) with GNU 11.4.0 for Linux x86_64 +0.00.004.645 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.004.645 I cmn common_param: device_info: +0.00.004.697 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (31647 MiB, 31647 MiB free) +0.00.004.773 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.004.907 I srv init: using 23 threads for HTTP server +0.00.004.927 I srv init: The UI is disabled +0.00.004.927 I srv init: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.004.987 W srv llama_server: ----------------- +0.00.005.006 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set +0.00.005.006 W srv llama_server: this can be a security risk (cross-origin attacks) +0.00.005.006 W srv llama_server: more info: https://github.com/ggml-org/llama.cpp/pull/25655 +0.00.005.006 W srv llama_server: ----------------- +0.00.005.028 I srv start: binding port with default address family +0.00.007.077 I srv load_model: loading model '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf' +0.00.007.097 I srv load_model: local path '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf' +0.00.070.920 I llama_model_loader: loaded meta data with 40 key-value pairs and 851 tensors from /home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.071.014 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.071.027 I llama_model_loader: - kv 0: general.architecture str = qwen35 +0.00.071.028 I llama_model_loader: - kv 1: general.type str = model +0.00.071.034 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +0.00.071.041 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.071.042 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.071.042 I llama_model_loader: - kv 5: general.name str = Qwen3.8-27B +0.00.071.043 I llama_model_loader: - kv 6: general.size_label str = 27B +0.00.071.043 I llama_model_loader: - kv 7: general.license str = apache-2.0 +0.00.071.068 I llama_model_loader: - kv 8: general.tags arr[str,1] = ["image-text-to-text"] +0.00.071.069 I llama_model_loader: - kv 9: qwen35.block_count u32 = 64 +0.00.071.070 I llama_model_loader: - kv 10: qwen35.context_length u32 = 262144 +0.00.071.070 I llama_model_loader: - kv 11: qwen35.embedding_length u32 = 5120 +0.00.071.071 I llama_model_loader: - kv 12: qwen35.feed_forward_length u32 = 17408 +0.00.071.071 I llama_model_loader: - kv 13: qwen35.attention.head_count u32 = 24 +0.00.071.071 I llama_model_loader: - kv 14: qwen35.attention.head_count_kv u32 = 4 +0.00.071.074 I llama_model_loader: - kv 15: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +0.00.071.076 I llama_model_loader: - kv 16: qwen35.rope.freq_base f32 = 10000000.000000 +0.00.071.077 I llama_model_loader: - kv 17: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.071.078 I llama_model_loader: - kv 18: qwen35.attention.key_length u32 = 256 +0.00.071.078 I llama_model_loader: - kv 19: qwen35.attention.value_length u32 = 256 +0.00.071.079 I llama_model_loader: - kv 20: qwen35.ssm.conv_kernel u32 = 4 +0.00.071.079 I llama_model_loader: - kv 21: qwen35.ssm.state_size u32 = 128 +0.00.071.079 I llama_model_loader: - kv 22: qwen35.ssm.group_count u32 = 16 +0.00.071.079 I llama_model_loader: - kv 23: qwen35.ssm.time_step_rank u32 = 48 +0.00.071.080 I llama_model_loader: - kv 24: qwen35.ssm.inner_size u32 = 6144 +0.00.071.084 I llama_model_loader: - kv 25: qwen35.attention.recurrent_layers arr[bool,64] = [true, true, true, false, true, true,... +0.00.071.084 I llama_model_loader: - kv 26: qwen35.full_attention_interval u32 = 4 +0.00.071.084 I llama_model_loader: - kv 27: qwen35.rope.dimension_count u32 = 64 +0.00.071.085 I llama_model_loader: - kv 28: tokenizer.ggml.model str = gpt2 +0.00.071.085 I llama_model_loader: - kv 29: tokenizer.ggml.pre str = qwen35 +0.00.089.105 I llama_model_loader: - kv 30: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +0.00.092.472 I llama_model_loader: - kv 31: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +0.00.108.554 I llama_model_loader: - kv 32: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +0.00.108.593 I llama_model_loader: - kv 33: tokenizer.ggml.eos_token_id u32 = 248046 +0.00.108.594 I llama_model_loader: - kv 34: tokenizer.ggml.padding_token_id u32 = 248044 +0.00.108.594 I llama_model_loader: - kv 35: tokenizer.ggml.bos_token_id u32 = 248044 +0.00.108.595 I llama_model_loader: - kv 36: tokenizer.ggml.add_bos_token bool = false +0.00.108.599 I llama_model_loader: - kv 37: tokenizer.chat_template str = {%- set image_count = namespace(value... +0.00.108.599 I llama_model_loader: - kv 38: general.quantization_version u32 = 2 +0.00.108.600 I llama_model_loader: - kv 39: general.file_type u32 = 15 +0.00.108.608 I llama_model_loader: - type f32: 353 tensors +0.00.108.609 I llama_model_loader: - type q8_0: 288 tensors +0.00.108.609 I llama_model_loader: - type q4_K: 193 tensors +0.00.108.609 I llama_model_loader: - type q6_K: 17 tensors +0.00.108.613 I print_info: file format = GGUF V3 (latest) +0.00.108.616 I print_info: file type = Q4_K - Medium +0.00.108.621 I print_info: file size = 17.66 GiB (5.64 BPW) +0.00.339.323 I load: 0 unused tokens +0.00.404.119 I load: printing all EOG tokens: +0.00.404.155 I load: - 248044 ('<|endoftext|>') +0.00.404.157 I load: - 248046 ('<|im_end|>') +0.00.404.158 I load: - 248063 ('<|fim_pad|>') +0.00.404.158 I load: - 248064 ('<|repo_name|>') +0.00.404.158 I load: - 248065 ('<|file_sep|>') +0.00.404.429 I load: special tokens cache size = 33 +0.00.461.091 I load: token to piece cache size = 1.7581 MB +0.00.461.146 I print_info: arch = qwen35 +0.00.461.147 I print_info: vocab_only = 0 +0.00.461.147 I print_info: no_alloc = 0 +0.00.461.148 I print_info: n_ctx_train = 262144 +0.00.461.149 I print_info: n_embd_inp = 5120 +0.00.461.150 I print_info: n_embd = 5120 +0.00.461.151 I print_info: n_embd_out = 5120 +0.00.461.151 I print_info: n_layer = 64 +0.00.461.151 I print_info: n_layer_all = 64 +0.00.461.169 I print_info: n_head = 24 +0.00.461.171 I print_info: n_head_kv = 4 +0.00.461.171 I print_info: n_rot = 64 +0.00.461.171 I print_info: n_swa = 0 +0.00.461.173 I print_info: is_swa_any = 0 +0.00.461.173 I print_info: non_causal_type = 0 +0.00.461.173 I print_info: n_embd_head_k = 256 +0.00.461.173 I print_info: n_embd_head_v = 256 +0.00.461.175 I print_info: n_gqa = 6 +0.00.461.181 I print_info: n_embd_k_gqa = 1024 +0.00.461.184 I print_info: n_embd_v_gqa = 1024 +0.00.461.185 I print_info: f_norm_eps = 0.0e+00 +0.00.461.187 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.461.187 I print_info: f_clamp_kqv = 0.0e+00 +0.00.461.187 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.461.187 I print_info: f_logit_scale = 0.0e+00 +0.00.461.188 I print_info: f_attn_scale = 0.0e+00 +0.00.461.188 I print_info: f_attn_value_scale = 0.0000 +0.00.461.190 I print_info: n_ff = 17408 +0.00.461.190 I print_info: n_expert = 0 +0.00.461.190 I print_info: n_expert_used = 0 +0.00.461.190 I print_info: n_expert_groups = 0 +0.00.461.191 I print_info: n_group_used = 0 +0.00.461.191 I print_info: causal attn = 1 +0.00.461.191 I print_info: pooling type = -1 +0.00.461.191 I print_info: rope type = 40 +0.00.461.192 I print_info: rope scaling = linear +0.00.461.193 I print_info: freq_base_train = 10000000.0 +0.00.461.193 I print_info: freq_scale_train = 1 +0.00.461.193 I print_info: n_ctx_orig_yarn = 262144 +0.00.461.193 I print_info: rope_yarn_log_mul = 0.0000 +0.00.461.193 I print_info: rope_finetuned = unknown +0.00.461.194 I print_info: mrope sections = [11, 11, 10, 0] +0.00.461.194 I print_info: ssm_d_conv = 4 +0.00.461.194 I print_info: ssm_d_inner = 6144 +0.00.461.194 I print_info: ssm_d_state = 128 +0.00.461.194 I print_info: ssm_dt_rank = 48 +0.00.461.194 I print_info: ssm_n_group = 16 +0.00.461.194 I print_info: ssm_dt_b_c_rms = 0 +0.00.461.195 I print_info: model type = 27B +0.00.461.196 I print_info: model params = 26.90 B +0.00.461.196 I print_info: general.name = Qwen3.8-27B +0.00.461.201 I print_info: vocab type = BPE +0.00.461.202 I print_info: n_vocab = 248320 +0.00.461.202 I print_info: n_merges = 247587 +0.00.461.203 I print_info: BOS token = 248044 '<|endoftext|>' +0.00.461.203 I print_info: EOS token = 248046 '<|im_end|>' +0.00.461.203 I print_info: EOT token = 248046 '<|im_end|>' +0.00.461.203 I print_info: PAD token = 248044 '<|endoftext|>' +0.00.461.204 I print_info: LF token = 198 'Ċ' +0.00.461.204 I print_info: FIM PRE token = 248060 '<|fim_prefix|>' +0.00.461.204 I print_info: FIM SUF token = 248062 '<|fim_suffix|>' +0.00.461.204 I print_info: FIM MID token = 248061 '<|fim_middle|>' +0.00.461.204 I print_info: FIM PAD token = 248063 '<|fim_pad|>' +0.00.461.204 I print_info: FIM REP token = 248064 '<|repo_name|>' +0.00.461.204 I print_info: FIM SEP token = 248065 '<|file_sep|>' +0.00.461.205 I print_info: EOG token = 248044 '<|endoftext|>' +0.00.461.206 I print_info: EOG token = 248046 '<|im_end|>' +0.00.461.217 I print_info: EOG token = 248063 '<|fim_pad|>' +0.00.461.218 I print_info: EOG token = 248064 '<|repo_name|>' +0.00.461.218 I print_info: EOG token = 248065 '<|file_sep|>' +0.00.461.218 I print_info: max token length = 256 +0.00.461.222 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap) +AMX is not ready to be used! +0.00.479.707 I load_tensors: CPU_Mapped model buffer size = 18084.41 MiB +0.00.479.737 I load_tensors: CPU_REPACK model buffer size = 9180.00 MiB +0.14.394.812 I cmn common_init_: added <|endoftext|> logit bias = -inf +0.14.394.847 I cmn common_init_: added <|im_end|> logit bias = -inf +0.14.394.848 I cmn common_init_: added <|fim_pad|> logit bias = -inf +0.14.394.848 I cmn common_init_: added <|repo_name|> logit bias = -inf +0.14.394.849 I cmn common_init_: added <|file_sep|> logit bias = -inf +0.14.394.957 I llama_context: constructing llama_context +0.14.394.962 I llama_context: n_seq_max = 1 +0.14.394.962 I llama_context: n_ctx = 2048 +0.14.394.962 I llama_context: n_ctx_seq = 2048 +0.14.394.962 I llama_context: n_batch = 2048 +0.14.394.963 I llama_context: n_ubatch = 512 +0.14.394.963 I llama_context: causal_attn = 1 +0.14.394.965 I llama_context: flash_attn = auto +0.14.394.965 I llama_context: kv_unified = false +0.14.394.968 I llama_context: freq_base = 10000000.0 +0.14.394.969 I llama_context: freq_scale = 1 +0.14.394.969 I llama_context: n_rs_seq = 0 +0.14.394.969 I llama_context: n_outputs_max = 1 +0.14.394.969 I llama_context: n_outputs_max_per_seq = 1 +0.14.394.970 I llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.14.395.033 I llama_context: CPU output buffer size = 0.95 MiB +0.14.395.178 I llama_kv_cache: CPU KV buffer size = 128.00 MiB +0.14.465.763 I llama_kv_cache: size = 128.00 MiB ( 2048 cells, 16 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB +0.14.465.794 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.14.465.794 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.14.522.121 I llama_memory_recurrent: CPU RS buffer size = 149.62 MiB +0.14.522.174 I llama_memory_recurrent: size = 149.62 MiB ( 1 cells, 64 layers, 1 seqs 0 rs_seq), R (f32): 5.62 MiB, S (f32): 144.00 MiB, P (f32): 0.00 MiB +0.14.522.198 I sched_reserve: reserving ... +0.14.530.268 I resolve_fused_ops: Flash Attention enabled +0.14.530.294 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.14.530.800 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.14.531.293 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.14.531.806 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.14.534.131 I sched_reserve: CPU compute buffer size = 134.02 MiB +0.14.534.153 I sched_reserve: graph nodes = 3751 +0.14.534.154 I sched_reserve: graph splits = 1 +0.14.534.156 I sched_reserve: reserve took 11.95 ms, sched copies = 1 +0.14.534.221 I cmn init: llama threadpool init, n_threads = 12 +0.14.534.272 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.15.294.041 I clip_model_loader: model name: Qwen3.8-27B +0.15.294.069 I clip_model_loader: description: +0.15.294.073 I clip_model_loader: GGUF version: 3 +0.15.294.075 I clip_model_loader: alignment: 32 +0.15.294.076 I clip_model_loader: n_tensors: 334 +0.15.294.077 I clip_model_loader: n_kv: 26 +0.15.294.078 I +0.15.294.082 I clip_model_loader: has vision encoder +0.15.294.369 I clip_ctx: CLIP using CPU backend +0.15.294.628 W load_hparams: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks +0.15.294.650 W load_hparams: if you encounter problems with accuracy, try adding --image-min-tokens 1024 +0.15.294.650 W load_hparams: more info: https://github.com/ggml-org/llama.cpp/issues/16842 + +0.15.294.651 I load_hparams: projector: qwen3vl_merger +0.15.294.651 I load_hparams: n_embd: 1152 +0.15.294.651 I load_hparams: n_head: 16 +0.15.294.651 I load_hparams: n_ff: 4304 +0.15.294.651 I load_hparams: n_layer: 27 +0.15.294.652 I load_hparams: ffn_op: gelu +0.15.294.652 I load_hparams: projection_dim: 5120 +0.15.294.652 I +--- vision hparams --- +0.15.294.652 I load_hparams: image_size: 768 +0.15.294.653 I load_hparams: patch_size: 16 +0.15.294.653 I load_hparams: has_llava_proj: 0 +0.15.294.653 I load_hparams: minicpmv_version: 0 +0.15.294.653 I load_hparams: n_merge: 2 +0.15.294.653 I load_hparams: n_wa_pattern: 0 +0.15.294.654 I load_hparams: image_min_pixels: 8192 +0.15.294.654 I load_hparams: image_max_pixels: 4194304 +0.15.294.654 I +0.15.294.659 I load_hparams: model size: 600.08 MiB +0.15.294.661 I load_hparams: metadata size: 0.12 MiB +0.16.644.354 I get_dummy_batch: warmup with image size = 1472 x 1472 +0.16.662.404 I get_dummy_batch: warmup with image size = 1472 x 1472 +0.16.672.021 I reserve_compute_meta: CPU compute buffer size = 248.10 MiB +0.16.672.058 I reserve_compute_meta: graph splits = 1, nodes = 823 +0.16.672.081 I warmup: flash attention is enabled +0.16.675.000 I srv load_model: loaded multimodal model, '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf' +0.19.587.564 I cmn common_conte: the context does not support partial sequence removal +0.19.598.280 I srv load_model: speculative decoding will use checkpoints +0.19.598.317 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 2048, kv_unified = 'false' +0.19.598.350 I spec common_specu: no implementations specified for speculative decoding +0.19.598.354 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048 +0.19.598.453 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.19.598.472 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.19.598.473 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.19.598.500 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.19.606.193 I srv init: init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + + + + +' +0.19.606.732 I srv init: init: chat template, thinking = 0 +0.19.606.772 I srv init: preserve_reasoning kwarg: true +0.19.606.772 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.19.606.829 I srv llama_server: model loaded +0.19.606.853 I srv llama_server: listening on http://127.0.0.1:33725 +0.19.606.995 I srv update_slots: all slots are idle +0.19.770.089 I srv operator(): operator(): cleaning up before exit... +0.19.771.302 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +0.19.771.323 I common_memory_breakdown_print: | - Host | 18496 = 18084 + 277 + 134 | +0.19.771.323 I common_memory_breakdown_print: | - CPU_REPACK | 9180 = 9180 + 0 + 0 | diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.driver.log new file mode 100644 index 000000000..360fb9253 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.driver.log @@ -0,0 +1,6 @@ +INFO 09-24 18:07:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-24 18:07:56 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-24 18:07:56 [patch.py:589] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-24 18:07:57 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-24 18:07:57 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 21] Is a directory: '/home/zhout/project/edge_infer/vllm-omni-edge' +{"status": "completed", "error": null} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.json new file mode 100644 index 000000000..60d80782d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.json @@ -0,0 +1,236 @@ +{ + "scope": "one pinned Qwen3.8 27B Q4_K_M GGUF + Q8 vision projector through Omni whole-session llama.cpp", + "device": "cpu", + "available_host_ram_before_bytes": 29979381760, + "reserved_host_ram_bytes": 25769803776, + "repeats": 1, + "model_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "mmproj_file": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf", + "startup_s": 25.010701490973588, + "execution_plan": { + "backend": "external.llamacpp.multimodal.v1", + "stage_id": 0, + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "model_sha256": "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747", + "mmproj_sha256": "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb", + "server_sha256": "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c", + "model_alias": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf", + "requested_device": "cpu", + "expected_device_name": null, + "layer_assignments": 0, + "offloaded_layers": null, + "model_buffer_devices": [], + "all_model_buffers": [ + "CPU_Mapped" + ], + "vision_backends": [ + "CPU" + ], + "cpu_only_layout": true, + "worker_pid": 346310, + "loaded_rss_bytes": 20026560512, + "reserved_bytes": { + "host_ram": 25769803776 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 4294967296, + "disable_repack": true, + "context_tokens": 2048, + "max_new_tokens": 128, + "max_io_bytes": 1048576, + "max_image_bytes": 262144, + "image_token_reserve": 512, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "kind": "text", + "wall_s": 1.958196632040199, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493858057328167", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.946947110001929, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "kind": "image", + "wall_s": 3.556307617982384, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493860015612375", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.5440641850000247, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "text_warmup": { + "kind": "text", + "wall_s": 1.9051429200335406, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493863571993703", + "stage_id": 0, + "epoch": 3, + "seq": 1, + "kind": "text", + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.8953040880151093, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + "text_measured": [ + { + "kind": "text", + "wall_s": 1.9479502299800515, + "answer": "ready", + "expected": "ready", + "stage_event": { + "request_id": "text-493865477197598", + "stage_id": 0, + "epoch": 4, + "seq": 1, + "kind": "text", + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 1.939045864972286, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 19, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_text_wall_s": 1.9479502299800515, + "nearest_rank_p95_text_wall_s": 1.9479502299800515, + "image_warmup": { + "kind": "image", + "wall_s": 3.6417471549939364, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493867425252186", + "stage_id": 0, + "epoch": 5, + "seq": 1, + "kind": "text", + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.629429625987541, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + "image_measured": [ + { + "kind": "image", + "wall_s": 3.647124751994852, + "answer": "Red", + "expected": "Red", + "stage_event": { + "request_id": "image-493871067080776", + "stage_id": 0, + "epoch": 6, + "seq": 1, + "kind": "text", + "worker_generation": "b9aa9595fd0f48b081c982d1cb597a69", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 3.640121451986488, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 37, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_image_wall_s": 3.647124751994852, + "nearest_rank_p95_image_wall_s": 3.647124751994852, + "status": "completed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 29979381760 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.server.log b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.server.log new file mode 100644 index 000000000..9cbfc905e --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/smoke4.server.log @@ -0,0 +1,426 @@ +warning: no usable GPU found, --gpu-layers option will be ignored +warning: one possible reason is that llama.cpp was compiled without GPU support +warning: consult docs/build.md for compilation instructions +0.00.003.118 I cmn common_param: common_params_print_info: build 10849 (e71b80510) with GNU 11.4.0 for Linux x86_64 +0.00.003.141 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.003.142 I cmn common_param: device_info: +0.00.003.155 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (31647 MiB, 31647 MiB free) +0.00.003.224 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.003.403 I srv init: using 23 threads for HTTP server +0.00.003.422 I srv init: The UI is disabled +0.00.003.423 I srv init: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.003.481 W srv llama_server: ----------------- +0.00.003.499 W srv llama_server: CORS is set to allow all origins ('*') and no API key is set +0.00.003.499 W srv llama_server: this can be a security risk (cross-origin attacks) +0.00.003.499 W srv llama_server: more info: https://github.com/ggml-org/llama.cpp/pull/25655 +0.00.003.500 W srv llama_server: ----------------- +0.00.003.521 I srv start: binding port with default address family +0.00.004.988 I srv load_model: loading model '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf' +0.00.005.007 I srv load_model: local path '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf' +0.00.049.640 I llama_model_loader: loaded meta data with 40 key-value pairs and 851 tensors from /home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.049.705 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.049.717 I llama_model_loader: - kv 0: general.architecture str = qwen35 +0.00.049.718 I llama_model_loader: - kv 1: general.type str = model +0.00.049.723 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 20 +0.00.049.729 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.049.730 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.049.730 I llama_model_loader: - kv 5: general.name str = Qwen3.8-27B +0.00.049.730 I llama_model_loader: - kv 6: general.size_label str = 27B +0.00.049.730 I llama_model_loader: - kv 7: general.license str = apache-2.0 +0.00.049.755 I llama_model_loader: - kv 8: general.tags arr[str,1] = ["image-text-to-text"] +0.00.049.756 I llama_model_loader: - kv 9: qwen35.block_count u32 = 64 +0.00.049.756 I llama_model_loader: - kv 10: qwen35.context_length u32 = 262144 +0.00.049.756 I llama_model_loader: - kv 11: qwen35.embedding_length u32 = 5120 +0.00.049.757 I llama_model_loader: - kv 12: qwen35.feed_forward_length u32 = 17408 +0.00.049.757 I llama_model_loader: - kv 13: qwen35.attention.head_count u32 = 24 +0.00.049.758 I llama_model_loader: - kv 14: qwen35.attention.head_count_kv u32 = 4 +0.00.049.760 I llama_model_loader: - kv 15: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +0.00.049.762 I llama_model_loader: - kv 16: qwen35.rope.freq_base f32 = 10000000.000000 +0.00.049.763 I llama_model_loader: - kv 17: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.049.763 I llama_model_loader: - kv 18: qwen35.attention.key_length u32 = 256 +0.00.049.764 I llama_model_loader: - kv 19: qwen35.attention.value_length u32 = 256 +0.00.049.764 I llama_model_loader: - kv 20: qwen35.ssm.conv_kernel u32 = 4 +0.00.049.764 I llama_model_loader: - kv 21: qwen35.ssm.state_size u32 = 128 +0.00.049.765 I llama_model_loader: - kv 22: qwen35.ssm.group_count u32 = 16 +0.00.049.765 I llama_model_loader: - kv 23: qwen35.ssm.time_step_rank u32 = 48 +0.00.049.765 I llama_model_loader: - kv 24: qwen35.ssm.inner_size u32 = 6144 +0.00.049.769 I llama_model_loader: - kv 25: qwen35.attention.recurrent_layers arr[bool,64] = [true, true, true, false, true, true,... +0.00.049.769 I llama_model_loader: - kv 26: qwen35.full_attention_interval u32 = 4 +0.00.049.769 I llama_model_loader: - kv 27: qwen35.rope.dimension_count u32 = 64 +0.00.049.769 I llama_model_loader: - kv 28: tokenizer.ggml.model str = gpt2 +0.00.049.769 I llama_model_loader: - kv 29: tokenizer.ggml.pre str = qwen35 +0.00.065.537 I llama_model_loader: - kv 30: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +0.00.068.926 I llama_model_loader: - kv 31: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +0.00.084.155 I llama_model_loader: - kv 32: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +0.00.084.211 I llama_model_loader: - kv 33: tokenizer.ggml.eos_token_id u32 = 248046 +0.00.084.212 I llama_model_loader: - kv 34: tokenizer.ggml.padding_token_id u32 = 248044 +0.00.084.213 I llama_model_loader: - kv 35: tokenizer.ggml.bos_token_id u32 = 248044 +0.00.084.213 I llama_model_loader: - kv 36: tokenizer.ggml.add_bos_token bool = false +0.00.084.219 I llama_model_loader: - kv 37: tokenizer.chat_template str = {%- set image_count = namespace(value... +0.00.084.219 I llama_model_loader: - kv 38: general.quantization_version u32 = 2 +0.00.084.220 I llama_model_loader: - kv 39: general.file_type u32 = 15 +0.00.084.228 I llama_model_loader: - type f32: 353 tensors +0.00.084.229 I llama_model_loader: - type q8_0: 288 tensors +0.00.084.230 I llama_model_loader: - type q4_K: 193 tensors +0.00.084.230 I llama_model_loader: - type q6_K: 17 tensors +0.00.084.235 I print_info: file format = GGUF V3 (latest) +0.00.084.238 I print_info: file type = Q4_K - Medium +0.00.084.244 I print_info: file size = 17.66 GiB (5.64 BPW) +0.00.323.776 I load: 0 unused tokens +0.00.387.123 I load: printing all EOG tokens: +0.00.387.159 I load: - 248044 ('<|endoftext|>') +0.00.387.161 I load: - 248046 ('<|im_end|>') +0.00.387.162 I load: - 248063 ('<|fim_pad|>') +0.00.387.162 I load: - 248064 ('<|repo_name|>') +0.00.387.162 I load: - 248065 ('<|file_sep|>') +0.00.387.417 I load: special tokens cache size = 33 +0.00.441.887 I load: token to piece cache size = 1.7581 MB +0.00.441.946 I print_info: arch = qwen35 +0.00.441.946 I print_info: vocab_only = 0 +0.00.441.947 I print_info: no_alloc = 0 +0.00.441.947 I print_info: n_ctx_train = 262144 +0.00.441.950 I print_info: n_embd_inp = 5120 +0.00.441.950 I print_info: n_embd = 5120 +0.00.441.951 I print_info: n_embd_out = 5120 +0.00.441.951 I print_info: n_layer = 64 +0.00.441.952 I print_info: n_layer_all = 64 +0.00.441.971 I print_info: n_head = 24 +0.00.441.973 I print_info: n_head_kv = 4 +0.00.441.973 I print_info: n_rot = 64 +0.00.441.974 I print_info: n_swa = 0 +0.00.441.975 I print_info: is_swa_any = 0 +0.00.441.976 I print_info: non_causal_type = 0 +0.00.441.976 I print_info: n_embd_head_k = 256 +0.00.441.976 I print_info: n_embd_head_v = 256 +0.00.441.978 I print_info: n_gqa = 6 +0.00.441.983 I print_info: n_embd_k_gqa = 1024 +0.00.441.987 I print_info: n_embd_v_gqa = 1024 +0.00.441.987 I print_info: f_norm_eps = 0.0e+00 +0.00.441.989 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.441.989 I print_info: f_clamp_kqv = 0.0e+00 +0.00.441.989 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.441.990 I print_info: f_logit_scale = 0.0e+00 +0.00.441.990 I print_info: f_attn_scale = 0.0e+00 +0.00.441.990 I print_info: f_attn_value_scale = 0.0000 +0.00.441.992 I print_info: n_ff = 17408 +0.00.441.992 I print_info: n_expert = 0 +0.00.441.992 I print_info: n_expert_used = 0 +0.00.441.992 I print_info: n_expert_groups = 0 +0.00.441.993 I print_info: n_group_used = 0 +0.00.441.993 I print_info: causal attn = 1 +0.00.441.993 I print_info: pooling type = -1 +0.00.441.993 I print_info: rope type = 40 +0.00.441.993 I print_info: rope scaling = linear +0.00.441.994 I print_info: freq_base_train = 10000000.0 +0.00.441.995 I print_info: freq_scale_train = 1 +0.00.441.995 I print_info: n_ctx_orig_yarn = 262144 +0.00.441.995 I print_info: rope_yarn_log_mul = 0.0000 +0.00.441.995 I print_info: rope_finetuned = unknown +0.00.441.995 I print_info: mrope sections = [11, 11, 10, 0] +0.00.441.995 I print_info: ssm_d_conv = 4 +0.00.441.996 I print_info: ssm_d_inner = 6144 +0.00.441.996 I print_info: ssm_d_state = 128 +0.00.441.996 I print_info: ssm_dt_rank = 48 +0.00.441.996 I print_info: ssm_n_group = 16 +0.00.441.996 I print_info: ssm_dt_b_c_rms = 0 +0.00.441.997 I print_info: model type = 27B +0.00.441.998 I print_info: model params = 26.90 B +0.00.441.998 I print_info: general.name = Qwen3.8-27B +0.00.442.002 I print_info: vocab type = BPE +0.00.442.004 I print_info: n_vocab = 248320 +0.00.442.004 I print_info: n_merges = 247587 +0.00.442.004 AMX is not ready to be used! +I print_info: BOS token = 248044 '<|endoftext|>' +0.00.442.004 I print_info: EOS token = 248046 '<|im_end|>' +0.00.442.005 I print_info: EOT token = 248046 '<|im_end|>' +0.00.442.005 I print_info: PAD token = 248044 '<|endoftext|>' +0.00.442.006 I print_info: LF token = 198 'Ċ' +0.00.442.006 I print_info: FIM PRE token = 248060 '<|fim_prefix|>' +0.00.442.006 I print_info: FIM SUF token = 248062 '<|fim_suffix|>' +0.00.442.006 I print_info: FIM MID token = 248061 '<|fim_middle|>' +0.00.442.006 I print_info: FIM PAD token = 248063 '<|fim_pad|>' +0.00.442.006 I print_info: FIM REP token = 248064 '<|repo_name|>' +0.00.442.007 I print_info: FIM SEP token = 248065 '<|file_sep|>' +0.00.442.007 I print_info: EOG token = 248044 '<|endoftext|>' +0.00.442.008 I print_info: EOG token = 248046 '<|im_end|>' +0.00.442.008 I print_info: EOG token = 248063 '<|fim_pad|>' +0.00.442.008 I print_info: EOG token = 248064 '<|repo_name|>' +0.00.442.008 I print_info: EOG token = 248065 '<|file_sep|>' +0.00.442.008 I print_info: max token length = 256 +0.00.442.012 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap) +0.00.513.398 I load_tensors: CPU_Mapped model buffer size = 18084.41 MiB +0.00.518.071 I cmn common_init_: added <|endoftext|> logit bias = -inf +0.00.518.100 I cmn common_init_: added <|im_end|> logit bias = -inf +0.00.518.101 I cmn common_init_: added <|fim_pad|> logit bias = -inf +0.00.518.101 I cmn common_init_: added <|repo_name|> logit bias = -inf +0.00.518.102 I cmn common_init_: added <|file_sep|> logit bias = -inf +0.00.518.225 I llama_context: constructing llama_context +0.00.518.251 I llama_context: n_seq_max = 1 +0.00.518.251 I llama_context: n_ctx = 2048 +0.00.518.251 I llama_context: n_ctx_seq = 2048 +0.00.518.251 I llama_context: n_batch = 2048 +0.00.518.252 I llama_context: n_ubatch = 512 +0.00.518.252 I llama_context: causal_attn = 1 +0.00.518.254 I llama_context: flash_attn = auto +0.00.518.254 I llama_context: kv_unified = false +0.00.518.258 I llama_context: freq_base = 10000000.0 +0.00.518.258 I llama_context: freq_scale = 1 +0.00.518.258 I llama_context: n_rs_seq = 0 +0.00.518.259 I llama_context: n_outputs_max = 1 +0.00.518.259 I llama_context: n_outputs_max_per_seq = 1 +0.00.518.259 I llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.00.518.327 I llama_context: CPU output buffer size = 0.95 MiB +0.00.518.448 I llama_kv_cache: CPU KV buffer size = 128.00 MiB +0.00.605.242 I llama_kv_cache: size = 128.00 MiB ( 2048 cells, 16 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB +0.00.605.270 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.00.605.271 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.00.713.625 I llama_memory_recurrent: CPU RS buffer size = 149.62 MiB +0.00.713.683 I llama_memory_recurrent: size = 149.62 MiB ( 1 cells, 64 layers, 1 seqs 0 rs_seq), R (f32): 5.62 MiB, S (f32): 144.00 MiB, P (f32): 0.00 MiB +0.00.713.706 I sched_reserve: reserving ... +0.00.731.562 I resolve_fused_ops: Flash Attention enabled +0.00.731.592 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.00.732.337 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.00.732.977 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.00.733.582 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.00.736.118 I sched_reserve: CPU compute buffer size = 134.02 MiB +0.00.736.144 I sched_reserve: graph nodes = 3751 +0.00.736.145 I sched_reserve: graph splits = 1 +0.00.736.148 I sched_reserve: reserve took 22.43 ms, sched copies = 1 +0.00.736.211 I cmn init: llama threadpool init, n_threads = 12 +0.00.736.250 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.01.131.297 I clip_model_loader: model name: Qwen3.8-27B +0.01.131.335 I clip_model_loader: description: +0.01.131.338 I clip_model_loader: GGUF version: 3 +0.01.131.341 I clip_model_loader: alignment: 32 +0.01.131.341 I clip_model_loader: n_tensors: 334 +0.01.131.342 I clip_model_loader: n_kv: 26 +0.01.131.343 I +0.01.131.346 I clip_model_loader: has vision encoder +0.01.131.647 I clip_ctx: CLIP using CPU backend +0.01.132.241 W load_hparams: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks +0.01.132.261 W load_hparams: if you encounter problems with accuracy, try adding --image-min-tokens 1024 +0.01.132.261 W load_hparams: more info: https://github.com/ggml-org/llama.cpp/issues/16842 + +0.01.132.262 I load_hparams: projector: qwen3vl_merger +0.01.132.262 I load_hparams: n_embd: 1152 +0.01.132.263 I load_hparams: n_head: 16 +0.01.132.263 I load_hparams: n_ff: 4304 +0.01.132.263 I load_hparams: n_layer: 27 +0.01.132.263 I load_hparams: ffn_op: gelu +0.01.132.263 I load_hparams: projection_dim: 5120 +0.01.132.264 I +--- vision hparams --- +0.01.132.264 I load_hparams: image_size: 768 +0.01.132.264 I load_hparams: patch_size: 16 +0.01.132.264 I load_hparams: has_llava_proj: 0 +0.01.132.264 I load_hparams: minicpmv_version: 0 +0.01.132.264 I load_hparams: n_merge: 2 +0.01.132.264 I load_hparams: n_wa_pattern: 0 +0.01.132.264 I load_hparams: image_min_pixels: 8192 +0.01.132.265 I load_hparams: image_max_pixels: 4194304 +0.01.132.265 I +0.01.132.270 I load_hparams: model size: 600.08 MiB +0.01.132.272 I load_hparams: metadata size: 0.12 MiB +0.01.644.456 I get_dummy_batch: warmup with image size = 1472 x 1472 +0.01.674.365 I get_dummy_batch: warmup with image size = 1472 x 1472 +0.01.683.463 I reserve_compute_meta: CPU compute buffer size = 248.10 MiB +0.01.683.507 I reserve_compute_meta: graph splits = 1, nodes = 823 +0.01.683.534 I warmup: flash attention is enabled +0.01.686.331 I srv load_model: loaded multimodal model, '/home/zhout/project/edge_infer/models/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf' +0.02.132.137 I cmn common_conte: the context does not support partial sequence removal +0.02.143.173 I srv load_model: speculative decoding will use checkpoints +0.02.143.214 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 2048, kv_unified = 'false' +0.02.143.233 I spec common_specu: no implementations specified for speculative decoding +0.02.143.253 I slot load_model: id 0 | task -1 | new slot, n_ctx = 2048 +0.02.143.332 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.02.143.336 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.02.143.336 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.02.143.362 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.02.147.190 I srv init: init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + + + + +' +0.02.147.691 I srv init: init: chat template, thinking = 0 +0.02.147.728 I srv init: preserve_reasoning kwarg: true +0.02.147.728 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.02.147.772 I srv llama_server: model loaded +0.02.147.795 I srv llama_server: listening on http://127.0.0.1:34289 +0.02.147.957 I srv update_slots: all slots are idle +0.02.241.520 I srv server_strea: conv_id= (empty=1) +0.02.241.770 I srv operator(): chat format: peg-native +0.02.241.991 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.02.242.009 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.02.242.068 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.02.242.098 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.02.242.099 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.02.242.114 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.02.242.117 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.03.349.092 I slot operator(): id 0 | task 0 | cached n_tokens = 15, memory_seq_rm [15, end) +0.03.349.182 I slot init_sampler: id 0 | task 0 | init sampler, took 0.02 ms, tokens: text = 19, total = 19 +0.03.444.604 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.04.185.775 I slot print_timing: id 0 | task 0 | prompt eval time = 1632.09 ms / 19 tokens ( 85.90 ms per token, 11.64 tokens per second) +0.04.185.821 I slot print_timing: id 0 | task 0 | eval time = 311.54 ms / 2 tokens ( 311.54 ms per token, 3.21 tokens per second) +0.04.185.822 I slot print_timing: id 0 | task 0 | total time = 1943.62 ms / 21 tokens +0.04.185.831 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.04.185.913 I slot release: id 0 | task 0 | stop processing: n_tokens = 20, truncated = 0 +0.04.185.929 I srv update_slots: all slots are idle +0.04.198.796 I srv server_strea: conv_id= (empty=1) +0.04.199.218 I srv operator(): chat format: peg-native +0.04.199.376 I slot get_availabl: id 0 | task -1 | - checking sim = 0.081 (3/37) > 0.100 +0.04.199.397 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 493860006106 +0.04.199.456 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.04.199.482 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.04.199.483 I slot launch_slot_: id 0 | task 4 | processing task, is_child = 0 +0.04.199.490 I slot operator(): id 0 | task 4 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.04.199.492 I slot operator(): id 0 | task 4 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.04.208.853 I slot operator(): id 0 | task 4 | cached n_tokens = 0, memory_seq_rm [0, end) +0.05.509.411 I slot operator(): id 0 | task 4 | cached n_tokens = 18, memory_seq_rm [18, end) +0.05.509.500 I slot process_mtmd: id 0 | task 4 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.05.571.320 I decoding image batch 1/1, n_tokens_batch = 9 +0.05.571.432 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.05.571.459 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.06.318.400 I image decoded (batch 1/1) in 747 ms +0.06.318.890 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.06.318.959 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.06.912.682 I slot operator(): id 0 | task 4 | cached n_tokens = 33, memory_seq_rm [27, end) +0.06.912.760 I slot init_sampler: id 0 | task 4 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.06.980.991 I slot create_check: id 0 | task 4 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.07.740.128 I slot print_timing: id 0 | task 4 | prompt eval time = 3227.13 ms / 37 tokens ( 87.22 ms per token, 11.47 tokens per second) +0.07.740.161 I slot print_timing: id 0 | task 4 | eval time = 313.48 ms / 2 tokens ( 313.48 ms per token, 3.19 tokens per second) +0.07.740.162 I slot print_timing: id 0 | task 4 | total time = 3540.61 ms / 39 tokens +0.07.740.164 I slot print_timing: id 0 | task 4 | graphs reused = 1 +0.07.740.217 I slot release: id 0 | task 4 | stop processing: n_tokens = 38, truncated = 0 +0.07.740.245 I srv update_slots: all slots are idle +0.07.755.538 I srv server_strea: conv_id= (empty=1) +0.07.755.707 I srv operator(): chat format: peg-native +0.07.755.921 I slot get_availabl: id 0 | task -1 | - checking sim = 0.158 (3/19) > 0.100 +0.07.755.942 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 0.158 (> 0.100 thold), f_keep = 0.079 +0.07.755.992 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.07.756.016 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.07.756.017 I slot launch_slot_: id 0 | task 9 | processing task, is_child = 0 +0.07.756.024 I slot operator(): id 0 | task 9 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.07.756.026 I slot operator(): id 0 | task 9 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.07.764.483 I slot operator(): id 0 | task 9 | cached n_tokens = 0, memory_seq_rm [0, end) +0.08.856.525 I slot operator(): id 0 | task 9 | cached n_tokens = 15, memory_seq_rm [15, end) +0.08.856.627 I slot init_sampler: id 0 | task 9 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.08.921.613 I slot create_check: id 0 | task 9 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.09.648.486 I slot print_timing: id 0 | task 9 | prompt eval time = 1582.94 ms / 19 tokens ( 83.31 ms per token, 12.00 tokens per second) +0.09.648.550 I slot print_timing: id 0 | task 9 | eval time = 309.48 ms / 2 tokens ( 309.48 ms per token, 3.23 tokens per second) +0.09.648.552 I slot print_timing: id 0 | task 9 | total time = 1892.42 ms / 21 tokens +0.09.648.554 I slot print_timing: id 0 | task 9 | graphs reused = 1 +0.09.648.623 I slot release: id 0 | task 9 | stop processing: n_tokens = 20, truncated = 0 +0.09.648.652 I srv update_slots: all slots are idle +0.09.659.990 I srv server_strea: conv_id= (empty=1) +0.09.660.185 I srv operator(): chat format: peg-native +0.09.660.340 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (19/19) > 0.100 +0.09.660.363 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.950 +0.09.660.413 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.09.660.417 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.09.660.418 I slot launch_slot_: id 0 | task 13 | processing task, is_child = 0 +0.09.660.425 I slot operator(): id 0 | task 13 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 19 +0.09.660.427 I slot operator(): id 0 | task 13 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.09.668.964 I slot operator(): id 0 | task 13 | cached n_tokens = 0, memory_seq_rm [0, end) +0.10.799.794 I slot operator(): id 0 | task 13 | cached n_tokens = 15, memory_seq_rm [15, end) +0.10.799.894 I slot init_sampler: id 0 | task 13 | init sampler, took 0.01 ms, tokens: text = 19, total = 19 +0.10.866.949 I slot create_check: id 0 | task 13 | created context checkpoint 1 of 32 (pos_min = 14, pos_max = 14, n_tokens = 15, size = 149.626 MiB) +0.11.596.501 I slot print_timing: id 0 | task 13 | prompt eval time = 1635.85 ms / 19 tokens ( 86.10 ms per token, 11.61 tokens per second) +0.11.596.539 I slot print_timing: id 0 | task 13 | eval time = 300.20 ms / 2 tokens ( 300.20 ms per token, 3.33 tokens per second) +0.11.596.540 I slot print_timing: id 0 | task 13 | total time = 1936.05 ms / 21 tokens +0.11.596.541 I slot print_timing: id 0 | task 13 | graphs reused = 1 +0.11.596.593 I slot release: id 0 | task 13 | stop processing: n_tokens = 20, truncated = 0 +0.11.596.610 I srv update_slots: all slots are idle +0.11.608.474 I srv server_strea: conv_id= (empty=1) +0.11.608.847 I srv operator(): chat format: peg-native +0.11.609.003 I slot get_availabl: id 0 | task -1 | - checking sim = 0.081 (3/37) > 0.100 +0.11.609.024 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 493867416785 +0.11.609.076 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.11.609.084 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.11.609.087 I slot launch_slot_: id 0 | task 17 | processing task, is_child = 0 +0.11.609.094 I slot operator(): id 0 | task 17 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.11.609.096 I slot operator(): id 0 | task 17 | erased invalidated context checkpoint (pos_min = 14, pos_max = 14, n_tokens = 15, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.11.618.134 I slot operator(): id 0 | task 17 | cached n_tokens = 0, memory_seq_rm [0, end) +0.12.975.520 I slot operator(): id 0 | task 17 | cached n_tokens = 18, memory_seq_rm [18, end) +0.12.975.603 I slot process_mtmd: id 0 | task 17 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.13.039.366 I decoding image batch 1/1, n_tokens_batch = 9 +0.13.039.573 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.13.039.605 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.13.801.404 I image decoded (batch 1/1) in 762 ms +0.13.801.710 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.13.801.767 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.14.389.213 I slot operator(): id 0 | task 17 | cached n_tokens = 33, memory_seq_rm [27, end) +0.14.389.287 I slot init_sampler: id 0 | task 17 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.14.500.054 I slot create_check: id 0 | task 17 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.15.235.429 I slot print_timing: id 0 | task 17 | prompt eval time = 3321.23 ms / 37 tokens ( 89.76 ms per token, 11.14 tokens per second) +0.15.235.477 I slot print_timing: id 0 | task 17 | eval time = 305.07 ms / 2 tokens ( 305.07 ms per token, 3.28 tokens per second) +0.15.235.478 I slot print_timing: id 0 | task 17 | total time = 3626.31 ms / 39 tokens +0.15.235.480 I slot print_timing: id 0 | task 17 | graphs reused = 1 +0.15.235.541 I slot release: id 0 | task 17 | stop processing: n_tokens = 38, truncated = 0 +0.15.235.562 I srv update_slots: all slots are idle +0.15.260.146 I srv server_strea: conv_id= (empty=1) +0.15.260.607 I srv operator(): chat format: peg-native +0.15.260.769 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (37/37) > 0.100 +0.15.260.788 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.974 +0.15.260.842 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.15.260.866 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 20, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.15.260.867 I slot launch_slot_: id 0 | task 22 | processing task, is_child = 0 +0.15.260.875 I slot operator(): id 0 | task 22 | new prompt, n_ctx_slot = 2048, n_keep = 0, task.n_tokens = 37 +0.15.260.877 I slot operator(): id 0 | task 22 | erased invalidated context checkpoint (pos_min = 26, pos_max = 26, n_tokens = 33, n_swa = 0, pos_next = 0, size = 149.626 MiB) +0.15.269.453 I slot operator(): id 0 | task 22 | cached n_tokens = 0, memory_seq_rm [0, end) +0.16.610.837 I slot operator(): id 0 | task 22 | cached n_tokens = 18, memory_seq_rm [18, end) +0.16.610.908 I slot process_mtmd: id 0 | task 22 | encoding mtmd batch from idx = 18, n_chunks = 1 +0.16.676.869 I decoding image batch 1/1, n_tokens_batch = 9 +0.16.676.977 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.16.677.009 W find_slot: non-consecutive token position 18 after 17 for sequence 0 with 9 new tokens +0.17.436.949 I image decoded (batch 1/1) in 760 ms +0.17.437.236 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.17.437.295 W find_slot: non-consecutive token position 26 after 18 for sequence 0 with 6 new tokens +0.18.061.847 I slot operator(): id 0 | task 22 | cached n_tokens = 33, memory_seq_rm [27, end) +0.18.061.923 I slot init_sampler: id 0 | task 22 | init sampler, took 0.01 ms, tokens: text = 28, total = 37 +0.18.131.774 I slot create_check: id 0 | task 22 | created context checkpoint 1 of 32 (pos_min = 26, pos_max = 26, n_tokens = 33, size = 149.626 MiB) +0.18.887.462 I slot print_timing: id 0 | task 22 | prompt eval time = 3290.39 ms / 37 tokens ( 88.93 ms per token, 11.24 tokens per second) +0.18.887.504 I slot print_timing: id 0 | task 22 | eval time = 336.17 ms / 2 tokens ( 336.17 ms per token, 2.97 tokens per second) +0.18.887.506 I slot print_timing: id 0 | task 22 | total time = 3626.55 ms / 39 tokens +0.18.887.507 I slot print_timing: id 0 | task 22 | graphs reused = 1 +0.18.887.569 I slot release: id 0 | task 22 | stop processing: n_tokens = 38, truncated = 0 +0.18.887.594 I srv update_slots: all slots are idle +0.18.894.469 I srv operator(): operator(): cleaning up before exit... +0.18.895.991 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +0.18.896.031 I common_memory_breakdown_print: | - Host | 18496 = 18084 + 277 + 134 | diff --git a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md index 36938eac0..5a41cf158 100644 --- a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md +++ b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md @@ -16,7 +16,7 @@ The [native Windows RTX Qwen3.8-27B NVFP4 attempt](../../../e2e_expansion_202609 [A native Windows RTX Qwen3.8-27B FP8 recovery](../../../e2e_expansion_20260924/evidence/qwen38_omni_windows_rtx/README.md) completed scoped Omni text and synthetic image-to-text requests via an explicit Triton FP8 linear backend. One warmup plus 20/20 serial measured requests of each kind produced stable outputs at 3.070/3.214 s text and 4.184/4.380 s image p50/p95. The separate NVFP4 Marlin/PTX failure remains unresolved. -A separate pinned [Qwen3.8-27B Q4_K_M GGUF CPU path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_cpu/README.md) and [Radeon 890M path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_radeon/README.md) now pass complete short text and synthetic red-image requests through Omni on native Windows. Each passed 20/20 serial measured requests per type after one warmup per type. CPU p50/p95 complete-request wall time was 1.656/1.697 s text and 2.925/2.976 s image; Radeon was 3.101/3.181 s text and 5.160/5.220 s image. The backend verified hashes, 28 GiB host admission, actual CPU or Radeon model and vision placement, and released the ledger. These two-token-output tests do not qualify long generation, broad quality or a Radeon speedup; the earlier WSL CPU and AMD NPU cells remain separate. +A separate pinned [Qwen3.8-27B Q4_K_M GGUF native Windows CPU path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_cpu/README.md), [Radeon 890M path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_windows_radeon/README.md), and [WSL CPU no-repack path](../../../e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md) now pass complete short text and synthetic red-image requests through Omni. Each passed 20/20 serial measured requests per type after one warmup per type. Native Windows CPU p50/p95 complete-request wall time was 1.656/1.697 s text and 2.925/2.976 s image; Radeon was 3.101/3.181 s text and 5.160/5.220 s image; WSL CPU no-repack was 1.934/1.983 s text and 3.642/3.789 s image. The backend verified pinned hashes, explicit host admission, actual model and vision placement, and ledger release. The WSL route uses a different llama.cpp build, repack setting and RAM quota, so the timings are not a paired OS comparison. These two-token-output tests do not qualify long generation, broad quality or a Radeon speedup; the AMD NPU cell remains separate component-only evidence. A [native Windows RTX Qwen3-TTS Omni hybrid route](../../../e2e_expansion_20260924/evidence/qwen_tts_windows_rtx/README.md) now returns complete 24 kHz WAVs with talker/codec on RTX Vulkan0 and predictor on CPU FP32. One warmup plus 20/20 serial outputs had identical PCM at p50/p95 complete-request wall 3.153/3.231 s for 2.72 s audio. Public audio, explicit host/GPU admission, refusal of an undersized GPU reservation and fresh-stage recovery after server-started cancellation passed. The PCM differs from the Radeon hybrid, and the complete-WAV RTF remains above 1; the existing CUDA sustained playback findings are separate. @@ -54,7 +54,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 | |---|---|---|---|---|---| -| PC HX370: CPU / WSL | scoped E2E: Omni GGUF or BF16 Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF combined image+audio → text+speech | synthetic policy pass: Omni CPU | +| PC HX370: CPU / WSL | scoped E2E: Omni GGUF or BF16 Spark text | scoped E2E: Omni CPU text → complete WAV | scoped E2E: Omni Q4_K_M GGUF CPU text+image (20 each; no repack) | scoped E2E: Omni GGUF combined image+audio → text+speech | synthetic policy pass: Omni CPU | | PC HX370: CPU / native Windows | scoped E2E: Omni Spark text; fresh restart after in-flight cancel | scoped E2E: Omni CPU text → complete WAV | scoped E2E: Omni Q4_K_M GGUF CPU text+image (20 each) | scoped E2E: Omni GGUF image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU | | PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA | | PC HX370 + RTX 5090 Laptop / Windows | scoped E2E: Omni Spark 1.7B GGUF text on RTX Vulkan; fresh restart after in-flight cancel (separate 4B BF16 CUDA profile) | scoped E2E: Omni RTX Vulkan talker/codec + CPU predictor complete WAV and restart (separate CUDA stream has playback jitter) | scoped E2E: Omni FP8 Triton text+image on RTX; separate NVFP4 Windows startup failed | scoped E2E: text/image/audio → speech | synthetic policy pass: admitted Omni CUDA | @@ -73,7 +73,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i |---|---|---|---|---| | PC HX370: CPU / WSL | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST; BF16_CPU_ACCEPTANCE_PARITY_PROFILE_AND_RESTART_PASS | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran on a CPU-only Linux llama.cpp server through the current Omni 0.29 `StageRuntime`/`StagePool` whole-session backend. Both named prompts passed (`Paris`, `1511`); one warmup plus 20/20 serial inventory requests had nearest-rank complete-request wall p50/p95 7.82/7.97 s, excluding 1.61 s startup. The stage verified model/server hashes, explicit CPU-only tensor buffers (`CPU_Mapped`, `CPU_REPACK`) and no usable GPU, reserved 4 GiB host RAM, refused an over-context request without truncation, and after in-flight cancellation returned no stale output or reservation. A separate public `AsyncOmni.generate` text request passed. The Linux CPU-only build omits the Windows 0/29 offload line; the verifier was extended to require positive CPU buffer evidence rather than inferring CPU from an absent GPU count. A separate pinned 1.7B BF16 checkpoint was completed and verified against the upstream index. The current Omni vLLM CPU text path then passed 12/12 named prompts at 128 generated tokens, cancellation cleanup and exact 12/12 greedy token-ID parity against standalone vLLM on the same BF16 checkpoint. The fork-branch acceptance load was 30.36 s with approximately 6.45 GiB process-tree RSS load delta; the plan reserved 7.64 GiB. One warmup plus 20 serial BF16 complete requests (28 prompt tokens, 64 output tokens) measured nearest-rank wall p50/p95 5.411/5.751 s and TTFT 0.129/0.146 s on HX370 WSL CPU. Separate neighboring-checkout evidence retained a 19.88 s acceptance load and 4.808/5.089 s profile; it is not the fork-branch timing. The installed vLLM was 0.28.0 with 0.29.0rc2 Omni source, so that version pairing is the tested boundary. The GGUF result and historical INT8 result remain distinct artifacts. A [BF16 restart probe](../../../e2e_expansion_20260924/evidence/spark_bf16_wsl_cpu/restart_branch_report.json) cancelled after eight token events, observed no late output, rejected the retired session handle and reproduced the exact 128-token sequence in a fresh session. None establishes broad text quality, incremental GGUF token streaming, concurrent recovery, loading peak or sustained performance. | Broaden text quality and context, qualify a matching Omni/vLLM version pair, then test incremental streaming, concurrent recovery and sustained load; [GGUF evidence](../../../e2e_expansion_20260923/evidence/spark_omni_wsl_cpu/README.md) and [BF16 evidence](../../../e2e_expansion_20260924/evidence/spark_bf16_wsl_cpu/README.md). | | PC HX370: CPU / WSL | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The earlier vLLM 0.28 CPU path passed four short streams but had four simulated playback underruns in one measured request; its wheel remains mismatched with Omni 0.29. Separately, the pinned BF16 0.6B CustomVoice checkpoint ran two named text-to-complete-WAV requests through an Omni 0.29 whole-session stage with an isolated CPU worker on WSL. One warmup plus 20 serial measured requests returned identical 3.44 s audio; nearest-rank complete-request wall p50/p95 was 8.27/8.47 s (RTF 2.40/2.46), excluding 19.82 s startup. Pinned Whisper tiny.en transcribed the first measured WAV exactly (WER 0, one-output intelligibility proxy). Public `AsyncOmni` audio output passed; the stage verified weights, interpreter and CPU placement, reserved 10 GiB host RAM, refused 4 GiB before launch, and drained in-flight cancellation with zero remaining reservation. WSL PCM differs from the same-checkpoint native Windows outputs despite the declared settings; its cause is unqualified. Sampled process-tree RSS reached 3.39 GB, not a loading peak. This worker delivers only a complete WAV, not a playable stream. | Qualify cross-OS numerical/quality difference, playable streaming without underruns, post-cancel restart, loading peak, concurrency and sustained power/thermal behavior; [WSL Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_wsl_cpu/README.md), [earlier stream evidence](../../../e2e_recovery_20260923/README.md). | -| PC HX370: CPU / WSL | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No compatible CPU artifact/deployment for this model was found in the checked model roots. Available Qwen 27B FP8/NVFP4 CUDA artifacts do not establish a CPU route. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | +| PC HX370: CPU / WSL | Qwen3.8-27B | SCOPED_E2E_OMNI_GGUF_CPU_TEXT_IMAGE_PROFILE20 | The pinned ggml-org/Qwen3.8-27B-GGUF Q4_K_M language model and Q8_0 vision projector at revision efbb3b1 ran through an admitted whole-session Omni StageRuntime/llama.cpp b10849 CPU-only backend on WSL. A 28 GiB reservation was twice refused before loading when available WSL RAM was below it; a 27 GiB attempt loaded but exceeded its RSS budget. Explicit --no-repack removed the 9,180 MiB CPU repack buffer without changing weights/precision. With a 24 GiB reservation, a smoke and one warmup of each type plus 20/20 serial measured text outputs ready and 20/20 synthetic image outputs Red passed. Nearest-rank complete-request p50/p95 was 1.934/1.983 s text and 3.642/3.789 s image, excluding 11.45 s startup. The stage verified hashes, CPU model and vision placement, a loaded RSS sample of 20,026,531,840 bytes, and ledger release. This two-token-output case does not qualify long context, broad image quality, public entrypoint, cancellation or sustained behavior. | Test public AsyncOmni, long-context/128-token output, cancellation/restart, sampled memory peak and sustained power; [WSL GGUF profile](../../../e2e_expansion_20260924/evidence/qwen38_gguf_wsl_cpu/README.md). | | PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The BF16/vLLM 0.28 three-stage path previously passed separate text, synthetic red-square image and synthetic 440 Hz audio inputs to text+speech; its 20-request serial **text-only** profile was p50/p95 21.86/22.97 s with swap and shutdown warnings. A separate pinned Q4_K_M/F16 GGUF worker now runs a **combined spoken question+image → text+speech** request through a bounded Omni StageRuntime/StagePool graph stage on WSL CPU, with installed vLLM 0.29 for the controller and CPU-only C++ model execution. All 20 measured requests after one warmup answered “The square in the image is red.”, emitted complete nonzero speech and a terminal audio event; nearest-rank complete-request wall p50/p95 was 22.67/24.83 s, speech duration 1.60–2.92 s, and maximum sampled worker RSS 13.70 GB. The first measured WAV independently transcribed exactly with pinned Whisper tiny.en (WER 0, one-output intelligibility proxy). Ten weight hashes and CPU placement were verified; 24 GiB shared RAM was reserved, 8 GiB was refused before launch, and in-flight cancellation left no stale output or reservation. Host RAM/swap samples covered only part of the profile. One synthetic fixture and variable speech PCM do not establish general quality, playable streaming, loading peak or sustained performance. | Validate real image/audio suites, speech alignment and quality, video input, concurrent admission, post-cancel restart, loading peak and sustained thermal behavior; [GGUF Omni raw evidence](../../../e2e_expansion_20260923/evidence/minicpmo_omni_wsl_cpu/README.md), [BF16/vLLM CPU evidence](../../../e2e_expansion_20260923/README.md), [earlier standalone GGUF run](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_cpu/README.md). | | PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS_OMNI; NOT_TASK_E2E | The real Place_Markpen checkpoint ran through an admitted Omni StageRuntime whole-policy stage with a CPU-only isolated worker. One warmup plus 20 serial patterned synthetic camera/state/noise requests returned identical finite `[1,50,32]` action chunks with terminal action events; nearest-rank complete-request wall p50/p95 was 3.175/3.219 s, excluding 40.03 s startup. The stage verified checkpoint/interpreter hashes and CPU placement, reserved 16 GiB of a declared 30 GiB WSL host-RAM pool, refused 8 GiB before launch, and released the reservation on shutdown. The earlier direct zero-input policy/CUDA comparison was a different fixture and remains separate. No real-observation task-quality tolerance exists. | Validate real observations/reference actions, physical units/order/step time, action deadlines, loading peak, concurrency and sustained power/thermal behavior; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/internvla_omni_policy/README.md), [earlier direct policy evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST; RESTART_AFTER_INFLIGHT_CANCEL_PASS | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran in native Windows llama.cpp CPU (`-dev none -ngl 0`) as a whole-session Omni StageRuntime/StagePool backend. `Paris` and the correct `1511` inventory answer passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 6.93/7.05 s. The public AsyncOmni short request passed. The stage verifies artifact hashes and CPU placement (0/29 layers offloaded), reserves 4 GiB host RAM, emits a terminal text event and releases reservation after in-flight cancellation. The separate prior standalone profile was 6.65/6.86 s. A separate confirmed server-start cancellation of the 120-item request terminated the worker, left no stale output or reservation, and a fresh StageRuntime with the same pinned artifact returned Paris under a new worker generation. This is fresh-stage recovery, not same-session continuation. No vLLM CPU attention repair, incremental stream, loading peak or sustained power/thermal gate is claimed. | Validate broader text/token quality, streaming, concurrent recovery, memory peak and sustained load; [restart evidence](../../../e2e_expansion_20260924/evidence/spark_omni_windows_restart/README.md), [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | diff --git a/vllm_omni/engine/backends/llamacpp.py b/vllm_omni/engine/backends/llamacpp.py index 4f491afc8..d5d281401 100644 --- a/vllm_omni/engine/backends/llamacpp.py +++ b/vllm_omni/engine/backends/llamacpp.py @@ -155,6 +155,8 @@ class LlamaCppTextStageClient(StageClientBase): "--reasoning", "off", "--no-webui", "--fit", "off", "--cache-ram", "0", "-lv", "4", ] + if config.get("disable_repack", False): + command.append("--no-repack") if self._mmproj is not None: command.extend(["--mmproj", str(self._mmproj)]) if self._placement == "cpu": @@ -243,6 +245,7 @@ class LlamaCppTextStageClient(StageClientBase): "reserved_bytes": dict(reservation.demands), "memory_pool": self._memory_pool, "memory_overhead_bytes": overhead, + "disable_repack": bool(config.get("disable_repack", False)), "context_tokens": self._context_tokens, "max_new_tokens": self._max_new_tokens, "max_io_bytes": self._max_io_bytes,